PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5
   6 use Cwd 'abs_path';
   7 use Digest::SHA;
   8 use Fcntl ':flock';
   9 use Fcntl;
  10 use File::Basename;
  11 use File::Copy qw(copy);
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use IO::Dir;
  16 use IO::File;
  17 use IO::Handle;
  18 use IO::Select;
  19 use IO::Socket::UNIX;
  20 use IPC::Open3;
  21 use JSON;
  22 use MIME::Base64;
  23 use POSIX;
  24 use Storable qw(dclone);
  25 use Time::HiRes qw(gettimeofday);
  26 use URI::Escape;
  27 use UUID;
  28
  29 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file cfs_lock_file);
  30 use PVE::DataCenterConfig;
  31 use PVE::Exception qw(raise raise_param_exc);
  32 use PVE::GuestHelpers;
  33 use PVE::INotify;
  34 use PVE::JSONSchema qw(get_standard_option);
  35 use PVE::ProcFSTools;
  36 use PVE::RPCEnvironment;
  37 use PVE::Storage;
  38 use PVE::SysFSTools;
  39 use PVE::Systemd;
  40 use PVE::Tools qw(run_command lock_file lock_file_full file_read_firstline dir_glob_foreach get_host_arch $IPV6RE);
  41
  42 use PVE::QMPClient;
  43 use PVE::QemuConfig;
  44 use PVE::QemuServer::Helpers qw(min_version);
  45 use PVE::QemuServer::Cloudinit;
  46 use PVE::QemuServer::Machine;
  47 use PVE::QemuServer::Memory;
  48 use PVE::QemuServer::Monitor qw(mon_cmd);
  49 use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr print_pcie_root_port);
  50 use PVE::QemuServer::USB qw(parse_usb_device);
  51
  52 my $EDK2_FW_BASE = '/usr/share/pve-edk2-firmware/';
  53 my $OVMF = {
  54     x86_64 => [
  55         "$EDK2_FW_BASE/OVMF_CODE.fd",
  56         "$EDK2_FW_BASE/OVMF_VARS.fd"
  57     ],
  58     aarch64 => [
  59         "$EDK2_FW_BASE/AAVMF_CODE.fd",
  60         "$EDK2_FW_BASE/AAVMF_VARS.fd"
  61     ],
  62 };
  63
  64 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  65
  66 my $QEMU_FORMAT_RE = qr/raw|cow|qcow|qcow2|qed|vmdk|cloop/;
  67
  68 # Note about locking: we use flock on the config file protect
  69 # against concurent actions.
  70 # Aditionaly, we have a 'lock' setting in the config file. This
  71 # can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
  72 # allowed when such lock is set. But you can ignore this kind of
  73 # lock with the --skiplock flag.
  74
  75 cfs_register_file('/qemu-server/',
  76                   \&parse_vm_config,
  77                   \&write_vm_config);
  78
  79 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
  80     description => "Some command save/restore state from this location.",
  81     type => 'string',
  82     maxLength => 128,
  83     optional => 1,
  84 });
  85
  86 PVE::JSONSchema::register_standard_option('pve-qm-image-format', {
  87     type => 'string',
  88     enum => [qw(raw cow qcow qed qcow2 vmdk cloop)],
  89     description => "The drive's backing file's data format.",
  90     optional => 1,
  91 });
  92
  93 PVE::JSONSchema::register_standard_option('pve-qemu-machine', {
  94         description => "Specifies the Qemu machine type.",
  95         type => 'string',
  96         pattern => '(pc|pc(-i440fx)?-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|q35|pc-q35-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|virt(?:-\d+(\.\d+)+)?(\+pve\d+)?)',
  97         maxLength => 40,
  98         optional => 1,
  99 });
 100
 101 #no warnings 'redefine';
 102
 103 sub cgroups_write {
 104    my ($controller, $vmid, $option, $value) = @_;
 105
 106    my $path = "/sys/fs/cgroup/$controller/qemu.slice/$vmid.scope/$option";
 107    PVE::ProcFSTools::write_proc_entry($path, $value);
 108
 109 }
 110
 111 my $nodename = PVE::INotify::nodename();
 112
 113 my $cpu_vendor_list = {
 114     # Intel CPUs
 115     486 => 'GenuineIntel',
 116     pentium => 'GenuineIntel',
 117     pentium2  => 'GenuineIntel',
 118     pentium3  => 'GenuineIntel',
 119     coreduo => 'GenuineIntel',
 120     core2duo => 'GenuineIntel',
 121     Conroe  => 'GenuineIntel',
 122     Penryn  => 'GenuineIntel',
 123     Nehalem  => 'GenuineIntel',
 124     'Nehalem-IBRS'  => 'GenuineIntel',
 125     Westmere => 'GenuineIntel',
 126     'Westmere-IBRS' => 'GenuineIntel',
 127     SandyBridge => 'GenuineIntel',
 128     'SandyBridge-IBRS' => 'GenuineIntel',
 129     IvyBridge => 'GenuineIntel',
 130     'IvyBridge-IBRS' => 'GenuineIntel',
 131     Haswell => 'GenuineIntel',
 132     'Haswell-IBRS' => 'GenuineIntel',
 133     'Haswell-noTSX' => 'GenuineIntel',
 134     'Haswell-noTSX-IBRS' => 'GenuineIntel',
 135     Broadwell => 'GenuineIntel',
 136     'Broadwell-IBRS' => 'GenuineIntel',
 137     'Broadwell-noTSX' => 'GenuineIntel',
 138     'Broadwell-noTSX-IBRS' => 'GenuineIntel',
 139     'Skylake-Client' => 'GenuineIntel',
 140     'Skylake-Client-IBRS' => 'GenuineIntel',
 141     'Skylake-Server' => 'GenuineIntel',
 142     'Skylake-Server-IBRS' => 'GenuineIntel',
 143     'Cascadelake-Server' => 'GenuineIntel',
 144     KnightsMill => 'GenuineIntel',
 145
 146
 147     # AMD CPUs
 148     athlon => 'AuthenticAMD',
 149     phenom  => 'AuthenticAMD',
 150     Opteron_G1  => 'AuthenticAMD',
 151     Opteron_G2  => 'AuthenticAMD',
 152     Opteron_G3  => 'AuthenticAMD',
 153     Opteron_G4  => 'AuthenticAMD',
 154     Opteron_G5  => 'AuthenticAMD',
 155     EPYC => 'AuthenticAMD',
 156     'EPYC-IBPB' => 'AuthenticAMD',
 157
 158     # generic types, use vendor from host node
 159     host => 'default',
 160     kvm32 => 'default',
 161     kvm64 => 'default',
 162     qemu32 => 'default',
 163     qemu64 => 'default',
 164     max => 'default',
 165 };
 166
 167 my @supported_cpu_flags = (
 168     'pcid',
 169     'spec-ctrl',
 170     'ibpb',
 171     'ssbd',
 172     'virt-ssbd',
 173     'amd-ssbd',
 174     'amd-no-ssb',
 175     'pdpe1gb',
 176     'md-clear',
 177     'hv-tlbflush',
 178     'hv-evmcs',
 179     'aes'
 180 );
 181 my $cpu_flag = qr/[+-](@{[join('|', @supported_cpu_flags)]})/;
 182
 183 my $cpu_fmt = {
 184     cputype => {
 185         description => "Emulated CPU type.",
 186         type => 'string',
 187         enum => [ sort { "\L$a" cmp "\L$b" } keys %$cpu_vendor_list ],
 188         default => 'kvm64',
 189         default_key => 1,
 190     },
 191     hidden => {
 192         description => "Do not identify as a KVM virtual machine.",
 193         type => 'boolean',
 194         optional => 1,
 195         default => 0
 196     },
 197     'hv-vendor-id' => {
 198         type => 'string',
 199         pattern => qr/[a-zA-Z0-9]{1,12}/,
 200         format_description => 'vendor-id',
 201         description => 'The Hyper-V vendor ID. Some drivers or programs inside Windows guests need a specific ID.',
 202         optional => 1,
 203     },
 204     flags => {
 205         description => "List of additional CPU flags separated by ';'."
 206                      . " Use '+FLAG' to enable, '-FLAG' to disable a flag."
 207                      . " Currently supported flags: @{[join(', ', @supported_cpu_flags)]}.",
 208         format_description => '+FLAG[;-FLAG...]',
 209         type => 'string',
 210         pattern => qr/$cpu_flag(;$cpu_flag)*/,
 211         optional => 1,
 212     },
 213 };
 214
 215 my $watchdog_fmt = {
 216     model => {
 217         default_key => 1,
 218         type => 'string',
 219         enum => [qw(i6300esb ib700)],
 220         description => "Watchdog type to emulate.",
 221         default => 'i6300esb',
 222         optional => 1,
 223     },
 224     action => {
 225         type => 'string',
 226         enum => [qw(reset shutdown poweroff pause debug none)],
 227         description => "The action to perform if after activation the guest fails to poll the watchdog in time.",
 228         optional => 1,
 229     },
 230 };
 231 PVE::JSONSchema::register_format('pve-qm-watchdog', $watchdog_fmt);
 232
 233 my $agent_fmt = {
 234     enabled => {
 235         description => "Enable/disable Qemu GuestAgent.",
 236         type => 'boolean',
 237         default => 0,
 238         default_key => 1,
 239     },
 240     fstrim_cloned_disks => {
 241         description => "Run fstrim after cloning/moving a disk.",
 242         type => 'boolean',
 243         optional => 1,
 244         default => 0
 245     },
 246     type => {
 247         description => "Select the agent type",
 248         type => 'string',
 249         default => 'virtio',
 250         optional => 1,
 251         enum => [qw(virtio isa)],
 252     },
 253 };
 254
 255 my $vga_fmt = {
 256     type => {
 257         description => "Select the VGA type.",
 258         type => 'string',
 259         default => 'std',
 260         optional => 1,
 261         default_key => 1,
 262         enum => [qw(cirrus qxl qxl2 qxl3 qxl4 none serial0 serial1 serial2 serial3 std virtio vmware)],
 263     },
 264     memory => {
 265         description => "Sets the VGA memory (in MiB). Has no effect with serial display.",
 266         type => 'integer',
 267         optional => 1,
 268         minimum => 4,
 269         maximum => 512,
 270     },
 271 };
 272
 273 my $ivshmem_fmt = {
 274     size => {
 275         type => 'integer',
 276         minimum => 1,
 277         description => "The size of the file in MB.",
 278     },
 279     name => {
 280         type => 'string',
 281         pattern => '[a-zA-Z0-9\-]+',
 282         optional => 1,
 283         format_description => 'string',
 284         description => "The name of the file. Will be prefixed with 'pve-shm-'. Default is the VMID. Will be deleted when the VM is stopped.",
 285     },
 286 };
 287
 288 my $audio_fmt = {
 289     device => {
 290         type => 'string',
 291         enum => [qw(ich9-intel-hda intel-hda AC97)],
 292         description =>  "Configure an audio device."
 293     },
 294     driver =>  {
 295         type => 'string',
 296         enum => ['spice'],
 297         default => 'spice',
 298         optional => 1,
 299         description => "Driver backend for the audio device."
 300     },
 301 };
 302
 303 my $spice_enhancements_fmt = {
 304     foldersharing => {
 305         type => 'boolean',
 306         optional => 1,
 307         default => '0',
 308         description =>  "Enable folder sharing via SPICE. Needs Spice-WebDAV daemon installed in the VM."
 309     },
 310     videostreaming =>  {
 311         type => 'string',
 312         enum => ['off', 'all', 'filter'],
 313         default => 'off',
 314         optional => 1,
 315         description => "Enable video streaming. Uses compression for detected video streams."
 316     },
 317 };
 318
 319 my $confdesc = {
 320     onboot => {
 321         optional => 1,
 322         type => 'boolean',
 323         description => "Specifies whether a VM will be started during system bootup.",
 324         default => 0,
 325     },
 326     autostart => {
 327         optional => 1,
 328         type => 'boolean',
 329         description => "Automatic restart after crash (currently ignored).",
 330         default => 0,
 331     },
 332     hotplug => {
 333         optional => 1,
 334         type => 'string', format => 'pve-hotplug-features',
 335         description => "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
 336         default => 'network,disk,usb',
 337     },
 338     reboot => {
 339         optional => 1,
 340         type => 'boolean',
 341         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 342         default => 1,
 343     },
 344     lock => {
 345         optional => 1,
 346         type => 'string',
 347         description => "Lock/unlock the VM.",
 348         enum => [qw(backup clone create migrate rollback snapshot snapshot-delete suspending suspended)],
 349     },
 350     cpulimit => {
 351         optional => 1,
 352         type => 'number',
 353         description => "Limit of CPU usage.",
 354         verbose_description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
 355         minimum => 0,
 356         maximum => 128,
 357         default => 0,
 358     },
 359     cpuunits => {
 360         optional => 1,
 361         type => 'integer',
 362         description => "CPU weight for a VM.",
 363         verbose_description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.",
 364         minimum => 2,
 365         maximum => 262144,
 366         default => 1024,
 367     },
 368     memory => {
 369         optional => 1,
 370         type => 'integer',
 371         description => "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
 372         minimum => 16,
 373         default => 512,
 374     },
 375     balloon => {
 376         optional => 1,
 377         type => 'integer',
 378         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 379         minimum => 0,
 380     },
 381     shares => {
 382         optional => 1,
 383         type => 'integer',
 384         description => "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning. Auto-ballooning is done by pvestatd.",
 385         minimum => 0,
 386         maximum => 50000,
 387         default => 1000,
 388     },
 389     keyboard => {
 390         optional => 1,
 391         type => 'string',
 392         description => "Keybord layout for vnc server. Default is read from the '/etc/pve/datacenter.cfg' configuration file.".
 393                        "It should not be necessary to set it.",
 394         enum => PVE::Tools::kvmkeymaplist(),
 395         default => undef,
 396     },
 397     name => {
 398         optional => 1,
 399         type => 'string', format => 'dns-name',
 400         description => "Set a name for the VM. Only used on the configuration web interface.",
 401     },
 402     scsihw => {
 403         optional => 1,
 404         type => 'string',
 405         description => "SCSI controller model",
 406         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 407         default => 'lsi',
 408     },
 409     description => {
 410         optional => 1,
 411         type => 'string',
 412         description => "Description for the VM. Only used on the configuration web interface. This is saved as comment inside the configuration file.",
 413     },
 414     ostype => {
 415         optional => 1,
 416         type => 'string',
 417         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 l24 l26 solaris)],
 418         description => "Specify guest operating system.",
 419         verbose_description => <<EODESC,
 420 Specify guest operating system. This is used to enable special
 421 optimization/features for specific operating systems:
 422
 423 [horizontal]
 424 other;; unspecified OS
 425 wxp;; Microsoft Windows XP
 426 w2k;; Microsoft Windows 2000
 427 w2k3;; Microsoft Windows 2003
 428 w2k8;; Microsoft Windows 2008
 429 wvista;; Microsoft Windows Vista
 430 win7;; Microsoft Windows 7
 431 win8;; Microsoft Windows 8/2012/2012r2
 432 win10;; Microsoft Windows 10/2016
 433 l24;; Linux 2.4 Kernel
 434 l26;; Linux 2.6 - 5.X Kernel
 435 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
 436 EODESC
 437     },
 438     boot => {
 439         optional => 1,
 440         type => 'string',
 441         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n).",
 442         pattern => '[acdn]{1,4}',
 443         default => 'cdn',
 444     },
 445     bootdisk => {
 446         optional => 1,
 447         type => 'string', format => 'pve-qm-bootdisk',
 448         description => "Enable booting from specified disk.",
 449         pattern => '(ide|sata|scsi|virtio)\d+',
 450     },
 451     smp => {
 452         optional => 1,
 453         type => 'integer',
 454         description => "The number of CPUs. Please use option -sockets instead.",
 455         minimum => 1,
 456         default => 1,
 457     },
 458     sockets => {
 459         optional => 1,
 460         type => 'integer',
 461         description => "The number of CPU sockets.",
 462         minimum => 1,
 463         default => 1,
 464     },
 465     cores => {
 466         optional => 1,
 467         type => 'integer',
 468         description => "The number of cores per socket.",
 469         minimum => 1,
 470         default => 1,
 471     },
 472     numa => {
 473         optional => 1,
 474         type => 'boolean',
 475         description => "Enable/disable NUMA.",
 476         default => 0,
 477     },
 478     hugepages => {
 479         optional => 1,
 480         type => 'string',
 481         description => "Enable/disable hugepages memory.",
 482         enum => [qw(any 2 1024)],
 483     },
 484     vcpus => {
 485         optional => 1,
 486         type => 'integer',
 487         description => "Number of hotplugged vcpus.",
 488         minimum => 1,
 489         default => 0,
 490     },
 491     acpi => {
 492         optional => 1,
 493         type => 'boolean',
 494         description => "Enable/disable ACPI.",
 495         default => 1,
 496     },
 497     agent => {
 498         optional => 1,
 499         description => "Enable/disable Qemu GuestAgent and its properties.",
 500         type => 'string',
 501         format => $agent_fmt,
 502     },
 503     kvm => {
 504         optional => 1,
 505         type => 'boolean',
 506         description => "Enable/disable KVM hardware virtualization.",
 507         default => 1,
 508     },
 509     tdf => {
 510         optional => 1,
 511         type => 'boolean',
 512         description => "Enable/disable time drift fix.",
 513         default => 0,
 514     },
 515     localtime => {
 516         optional => 1,
 517         type => 'boolean',
 518         description => "Set the real time clock to local time. This is enabled by default if ostype indicates a Microsoft OS.",
 519     },
 520     freeze => {
 521         optional => 1,
 522         type => 'boolean',
 523         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 524     },
 525     vga => {
 526         optional => 1,
 527         type => 'string', format => $vga_fmt,
 528         description => "Configure the VGA hardware.",
 529         verbose_description => "Configure the VGA Hardware. If you want to use ".
 530             "high resolution modes (>= 1280x1024x16) you may need to increase " .
 531             "the vga memory option. Since QEMU 2.9 the default VGA display type " .
 532             "is 'std' for all OS types besides some Windows versions (XP and " .
 533             "older) which use 'cirrus'. The 'qxl' option enables the SPICE " .
 534             "display server. For win* OS you can select how many independent " .
 535             "displays you want, Linux guests can add displays them self.\n".
 536             "You can also run without any graphic card, using a serial device as terminal.",
 537     },
 538     watchdog => {
 539         optional => 1,
 540         type => 'string', format => 'pve-qm-watchdog',
 541         description => "Create a virtual hardware watchdog device.",
 542         verbose_description => "Create a virtual hardware watchdog device. Once enabled" .
 543             " (by a guest action), the watchdog must be periodically polled " .
 544             "by an agent inside the guest or else the watchdog will reset " .
 545             "the guest (or execute the respective action specified)",
 546     },
 547     startdate => {
 548         optional => 1,
 549         type => 'string',
 550         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 551         description => "Set the initial date of the real time clock. Valid format for date are: 'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 552         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 553         default => 'now',
 554     },
 555     startup =>  get_standard_option('pve-startup-order'),
 556     template => {
 557         optional => 1,
 558         type => 'boolean',
 559         description => "Enable/disable Template.",
 560         default => 0,
 561     },
 562     args => {
 563         optional => 1,
 564         type => 'string',
 565         description => "Arbitrary arguments passed to kvm.",
 566         verbose_description => <<EODESCR,
 567 Arbitrary arguments passed to kvm, for example:
 568
 569 args: -no-reboot -no-hpet
 570
 571 NOTE: this option is for experts only.
 572 EODESCR
 573     },
 574     tablet => {
 575         optional => 1,
 576         type => 'boolean',
 577         default => 1,
 578         description => "Enable/disable the USB tablet device.",
 579         verbose_description => "Enable/disable the USB tablet device. This device is " .
 580             "usually needed to allow absolute mouse positioning with VNC. " .
 581             "Else the mouse runs out of sync with normal VNC clients. " .
 582             "If you're running lots of console-only guests on one host, " .
 583             "you may consider disabling this to save some context switches. " .
 584             "This is turned off by default if you use spice (-vga=qxl).",
 585     },
 586     migrate_speed => {
 587         optional => 1,
 588         type => 'integer',
 589         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 590         minimum => 0,
 591         default => 0,
 592     },
 593     migrate_downtime => {
 594         optional => 1,
 595         type => 'number',
 596         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 597         minimum => 0,
 598         default => 0.1,
 599     },
 600     cdrom => {
 601         optional => 1,
 602         type => 'string', format => 'pve-qm-ide',
 603         typetext => '<volume>',
 604         description => "This is an alias for option -ide2",
 605     },
 606     cpu => {
 607         optional => 1,
 608         description => "Emulated CPU type.",
 609         type => 'string',
 610         format => $cpu_fmt,
 611     },
 612     parent => get_standard_option('pve-snapshot-name', {
 613         optional => 1,
 614         description => "Parent snapshot name. This is used internally, and should not be modified.",
 615     }),
 616     snaptime => {
 617         optional => 1,
 618         description => "Timestamp for snapshots.",
 619         type => 'integer',
 620         minimum => 0,
 621     },
 622     vmstate => {
 623         optional => 1,
 624         type => 'string', format => 'pve-volume-id',
 625         description => "Reference to a volume which stores the VM state. This is used internally for snapshots.",
 626     },
 627     vmstatestorage => get_standard_option('pve-storage-id', {
 628         description => "Default storage for VM state volumes/files.",
 629         optional => 1,
 630     }),
 631     runningmachine => get_standard_option('pve-qemu-machine', {
 632         description => "Specifies the Qemu machine type of the running vm. This is used internally for snapshots.",
 633     }),
 634     machine => get_standard_option('pve-qemu-machine'),
 635     arch => {
 636         description => "Virtual processor architecture. Defaults to the host.",
 637         optional => 1,
 638         type => 'string',
 639         enum => [qw(x86_64 aarch64)],
 640     },
 641     smbios1 => {
 642         description => "Specify SMBIOS type 1 fields.",
 643         type => 'string', format => 'pve-qm-smbios1',
 644         maxLength => 512,
 645         optional => 1,
 646     },
 647     protection => {
 648         optional => 1,
 649         type => 'boolean',
 650         description => "Sets the protection flag of the VM. This will disable the remove VM and remove disk operations.",
 651         default => 0,
 652     },
 653     bios => {
 654         optional => 1,
 655         type => 'string',
 656         enum => [ qw(seabios ovmf) ],
 657         description => "Select BIOS implementation.",
 658         default => 'seabios',
 659     },
 660     vmgenid => {
 661         type => 'string',
 662         pattern => '(?:[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}|[01])',
 663         format_description => 'UUID',
 664         description => "Set VM Generation ID. Use '1' to autogenerate on create or update, pass '0' to disable explicitly.",
 665         verbose_description => "The VM generation ID (vmgenid) device exposes a".
 666             " 128-bit integer value identifier to the guest OS. This allows to".
 667             " notify the guest operating system when the virtual machine is".
 668             " executed with a different configuration (e.g. snapshot execution".
 669             " or creation from a template). The guest operating system notices".
 670             " the change, and is then able to react as appropriate by marking".
 671             " its copies of distributed databases as dirty, re-initializing its".
 672             " random number generator, etc.\n".
 673             "Note that auto-creation only works when done throug API/CLI create".
 674             " or update methods, but not when manually editing the config file.",
 675         default => "1 (autogenerated)",
 676         optional => 1,
 677     },
 678     hookscript => {
 679         type => 'string',
 680         format => 'pve-volume-id',
 681         optional => 1,
 682         description => "Script that will be executed during various steps in the vms lifetime.",
 683     },
 684     ivshmem => {
 685         type => 'string',
 686         format => $ivshmem_fmt,
 687         description => "Inter-VM shared memory. Useful for direct communication between VMs, or to the host.",
 688         optional => 1,
 689     },
 690     audio0 => {
 691         type => 'string',
 692         format => $audio_fmt,
 693         description => "Configure a audio device, useful in combination with QXL/Spice.",
 694         optional => 1
 695     },
 696     spice_enhancements => {
 697         type => 'string',
 698         format => $spice_enhancements_fmt,
 699         description => "Configure additional enhancements for SPICE.",
 700         optional => 1
 701     },
 702     tags => {
 703         type => 'string', format => 'pve-tag-list',
 704         description => 'Tags of the VM. This is only meta information.',
 705         optional => 1,
 706     },
 707 };
 708
 709 my $cicustom_fmt = {
 710     meta => {
 711         type => 'string',
 712         optional => 1,
 713         description => 'Specify a custom file containing all meta data passed to the VM via cloud-init. This is provider specific meaning configdrive2 and nocloud differ.',
 714         format => 'pve-volume-id',
 715         format_description => 'volume',
 716     },
 717     network => {
 718         type => 'string',
 719         optional => 1,
 720         description => 'Specify a custom file containing all network data passed to the VM via cloud-init.',
 721         format => 'pve-volume-id',
 722         format_description => 'volume',
 723     },
 724     user => {
 725         type => 'string',
 726         optional => 1,
 727         description => 'Specify a custom file containing all user data passed to the VM via cloud-init.',
 728         format => 'pve-volume-id',
 729         format_description => 'volume',
 730     },
 731 };
 732 PVE::JSONSchema::register_format('pve-qm-cicustom', $cicustom_fmt);
 733
 734 my $confdesc_cloudinit = {
 735     citype => {
 736         optional => 1,
 737         type => 'string',
 738         description => 'Specifies the cloud-init configuration format. The default depends on the configured operating system type (`ostype`. We use the `nocloud` format for Linux, and `configdrive2` for windows.',
 739         enum => ['configdrive2', 'nocloud'],
 740     },
 741     ciuser => {
 742         optional => 1,
 743         type => 'string',
 744         description => "cloud-init: User name to change ssh keys and password for instead of the image's configured default user.",
 745     },
 746     cipassword => {
 747         optional => 1,
 748         type => 'string',
 749         description => 'cloud-init: Password to assign the user. Using this is generally not recommended. Use ssh keys instead. Also note that older cloud-init versions do not support hashed passwords.',
 750     },
 751     cicustom => {
 752         optional => 1,
 753         type => 'string',
 754         description => 'cloud-init: Specify custom files to replace the automatically generated ones at start.',
 755         format => 'pve-qm-cicustom',
 756     },
 757     searchdomain => {
 758         optional => 1,
 759         type => 'string',
 760         description => "cloud-init: Sets DNS search domains for a container. Create will automatically use the setting from the host if neither searchdomain nor nameserver are set.",
 761     },
 762     nameserver => {
 763         optional => 1,
 764         type => 'string', format => 'address-list',
 765         description => "cloud-init: Sets DNS server IP address for a container. Create will automatically use the setting from the host if neither searchdomain nor nameserver are set.",
 766     },
 767     sshkeys => {
 768         optional => 1,
 769         type => 'string',
 770         format => 'urlencoded',
 771         description => "cloud-init: Setup public SSH keys (one key per line, OpenSSH format).",
 772     },
 773 };
 774
 775 # what about other qemu settings ?
 776 #cpu => 'string',
 777 #machine => 'string',
 778 #fda => 'file',
 779 #fdb => 'file',
 780 #mtdblock => 'file',
 781 #sd => 'file',
 782 #pflash => 'file',
 783 #snapshot => 'bool',
 784 #bootp => 'file',
 785 ##tftp => 'dir',
 786 ##smb => 'dir',
 787 #kernel => 'file',
 788 #append => 'string',
 789 #initrd => 'file',
 790 ##soundhw => 'string',
 791
 792 while (my ($k, $v) = each %$confdesc) {
 793     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 794 }
 795
 796 my $MAX_IDE_DISKS = 4;
 797 my $MAX_SCSI_DISKS = 14;
 798 my $MAX_VIRTIO_DISKS = 16;
 799 my $MAX_SATA_DISKS = 6;
 800 my $MAX_USB_DEVICES = 5;
 801 my $MAX_NETS = 32;
 802 my $MAX_UNUSED_DISKS = 256;
 803 my $MAX_HOSTPCI_DEVICES = 16;
 804 my $MAX_SERIAL_PORTS = 4;
 805 my $MAX_PARALLEL_PORTS = 3;
 806 my $MAX_NUMA = 8;
 807
 808 my $numa_fmt = {
 809     cpus => {
 810         type => "string",
 811         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 812         description => "CPUs accessing this NUMA node.",
 813         format_description => "id[-id];...",
 814     },
 815     memory => {
 816         type => "number",
 817         description => "Amount of memory this NUMA node provides.",
 818         optional => 1,
 819     },
 820     hostnodes => {
 821         type => "string",
 822         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 823         description => "Host NUMA nodes to use.",
 824         format_description => "id[-id];...",
 825         optional => 1,
 826     },
 827     policy => {
 828         type => 'string',
 829         enum => [qw(preferred bind interleave)],
 830         description => "NUMA allocation policy.",
 831         optional => 1,
 832     },
 833 };
 834 PVE::JSONSchema::register_format('pve-qm-numanode', $numa_fmt);
 835 my $numadesc = {
 836     optional => 1,
 837     type => 'string', format => $numa_fmt,
 838     description => "NUMA topology.",
 839 };
 840 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 841
 842 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 843     $confdesc->{"numa$i"} = $numadesc;
 844 }
 845
 846 my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000',  'pcnet',  'virtio',
 847                       'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3',
 848                       'e1000-82540em', 'e1000-82544gc', 'e1000-82545em'];
 849 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 850
 851 my $net_fmt_bridge_descr = <<__EOD__;
 852 Bridge to attach the network device to. The Proxmox VE standard bridge
 853 is called 'vmbr0'.
 854
 855 If you do not specify a bridge, we create a kvm user (NATed) network
 856 device, which provides DHCP and DNS services. The following addresses
 857 are used:
 858
 859  10.0.2.2   Gateway
 860  10.0.2.3   DNS Server
 861  10.0.2.4   SMB Server
 862
 863 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 864 __EOD__
 865
 866 my $net_fmt = {
 867     macaddr  => get_standard_option('mac-addr', {
 868         description => "MAC address. That address must be unique withing your network. This is automatically generated if not specified.",
 869     }),
 870     model => {
 871         type => 'string',
 872         description => "Network Card Model. The 'virtio' model provides the best performance with very low CPU overhead. If your guest does not support this driver, it is usually best to use 'e1000'.",
 873         enum => $nic_model_list,
 874         default_key => 1,
 875     },
 876     (map { $_ => { keyAlias => 'model', alias => 'macaddr' }} @$nic_model_list),
 877     bridge => {
 878         type => 'string',
 879         description => $net_fmt_bridge_descr,
 880         format_description => 'bridge',
 881         optional => 1,
 882     },
 883     queues => {
 884         type => 'integer',
 885         minimum => 0, maximum => 16,
 886         description => 'Number of packet queues to be used on the device.',
 887         optional => 1,
 888     },
 889     rate => {
 890         type => 'number',
 891         minimum => 0,
 892         description => "Rate limit in mbps (megabytes per second) as floating point number.",
 893         optional => 1,
 894     },
 895     tag => {
 896         type => 'integer',
 897         minimum => 1, maximum => 4094,
 898         description => 'VLAN tag to apply to packets on this interface.',
 899         optional => 1,
 900     },
 901     trunks => {
 902         type => 'string',
 903         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 904         description => 'VLAN trunks to pass through this interface.',
 905         format_description => 'vlanid[;vlanid...]',
 906         optional => 1,
 907     },
 908     firewall => {
 909         type => 'boolean',
 910         description => 'Whether this interface should be protected by the firewall.',
 911         optional => 1,
 912     },
 913     link_down => {
 914         type => 'boolean',
 915         description => 'Whether this interface should be disconnected (like pulling the plug).',
 916         optional => 1,
 917     },
 918 };
 919
 920 my $netdesc = {
 921     optional => 1,
 922     type => 'string', format => $net_fmt,
 923     description => "Specify network devices.",
 924 };
 925
 926 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 927
 928 my $ipconfig_fmt = {
 929     ip => {
 930         type => 'string',
 931         format => 'pve-ipv4-config',
 932         format_description => 'IPv4Format/CIDR',
 933         description => 'IPv4 address in CIDR format.',
 934         optional => 1,
 935         default => 'dhcp',
 936     },
 937     gw => {
 938         type => 'string',
 939         format => 'ipv4',
 940         format_description => 'GatewayIPv4',
 941         description => 'Default gateway for IPv4 traffic.',
 942         optional => 1,
 943         requires => 'ip',
 944     },
 945     ip6 => {
 946         type => 'string',
 947         format => 'pve-ipv6-config',
 948         format_description => 'IPv6Format/CIDR',
 949         description => 'IPv6 address in CIDR format.',
 950         optional => 1,
 951         default => 'dhcp',
 952     },
 953     gw6 => {
 954         type => 'string',
 955         format => 'ipv6',
 956         format_description => 'GatewayIPv6',
 957         description => 'Default gateway for IPv6 traffic.',
 958         optional => 1,
 959         requires => 'ip6',
 960     },
 961 };
 962 PVE::JSONSchema::register_format('pve-qm-ipconfig', $ipconfig_fmt);
 963 my $ipconfigdesc = {
 964     optional => 1,
 965     type => 'string', format => 'pve-qm-ipconfig',
 966     description => <<'EODESCR',
 967 cloud-init: Specify IP addresses and gateways for the corresponding interface.
 968
 969 IP addresses use CIDR notation, gateways are optional but need an IP of the same type specified.
 970
 971 The special string 'dhcp' can be used for IP addresses to use DHCP, in which case no explicit gateway should be provided.
 972 For IPv6 the special string 'auto' can be used to use stateless autoconfiguration.
 973
 974 If cloud-init is enabled and neither an IPv4 nor an IPv6 address is specified, it defaults to using dhcp on IPv4.
 975 EODESCR
 976 };
 977 PVE::JSONSchema::register_standard_option("pve-qm-ipconfig", $netdesc);
 978
 979 for (my $i = 0; $i < $MAX_NETS; $i++)  {
 980     $confdesc->{"net$i"} = $netdesc;
 981     $confdesc_cloudinit->{"ipconfig$i"} = $ipconfigdesc;
 982 }
 983
 984 foreach my $key (keys %$confdesc_cloudinit) {
 985     $confdesc->{$key} = $confdesc_cloudinit->{$key};
 986 }
 987
 988 PVE::JSONSchema::register_format('pve-volume-id-or-qm-path', \&verify_volume_id_or_qm_path);
 989 sub verify_volume_id_or_qm_path {
 990     my ($volid, $noerr) = @_;
 991
 992     if ($volid eq 'none' || $volid eq 'cdrom' || $volid =~ m|^/|) {
 993         return $volid;
 994     }
 995
 996     # if its neither 'none' nor 'cdrom' nor a path, check if its a volume-id
 997     $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') };
 998     if ($@) {
 999         return undef if $noerr;
1000         die $@;
1001     }
1002     return $volid;
1003 }
1004
1005 my $drivename_hash;
1006
1007 my %drivedesc_base = (
1008     volume => { alias => 'file' },
1009     file => {
1010         type => 'string',
1011         format => 'pve-volume-id-or-qm-path',
1012         default_key => 1,
1013         format_description => 'volume',
1014         description => "The drive's backing volume.",
1015     },
1016     media => {
1017         type => 'string',
1018         enum => [qw(cdrom disk)],
1019         description => "The drive's media type.",
1020         default => 'disk',
1021         optional => 1
1022     },
1023     cyls => {
1024         type => 'integer',
1025         description => "Force the drive's physical geometry to have a specific cylinder count.",
1026         optional => 1
1027     },
1028     heads => {
1029         type => 'integer',
1030         description => "Force the drive's physical geometry to have a specific head count.",
1031         optional => 1
1032     },
1033     secs => {
1034         type => 'integer',
1035         description => "Force the drive's physical geometry to have a specific sector count.",
1036         optional => 1
1037     },
1038     trans => {
1039         type => 'string',
1040         enum => [qw(none lba auto)],
1041         description => "Force disk geometry bios translation mode.",
1042         optional => 1,
1043     },
1044     snapshot => {
1045         type => 'boolean',
1046         description => "Controls qemu's snapshot mode feature."
1047             . " If activated, changes made to the disk are temporary and will"
1048             . " be discarded when the VM is shutdown.",
1049         optional => 1,
1050     },
1051     cache => {
1052         type => 'string',
1053         enum => [qw(none writethrough writeback unsafe directsync)],
1054         description => "The drive's cache mode",
1055         optional => 1,
1056     },
1057     format => get_standard_option('pve-qm-image-format'),
1058     size => {
1059         type => 'string',
1060         format => 'disk-size',
1061         format_description => 'DiskSize',
1062         description => "Disk size. This is purely informational and has no effect.",
1063         optional => 1,
1064     },
1065     backup => {
1066         type => 'boolean',
1067         description => "Whether the drive should be included when making backups.",
1068         optional => 1,
1069     },
1070     replicate => {
1071         type => 'boolean',
1072         description => 'Whether the drive should considered for replication jobs.',
1073         optional => 1,
1074         default => 1,
1075     },
1076     rerror => {
1077         type => 'string',
1078         enum => [qw(ignore report stop)],
1079         description => 'Read error action.',
1080         optional => 1,
1081     },
1082     werror => {
1083         type => 'string',
1084         enum => [qw(enospc ignore report stop)],
1085         description => 'Write error action.',
1086         optional => 1,
1087     },
1088     aio => {
1089         type => 'string',
1090         enum => [qw(native threads)],
1091         description => 'AIO type to use.',
1092         optional => 1,
1093     },
1094     discard => {
1095         type => 'string',
1096         enum => [qw(ignore on)],
1097         description => 'Controls whether to pass discard/trim requests to the underlying storage.',
1098         optional => 1,
1099     },
1100     detect_zeroes => {
1101         type => 'boolean',
1102         description => 'Controls whether to detect and try to optimize writes of zeroes.',
1103         optional => 1,
1104     },
1105     serial => {
1106         type => 'string',
1107         format => 'urlencoded',
1108         format_description => 'serial',
1109         maxLength => 20*3, # *3 since it's %xx url enoded
1110         description => "The drive's reported serial number, url-encoded, up to 20 bytes long.",
1111         optional => 1,
1112     },
1113     shared => {
1114         type => 'boolean',
1115         description => 'Mark this locally-managed volume as available on all nodes',
1116         verbose_description => "Mark this locally-managed volume as available on all nodes.\n\nWARNING: This option does not share the volume automatically, it assumes it is shared already!",
1117         optional => 1,
1118         default => 0,
1119     }
1120 );
1121
1122 my %iothread_fmt = ( iothread => {
1123         type => 'boolean',
1124         description => "Whether to use iothreads for this drive",
1125         optional => 1,
1126 });
1127
1128 my %model_fmt = (
1129     model => {
1130         type => 'string',
1131         format => 'urlencoded',
1132         format_description => 'model',
1133         maxLength => 40*3, # *3 since it's %xx url enoded
1134         description => "The drive's reported model name, url-encoded, up to 40 bytes long.",
1135         optional => 1,
1136     },
1137 );
1138
1139 my %queues_fmt = (
1140     queues => {
1141         type => 'integer',
1142         description => "Number of queues.",
1143         minimum => 2,
1144         optional => 1
1145     }
1146 );
1147
1148 my %scsiblock_fmt = (
1149     scsiblock => {
1150         type => 'boolean',
1151         description => "whether to use scsi-block for full passthrough of host block device\n\nWARNING: can lead to I/O errors in combination with low memory or high memory fragmentation on host",
1152         optional => 1,
1153         default => 0,
1154     },
1155 );
1156
1157 my %ssd_fmt = (
1158     ssd => {
1159         type => 'boolean',
1160         description => "Whether to expose this drive as an SSD, rather than a rotational hard disk.",
1161         optional => 1,
1162     },
1163 );
1164
1165 my %wwn_fmt = (
1166     wwn => {
1167         type => 'string',
1168         pattern => qr/^(0x)[0-9a-fA-F]{16}/,
1169         format_description => 'wwn',
1170         description => "The drive's worldwide name, encoded as 16 bytes hex string, prefixed by '0x'.",
1171         optional => 1,
1172     },
1173 );
1174
1175 my $add_throttle_desc = sub {
1176     my ($key, $type, $what, $unit, $longunit, $minimum) = @_;
1177     my $d = {
1178         type => $type,
1179         format_description => $unit,
1180         description => "Maximum $what in $longunit.",
1181         optional => 1,
1182     };
1183     $d->{minimum} = $minimum if defined($minimum);
1184     $drivedesc_base{$key} = $d;
1185 };
1186 # throughput: (leaky bucket)
1187 $add_throttle_desc->('bps',     'integer', 'r/w speed',   'bps',  'bytes per second');
1188 $add_throttle_desc->('bps_rd',  'integer', 'read speed',  'bps',  'bytes per second');
1189 $add_throttle_desc->('bps_wr',  'integer', 'write speed', 'bps',  'bytes per second');
1190 $add_throttle_desc->('mbps',    'number',  'r/w speed',   'mbps', 'megabytes per second');
1191 $add_throttle_desc->('mbps_rd', 'number',  'read speed',  'mbps', 'megabytes per second');
1192 $add_throttle_desc->('mbps_wr', 'number',  'write speed', 'mbps', 'megabytes per second');
1193 $add_throttle_desc->('iops',    'integer', 'r/w I/O',     'iops', 'operations per second');
1194 $add_throttle_desc->('iops_rd', 'integer', 'read I/O',    'iops', 'operations per second');
1195 $add_throttle_desc->('iops_wr', 'integer', 'write I/O',   'iops', 'operations per second');
1196
1197 # pools: (pool of IO before throttling starts taking effect)
1198 $add_throttle_desc->('mbps_max',    'number',  'unthrottled r/w pool',       'mbps', 'megabytes per second');
1199 $add_throttle_desc->('mbps_rd_max', 'number',  'unthrottled read pool',      'mbps', 'megabytes per second');
1200 $add_throttle_desc->('mbps_wr_max', 'number',  'unthrottled write pool',     'mbps', 'megabytes per second');
1201 $add_throttle_desc->('iops_max',    'integer', 'unthrottled r/w I/O pool',   'iops', 'operations per second');
1202 $add_throttle_desc->('iops_rd_max', 'integer', 'unthrottled read I/O pool',  'iops', 'operations per second');
1203 $add_throttle_desc->('iops_wr_max', 'integer', 'unthrottled write I/O pool', 'iops', 'operations per second');
1204
1205 # burst lengths
1206 $add_throttle_desc->('bps_max_length',     'integer', 'length of I/O bursts',       'seconds', 'seconds', 1);
1207 $add_throttle_desc->('bps_rd_max_length',  'integer', 'length of read I/O bursts',  'seconds', 'seconds', 1);
1208 $add_throttle_desc->('bps_wr_max_length',  'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
1209 $add_throttle_desc->('iops_max_length',    'integer', 'length of I/O bursts',       'seconds', 'seconds', 1);
1210 $add_throttle_desc->('iops_rd_max_length', 'integer', 'length of read I/O bursts',  'seconds', 'seconds', 1);
1211 $add_throttle_desc->('iops_wr_max_length', 'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
1212
1213 # legacy support
1214 $drivedesc_base{'bps_rd_length'} = { alias => 'bps_rd_max_length' };
1215 $drivedesc_base{'bps_wr_length'} = { alias => 'bps_wr_max_length' };
1216 $drivedesc_base{'iops_rd_length'} = { alias => 'iops_rd_max_length' };
1217 $drivedesc_base{'iops_wr_length'} = { alias => 'iops_wr_max_length' };
1218
1219 my $ide_fmt = {
1220     %drivedesc_base,
1221     %model_fmt,
1222     %ssd_fmt,
1223     %wwn_fmt,
1224 };
1225 PVE::JSONSchema::register_format("pve-qm-ide", $ide_fmt);
1226
1227 my $idedesc = {
1228     optional => 1,
1229     type => 'string', format => $ide_fmt,
1230     description => "Use volume as IDE hard disk or CD-ROM (n is 0 to " .($MAX_IDE_DISKS -1) . ").",
1231 };
1232 PVE::JSONSchema::register_standard_option("pve-qm-ide", $idedesc);
1233
1234 my $scsi_fmt = {
1235     %drivedesc_base,
1236     %iothread_fmt,
1237     %queues_fmt,
1238     %scsiblock_fmt,
1239     %ssd_fmt,
1240     %wwn_fmt,
1241 };
1242 my $scsidesc = {
1243     optional => 1,
1244     type => 'string', format => $scsi_fmt,
1245     description => "Use volume as SCSI hard disk or CD-ROM (n is 0 to " . ($MAX_SCSI_DISKS - 1) . ").",
1246 };
1247 PVE::JSONSchema::register_standard_option("pve-qm-scsi", $scsidesc);
1248
1249 my $sata_fmt = {
1250     %drivedesc_base,
1251     %ssd_fmt,
1252     %wwn_fmt,
1253 };
1254 my $satadesc = {
1255     optional => 1,
1256     type => 'string', format => $sata_fmt,
1257     description => "Use volume as SATA hard disk or CD-ROM (n is 0 to " . ($MAX_SATA_DISKS - 1). ").",
1258 };
1259 PVE::JSONSchema::register_standard_option("pve-qm-sata", $satadesc);
1260
1261 my $virtio_fmt = {
1262     %drivedesc_base,
1263     %iothread_fmt,
1264 };
1265 my $virtiodesc = {
1266     optional => 1,
1267     type => 'string', format => $virtio_fmt,
1268     description => "Use volume as VIRTIO hard disk (n is 0 to " . ($MAX_VIRTIO_DISKS - 1) . ").",
1269 };
1270 PVE::JSONSchema::register_standard_option("pve-qm-virtio", $virtiodesc);
1271
1272 my $alldrive_fmt = {
1273     %drivedesc_base,
1274     %iothread_fmt,
1275     %model_fmt,
1276     %queues_fmt,
1277     %scsiblock_fmt,
1278     %ssd_fmt,
1279     %wwn_fmt,
1280 };
1281
1282 my $efidisk_fmt = {
1283     volume => { alias => 'file' },
1284     file => {
1285         type => 'string',
1286         format => 'pve-volume-id-or-qm-path',
1287         default_key => 1,
1288         format_description => 'volume',
1289         description => "The drive's backing volume.",
1290     },
1291     format => get_standard_option('pve-qm-image-format'),
1292     size => {
1293         type => 'string',
1294         format => 'disk-size',
1295         format_description => 'DiskSize',
1296         description => "Disk size. This is purely informational and has no effect.",
1297         optional => 1,
1298     },
1299 };
1300
1301 my $efidisk_desc = {
1302     optional => 1,
1303     type => 'string', format => $efidisk_fmt,
1304     description => "Configure a Disk for storing EFI vars",
1305 };
1306
1307 PVE::JSONSchema::register_standard_option("pve-qm-efidisk", $efidisk_desc);
1308
1309 my $usb_fmt = {
1310     host => {
1311         default_key => 1,
1312         type => 'string', format => 'pve-qm-usb-device',
1313         format_description => 'HOSTUSBDEVICE|spice',
1314         description => <<EODESCR,
1315 The Host USB device or port or the value 'spice'. HOSTUSBDEVICE syntax is:
1316
1317  'bus-port(.port)*' (decimal numbers) or
1318  'vendor_id:product_id' (hexadeciaml numbers) or
1319  'spice'
1320
1321 You can use the 'lsusb -t' command to list existing usb devices.
1322
1323 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1324
1325 The value 'spice' can be used to add a usb redirection devices for spice.
1326 EODESCR
1327     },
1328     usb3 => {
1329         optional => 1,
1330         type => 'boolean',
1331         description => "Specifies whether if given host option is a USB3 device or port.",
1332         default => 0,
1333     },
1334 };
1335
1336 my $usbdesc = {
1337     optional => 1,
1338     type => 'string', format => $usb_fmt,
1339     description => "Configure an USB device (n is 0 to 4).",
1340 };
1341 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
1342
1343 my $PCIRE = qr/([a-f0-9]{4}:)?[a-f0-9]{2}:[a-f0-9]{2}(?:\.[a-f0-9])?/;
1344 my $hostpci_fmt = {
1345     host => {
1346         default_key => 1,
1347         type => 'string',
1348         pattern => qr/$PCIRE(;$PCIRE)*/,
1349         format_description => 'HOSTPCIID[;HOSTPCIID2...]',
1350         description => <<EODESCR,
1351 Host PCI device pass through. The PCI ID of a host's PCI device or a list
1352 of PCI virtual functions of the host. HOSTPCIID syntax is:
1353
1354 'bus:dev.func' (hexadecimal numbers)
1355
1356 You can us the 'lspci' command to list existing PCI devices.
1357 EODESCR
1358     },
1359     rombar => {
1360         type => 'boolean',
1361         description =>  "Specify whether or not the device's ROM will be visible in the guest's memory map.",
1362         optional => 1,
1363         default => 1,
1364     },
1365     romfile => {
1366         type => 'string',
1367         pattern => '[^,;]+',
1368         format_description => 'string',
1369         description => "Custom pci device rom filename (must be located in /usr/share/kvm/).",
1370         optional => 1,
1371     },
1372     pcie => {
1373         type => 'boolean',
1374         description =>  "Choose the PCI-express bus (needs the 'q35' machine model).",
1375         optional => 1,
1376         default => 0,
1377     },
1378     'x-vga' => {
1379         type => 'boolean',
1380         description =>  "Enable vfio-vga device support.",
1381         optional => 1,
1382         default => 0,
1383     },
1384     'mdev' => {
1385         type => 'string',
1386         format_description => 'string',
1387         pattern => '[^/\.:]+',
1388         optional => 1,
1389         description => <<EODESCR
1390 The type of mediated device to use.
1391 An instance of this type will be created on startup of the VM and
1392 will be cleaned up when the VM stops.
1393 EODESCR
1394     }
1395 };
1396 PVE::JSONSchema::register_format('pve-qm-hostpci', $hostpci_fmt);
1397
1398 my $hostpcidesc = {
1399         optional => 1,
1400         type => 'string', format => 'pve-qm-hostpci',
1401         description => "Map host PCI devices into guest.",
1402         verbose_description =>  <<EODESCR,
1403 Map host PCI devices into guest.
1404
1405 NOTE: This option allows direct access to host hardware. So it is no longer
1406 possible to migrate such machines - use with special care.
1407
1408 CAUTION: Experimental! User reported problems with this option.
1409 EODESCR
1410 };
1411 PVE::JSONSchema::register_standard_option("pve-qm-hostpci", $hostpcidesc);
1412
1413 my $serialdesc = {
1414         optional => 1,
1415         type => 'string',
1416         pattern => '(/dev/.+|socket)',
1417         description =>  "Create a serial device inside the VM (n is 0 to 3)",
1418         verbose_description =>  <<EODESCR,
1419 Create a serial device inside the VM (n is 0 to 3), and pass through a
1420 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1421 host side (use 'qm terminal' to open a terminal connection).
1422
1423 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines - use with special care.
1424
1425 CAUTION: Experimental! User reported problems with this option.
1426 EODESCR
1427 };
1428
1429 my $paralleldesc= {
1430         optional => 1,
1431         type => 'string',
1432         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
1433         description =>  "Map host parallel devices (n is 0 to 2).",
1434         verbose_description =>  <<EODESCR,
1435 Map host parallel devices (n is 0 to 2).
1436
1437 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1438
1439 CAUTION: Experimental! User reported problems with this option.
1440 EODESCR
1441 };
1442
1443 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
1444     $confdesc->{"parallel$i"} = $paralleldesc;
1445 }
1446
1447 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
1448     $confdesc->{"serial$i"} = $serialdesc;
1449 }
1450
1451 for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
1452     $confdesc->{"hostpci$i"} = $hostpcidesc;
1453 }
1454
1455 for (my $i = 0; $i < $MAX_IDE_DISKS; $i++)  {
1456     $drivename_hash->{"ide$i"} = 1;
1457     $confdesc->{"ide$i"} = $idedesc;
1458 }
1459
1460 for (my $i = 0; $i < $MAX_SATA_DISKS; $i++)  {
1461     $drivename_hash->{"sata$i"} = 1;
1462     $confdesc->{"sata$i"} = $satadesc;
1463 }
1464
1465 for (my $i = 0; $i < $MAX_SCSI_DISKS; $i++)  {
1466     $drivename_hash->{"scsi$i"} = 1;
1467     $confdesc->{"scsi$i"} = $scsidesc ;
1468 }
1469
1470 for (my $i = 0; $i < $MAX_VIRTIO_DISKS; $i++)  {
1471     $drivename_hash->{"virtio$i"} = 1;
1472     $confdesc->{"virtio$i"} = $virtiodesc;
1473 }
1474
1475 $drivename_hash->{efidisk0} = 1;
1476 $confdesc->{efidisk0} = $efidisk_desc;
1477
1478 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
1479     $confdesc->{"usb$i"} = $usbdesc;
1480 }
1481
1482 my $unuseddesc = {
1483     optional => 1,
1484     type => 'string', format => 'pve-volume-id',
1485     description => "Reference to unused volumes. This is used internally, and should not be modified manually.",
1486 };
1487
1488 for (my $i = 0; $i < $MAX_UNUSED_DISKS; $i++)  {
1489     $confdesc->{"unused$i"} = $unuseddesc;
1490 }
1491
1492 my $kvm_api_version = 0;
1493
1494 sub kvm_version {
1495     return $kvm_api_version if $kvm_api_version;
1496
1497     open my $fh, '<', '/dev/kvm'
1498         or return undef;
1499
1500     # 0xae00 => KVM_GET_API_VERSION
1501     $kvm_api_version = ioctl($fh, 0xae00, 0);
1502
1503     return $kvm_api_version;
1504 }
1505
1506 my $kvm_user_version = {};
1507 my $kvm_mtime = {};
1508
1509 sub kvm_user_version {
1510     my ($binary) = @_;
1511
1512     $binary //= get_command_for_arch(get_host_arch()); # get the native arch by default
1513     my $st = stat($binary);
1514
1515     my $cachedmtime = $kvm_mtime->{$binary} // -1;
1516     return $kvm_user_version->{$binary} if $kvm_user_version->{$binary} &&
1517         $cachedmtime == $st->mtime;
1518
1519     $kvm_user_version->{$binary} = 'unknown';
1520     $kvm_mtime->{$binary} = $st->mtime;
1521
1522     my $code = sub {
1523         my $line = shift;
1524         if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1525             $kvm_user_version->{$binary} = $2;
1526         }
1527     };
1528
1529     eval { run_command([$binary, '--version'], outfunc => $code); };
1530     warn $@ if $@;
1531
1532     return $kvm_user_version->{$binary};
1533
1534 }
1535
1536 sub kernel_has_vhost_net {
1537     return -c '/dev/vhost-net';
1538 }
1539
1540 sub valid_drive_names {
1541     # order is important - used to autoselect boot disk
1542     return ((map { "ide$_" } (0 .. ($MAX_IDE_DISKS - 1))),
1543             (map { "scsi$_" } (0 .. ($MAX_SCSI_DISKS - 1))),
1544             (map { "virtio$_" } (0 .. ($MAX_VIRTIO_DISKS - 1))),
1545             (map { "sata$_" } (0 .. ($MAX_SATA_DISKS - 1))),
1546             'efidisk0');
1547 }
1548
1549 sub is_valid_drivename {
1550     my $dev = shift;
1551
1552     return defined($drivename_hash->{$dev});
1553 }
1554
1555 sub option_exists {
1556     my $key = shift;
1557     return defined($confdesc->{$key});
1558 }
1559
1560 my $cdrom_path;
1561 sub get_cdrom_path {
1562
1563     return  $cdrom_path if $cdrom_path;
1564
1565     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
1566     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
1567     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
1568 }
1569
1570 sub get_iso_path {
1571     my ($storecfg, $vmid, $cdrom) = @_;
1572
1573     if ($cdrom eq 'cdrom') {
1574         return get_cdrom_path();
1575     } elsif ($cdrom eq 'none') {
1576         return '';
1577     } elsif ($cdrom =~ m|^/|) {
1578         return $cdrom;
1579     } else {
1580         return PVE::Storage::path($storecfg, $cdrom);
1581     }
1582 }
1583
1584 # try to convert old style file names to volume IDs
1585 sub filename_to_volume_id {
1586     my ($vmid, $file, $media) = @_;
1587
1588      if (!($file eq 'none' || $file eq 'cdrom' ||
1589           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1590
1591         return undef if $file =~ m|/|;
1592
1593         if ($media && $media eq 'cdrom') {
1594             $file = "local:iso/$file";
1595         } else {
1596             $file = "local:$vmid/$file";
1597         }
1598     }
1599
1600     return $file;
1601 }
1602
1603 sub verify_media_type {
1604     my ($opt, $vtype, $media) = @_;
1605
1606     return if !$media;
1607
1608     my $etype;
1609     if ($media eq 'disk') {
1610         $etype = 'images';
1611     } elsif ($media eq 'cdrom') {
1612         $etype = 'iso';
1613     } else {
1614         die "internal error";
1615     }
1616
1617     return if ($vtype eq $etype);
1618
1619     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1620 }
1621
1622 sub cleanup_drive_path {
1623     my ($opt, $storecfg, $drive) = @_;
1624
1625     # try to convert filesystem paths to volume IDs
1626
1627     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1628         ($drive->{file} !~ m|^/dev/.+|) &&
1629         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1630         ($drive->{file} !~ m/^\d+$/)) {
1631         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1632         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"}) if !$vtype;
1633         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1634         verify_media_type($opt, $vtype, $drive->{media});
1635         $drive->{file} = $volid;
1636     }
1637
1638     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1639 }
1640
1641 sub parse_hotplug_features {
1642     my ($data) = @_;
1643
1644     my $res = {};
1645
1646     return $res if $data eq '0';
1647
1648     $data = $confdesc->{hotplug}->{default} if $data eq '1';
1649
1650     foreach my $feature (PVE::Tools::split_list($data)) {
1651         if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
1652             $res->{$1} = 1;
1653         } else {
1654             die "invalid hotplug feature '$feature'\n";
1655         }
1656     }
1657     return $res;
1658 }
1659
1660 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1661 sub pve_verify_hotplug_features {
1662     my ($value, $noerr) = @_;
1663
1664     return $value if parse_hotplug_features($value);
1665
1666     return undef if $noerr;
1667
1668     die "unable to parse hotplug option\n";
1669 }
1670
1671 # ideX = [volume=]volume-id[,media=d][,cyls=c,heads=h,secs=s[,trans=t]]
1672 #        [,snapshot=on|off][,cache=on|off][,format=f][,backup=yes|no]
1673 #        [,rerror=ignore|report|stop][,werror=enospc|ignore|report|stop]
1674 #        [,aio=native|threads][,discard=ignore|on][,detect_zeroes=on|off]
1675 #        [,iothread=on][,serial=serial][,model=model]
1676
1677 sub parse_drive {
1678     my ($key, $data) = @_;
1679
1680     my ($interface, $index);
1681
1682     if ($key =~ m/^([^\d]+)(\d+)$/) {
1683         $interface = $1;
1684         $index = $2;
1685     } else {
1686         return undef;
1687     }
1688
1689     my $desc = $key =~ /^unused\d+$/ ? $alldrive_fmt
1690                                      : $confdesc->{$key}->{format};
1691     if (!$desc) {
1692         warn "invalid drive key: $key\n";
1693         return undef;
1694     }
1695     my $res = eval { PVE::JSONSchema::parse_property_string($desc, $data) };
1696     return undef if !$res;
1697     $res->{interface} = $interface;
1698     $res->{index} = $index;
1699
1700     my $error = 0;
1701     foreach my $opt (qw(bps bps_rd bps_wr)) {
1702         if (my $bps = defined(delete $res->{$opt})) {
1703             if (defined($res->{"m$opt"})) {
1704                 warn "both $opt and m$opt specified\n";
1705                 ++$error;
1706                 next;
1707             }
1708             $res->{"m$opt"} = sprintf("%.3f", $bps / (1024*1024.0));
1709         }
1710     }
1711
1712     # can't use the schema's 'requires' because of the mbps* => bps* "transforming aliases"
1713     for my $requirement (
1714         [mbps_max => 'mbps'],
1715         [mbps_rd_max => 'mbps_rd'],
1716         [mbps_wr_max => 'mbps_wr'],
1717         [miops_max => 'miops'],
1718         [miops_rd_max => 'miops_rd'],
1719         [miops_wr_max => 'miops_wr'],
1720         [bps_max_length => 'mbps_max'],
1721         [bps_rd_max_length => 'mbps_rd_max'],
1722         [bps_wr_max_length => 'mbps_wr_max'],
1723         [iops_max_length => 'iops_max'],
1724         [iops_rd_max_length => 'iops_rd_max'],
1725         [iops_wr_max_length => 'iops_wr_max']) {
1726         my ($option, $requires) = @$requirement;
1727         if ($res->{$option} && !$res->{$requires}) {
1728             warn "$option requires $requires\n";
1729             ++$error;
1730         }
1731     }
1732
1733     return undef if $error;
1734
1735     return undef if $res->{mbps_rd} && $res->{mbps};
1736     return undef if $res->{mbps_wr} && $res->{mbps};
1737     return undef if $res->{iops_rd} && $res->{iops};
1738     return undef if $res->{iops_wr} && $res->{iops};
1739
1740     if ($res->{media} && ($res->{media} eq 'cdrom')) {
1741         return undef if $res->{snapshot} || $res->{trans} || $res->{format};
1742         return undef if $res->{heads} || $res->{secs} || $res->{cyls};
1743         return undef if $res->{interface} eq 'virtio';
1744     }
1745
1746     if (my $size = $res->{size}) {
1747         return undef if !defined($res->{size} = PVE::JSONSchema::parse_size($size));
1748     }
1749
1750     return $res;
1751 }
1752
1753 sub print_drive {
1754     my ($vmid, $drive) = @_;
1755     my $data = { %$drive };
1756     delete $data->{$_} for qw(index interface);
1757     return PVE::JSONSchema::print_property_string($data, $alldrive_fmt);
1758 }
1759
1760 sub scsi_inquiry {
1761     my($fh, $noerr) = @_;
1762
1763     my $SG_IO = 0x2285;
1764     my $SG_GET_VERSION_NUM = 0x2282;
1765
1766     my $versionbuf = "\x00" x 8;
1767     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1768     if (!$ret) {
1769         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1770         return undef;
1771     }
1772     my $version = unpack("I", $versionbuf);
1773     if ($version < 30000) {
1774         die "scsi generic interface too old\n"  if !$noerr;
1775         return undef;
1776     }
1777
1778     my $buf = "\x00" x 36;
1779     my $sensebuf = "\x00" x 8;
1780     my $cmd = pack("C x3 C x1", 0x12, 36);
1781
1782     # see /usr/include/scsi/sg.h
1783     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1784
1785     my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1786                       length($sensebuf), 0, length($buf), $buf,
1787                       $cmd, $sensebuf, 6000);
1788
1789     $ret = ioctl($fh, $SG_IO, $packet);
1790     if (!$ret) {
1791         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1792         return undef;
1793     }
1794
1795     my @res = unpack($sg_io_hdr_t, $packet);
1796     if ($res[17] || $res[18]) {
1797         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1798         return undef;
1799     }
1800
1801     my $res = {};
1802     (my $byte0, my $byte1, $res->{vendor},
1803      $res->{product}, $res->{revision}) = unpack("C C x6 A8 A16 A4", $buf);
1804
1805     $res->{removable} = $byte1 & 128 ? 1 : 0;
1806     $res->{type} = $byte0 & 31;
1807
1808     return $res;
1809 }
1810
1811 sub path_is_scsi {
1812     my ($path) = @_;
1813
1814     my $fh = IO::File->new("+<$path") || return undef;
1815     my $res = scsi_inquiry($fh, 1);
1816     close($fh);
1817
1818     return $res;
1819 }
1820
1821 sub print_tabletdevice_full {
1822     my ($conf, $arch) = @_;
1823
1824     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
1825
1826     # we use uhci for old VMs because tablet driver was buggy in older qemu
1827     my $usbbus;
1828     if (PVE::QemuServer::Machine::machine_type_is_q35($conf) || $arch eq 'aarch64') {
1829         $usbbus = 'ehci';
1830     } else {
1831         $usbbus = 'uhci';
1832     }
1833
1834     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1835 }
1836
1837 sub print_keyboarddevice_full {
1838     my ($conf, $arch, $machine) = @_;
1839
1840     return undef if $arch ne 'aarch64';
1841
1842     return "usb-kbd,id=keyboard,bus=ehci.0,port=2";
1843 }
1844
1845 sub print_drivedevice_full {
1846     my ($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type) = @_;
1847
1848     my $device = '';
1849     my $maxdev = 0;
1850
1851     if ($drive->{interface} eq 'virtio') {
1852         my $pciaddr = print_pci_addr("$drive->{interface}$drive->{index}", $bridges, $arch, $machine_type);
1853         $device = "virtio-blk-pci,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}$pciaddr";
1854         $device .= ",iothread=iothread-$drive->{interface}$drive->{index}" if $drive->{iothread};
1855     } elsif ($drive->{interface} eq 'scsi') {
1856
1857         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1858         my $unit = $drive->{index} % $maxdev;
1859         my $devicetype = 'hd';
1860         my $path = '';
1861         if (drive_is_cdrom($drive)) {
1862             $devicetype = 'cd';
1863         } else {
1864             if ($drive->{file} =~ m|^/|) {
1865                 $path = $drive->{file};
1866                 if (my $info = path_is_scsi($path)) {
1867                     if ($info->{type} == 0 && $drive->{scsiblock}) {
1868                         $devicetype = 'block';
1869                     } elsif ($info->{type} == 1) { # tape
1870                         $devicetype = 'generic';
1871                     }
1872                 }
1873             } else {
1874                  $path = PVE::Storage::path($storecfg, $drive->{file});
1875             }
1876
1877             # for compatibility only, we prefer scsi-hd (#2408, #2355, #2380)
1878             my $version = PVE::QemuServer::Machine::extract_version($machine_type, kvm_user_version());
1879             if ($path =~ m/^iscsi\:\/\// &&
1880                !min_version($version, 4, 1)) {
1881                 $devicetype = 'generic';
1882             }
1883         }
1884
1885         if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)){
1886             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1887         } else {
1888             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0,lun=$drive->{index},drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1889         }
1890
1891         if ($drive->{ssd} && ($devicetype eq 'block' || $devicetype eq 'hd')) {
1892             $device .= ",rotation_rate=1";
1893         }
1894         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1895
1896     } elsif ($drive->{interface} eq 'ide' || $drive->{interface} eq 'sata') {
1897         my $maxdev = ($drive->{interface} eq 'sata') ? $MAX_SATA_DISKS : 2;
1898         my $controller = int($drive->{index} / $maxdev);
1899         my $unit = $drive->{index} % $maxdev;
1900         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1901
1902         $device = "ide-$devicetype";
1903         if ($drive->{interface} eq 'ide') {
1904             $device .= ",bus=ide.$controller,unit=$unit";
1905         } else {
1906             $device .= ",bus=ahci$controller.$unit";
1907         }
1908         $device .= ",drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1909
1910         if ($devicetype eq 'hd') {
1911             if (my $model = $drive->{model}) {
1912                 $model = URI::Escape::uri_unescape($model);
1913                 $device .= ",model=$model";
1914             }
1915             if ($drive->{ssd}) {
1916                 $device .= ",rotation_rate=1";
1917             }
1918         }
1919         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1920     } elsif ($drive->{interface} eq 'usb') {
1921         die "implement me";
1922         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1923     } else {
1924         die "unsupported interface type";
1925     }
1926
1927     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1928
1929     if (my $serial = $drive->{serial}) {
1930         $serial = URI::Escape::uri_unescape($serial);
1931         $device .= ",serial=$serial";
1932     }
1933
1934
1935     return $device;
1936 }
1937
1938 sub get_initiator_name {
1939     my $initiator;
1940
1941     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return undef;
1942     while (defined(my $line = <$fh>)) {
1943         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1944         $initiator = $1;
1945         last;
1946     }
1947     $fh->close();
1948
1949     return $initiator;
1950 }
1951
1952 sub print_drive_full {
1953     my ($storecfg, $vmid, $drive) = @_;
1954
1955     my $path;
1956     my $volid = $drive->{file};
1957     my $format;
1958
1959     if (drive_is_cdrom($drive)) {
1960         $path = get_iso_path($storecfg, $vmid, $volid);
1961     } else {
1962         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1963         if ($storeid) {
1964             $path = PVE::Storage::path($storecfg, $volid);
1965             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
1966             $format = qemu_img_format($scfg, $volname);
1967         } else {
1968             $path = $volid;
1969             $format = "raw";
1970         }
1971    }
1972
1973     my $opts = '';
1974     my @qemu_drive_options = qw(heads secs cyls trans media format cache rerror werror aio discard);
1975     foreach my $o (@qemu_drive_options) {
1976         $opts .= ",$o=$drive->{$o}" if defined($drive->{$o});
1977     }
1978
1979     # snapshot only accepts on|off
1980     if (defined($drive->{snapshot})) {
1981         my $v = $drive->{snapshot} ? 'on' : 'off';
1982         $opts .= ",snapshot=$v";
1983     }
1984
1985     foreach my $type (['', '-total'], [_rd => '-read'], [_wr => '-write']) {
1986         my ($dir, $qmpname) = @$type;
1987         if (my $v = $drive->{"mbps$dir"}) {
1988             $opts .= ",throttling.bps$qmpname=".int($v*1024*1024);
1989         }
1990         if (my $v = $drive->{"mbps${dir}_max"}) {
1991             $opts .= ",throttling.bps$qmpname-max=".int($v*1024*1024);
1992         }
1993         if (my $v = $drive->{"bps${dir}_max_length"}) {
1994             $opts .= ",throttling.bps$qmpname-max-length=$v";
1995         }
1996         if (my $v = $drive->{"iops${dir}"}) {
1997             $opts .= ",throttling.iops$qmpname=$v";
1998         }
1999         if (my $v = $drive->{"iops${dir}_max"}) {
2000             $opts .= ",throttling.iops$qmpname-max=$v";
2001         }
2002         if (my $v = $drive->{"iops${dir}_max_length"}) {
2003             $opts .= ",throttling.iops$qmpname-max-length=$v";
2004         }
2005     }
2006
2007     $opts .= ",format=$format" if $format && !$drive->{format};
2008
2009     my $cache_direct = 0;
2010
2011     if (my $cache = $drive->{cache}) {
2012         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
2013     } elsif (!drive_is_cdrom($drive)) {
2014         $opts .= ",cache=none";
2015         $cache_direct = 1;
2016     }
2017
2018     # aio native works only with O_DIRECT
2019     if (!$drive->{aio}) {
2020         if($cache_direct) {
2021             $opts .= ",aio=native";
2022         } else {
2023             $opts .= ",aio=threads";
2024         }
2025     }
2026
2027     if (!drive_is_cdrom($drive)) {
2028         my $detectzeroes;
2029         if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
2030             $detectzeroes = 'off';
2031         } elsif ($drive->{discard}) {
2032             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
2033         } else {
2034             # This used to be our default with discard not being specified:
2035             $detectzeroes = 'on';
2036         }
2037         $opts .= ",detect-zeroes=$detectzeroes" if $detectzeroes;
2038     }
2039
2040     my $pathinfo = $path ? "file=$path," : '';
2041
2042     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
2043 }
2044
2045 sub print_netdevice_full {
2046     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files, $arch, $machine_type) = @_;
2047
2048     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
2049
2050     my $device = $net->{model};
2051     if ($net->{model} eq 'virtio') {
2052          $device = 'virtio-net-pci';
2053      };
2054
2055     my $pciaddr = print_pci_addr("$netid", $bridges, $arch, $machine_type);
2056     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
2057     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
2058         #Consider we have N queues, the number of vectors needed is 2*N + 2 (plus one config interrupt and control vq)
2059         my $vectors = $net->{queues} * 2 + 2;
2060         $tmpstr .= ",vectors=$vectors,mq=on";
2061     }
2062     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
2063
2064     if ($use_old_bios_files) {
2065         my $romfile;
2066         if ($device eq 'virtio-net-pci') {
2067             $romfile = 'pxe-virtio.rom';
2068         } elsif ($device eq 'e1000') {
2069             $romfile = 'pxe-e1000.rom';
2070         } elsif ($device eq 'ne2k') {
2071             $romfile = 'pxe-ne2k_pci.rom';
2072         } elsif ($device eq 'pcnet') {
2073             $romfile = 'pxe-pcnet.rom';
2074         } elsif ($device eq 'rtl8139') {
2075             $romfile = 'pxe-rtl8139.rom';
2076         }
2077         $tmpstr .= ",romfile=$romfile" if $romfile;
2078     }
2079
2080     return $tmpstr;
2081 }
2082
2083 sub print_netdev_full {
2084     my ($vmid, $conf, $arch, $net, $netid, $hotplug) = @_;
2085
2086     my $i = '';
2087     if ($netid =~ m/^net(\d+)$/) {
2088         $i = int($1);
2089     }
2090
2091     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
2092
2093     my $ifname = "tap${vmid}i$i";
2094
2095     # kvm uses TUNSETIFF ioctl, and that limits ifname length
2096     die "interface name '$ifname' is too long (max 15 character)\n"
2097         if length($ifname) >= 16;
2098
2099     my $vhostparam = '';
2100     if (is_native($arch)) {
2101         $vhostparam = ',vhost=on' if kernel_has_vhost_net() && $net->{model} eq 'virtio';
2102     }
2103
2104     my $vmname = $conf->{name} || "vm$vmid";
2105
2106     my $netdev = "";
2107     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
2108
2109     if ($net->{bridge}) {
2110         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script,downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
2111     } else {
2112         $netdev = "type=user,id=$netid,hostname=$vmname";
2113     }
2114
2115     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
2116
2117     return $netdev;
2118 }
2119
2120
2121 sub print_cpu_device {
2122     my ($conf, $id) = @_;
2123
2124     my $kvm = $conf->{kvm} // 1;
2125     my $cpu = $kvm ? "kvm64" : "qemu64";
2126     if (my $cputype = $conf->{cpu}) {
2127         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
2128             or die "Cannot parse cpu description: $cputype\n";
2129         $cpu = $cpuconf->{cputype};
2130     }
2131
2132     my $cores = $conf->{cores} || 1;
2133
2134     my $current_core = ($id - 1) % $cores;
2135     my $current_socket = int(($id - 1 - $current_core)/$cores);
2136
2137     return "$cpu-x86_64-cpu,id=cpu$id,socket-id=$current_socket,core-id=$current_core,thread-id=0";
2138 }
2139
2140 my $vga_map = {
2141     'cirrus' => 'cirrus-vga',
2142     'std' => 'VGA',
2143     'vmware' => 'vmware-svga',
2144     'virtio' => 'virtio-vga',
2145 };
2146
2147 sub print_vga_device {
2148     my ($conf, $vga, $arch, $machine_version, $machine, $id, $qxlnum, $bridges) = @_;
2149
2150     my $type = $vga_map->{$vga->{type}};
2151     if ($arch eq 'aarch64' && defined($type) && $type eq 'virtio-vga') {
2152         $type = 'virtio-gpu';
2153     }
2154     my $vgamem_mb = $vga->{memory};
2155
2156     my $max_outputs = '';
2157     if ($qxlnum) {
2158         $type = $id ? 'qxl' : 'qxl-vga';
2159
2160         if (!$conf->{ostype} || $conf->{ostype} =~ m/^(?:l\d\d)|(?:other)$/) {
2161             # set max outputs so linux can have up to 4 qxl displays with one device
2162             if (min_version($machine_version, 4, 1)) {
2163                 $max_outputs = ",max_outputs=4";
2164             }
2165         }
2166     }
2167
2168     die "no devicetype for $vga->{type}\n" if !$type;
2169
2170     my $memory = "";
2171     if ($vgamem_mb) {
2172         if ($vga->{type} eq 'virtio') {
2173             my $bytes = PVE::Tools::convert_size($vgamem_mb, "mb" => "b");
2174             $memory = ",max_hostmem=$bytes";
2175         } elsif ($qxlnum) {
2176             # from https://www.spice-space.org/multiple-monitors.html
2177             $memory = ",vgamem_mb=$vga->{memory}";
2178             my $ram = $vgamem_mb * 4;
2179             my $vram = $vgamem_mb * 2;
2180             $memory .= ",ram_size_mb=$ram,vram_size_mb=$vram";
2181         } else {
2182             $memory = ",vgamem_mb=$vga->{memory}";
2183         }
2184     } elsif ($qxlnum && $id) {
2185         $memory = ",ram_size=67108864,vram_size=33554432";
2186     }
2187
2188     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
2189     my $vgaid = "vga" . ($id // '');
2190     my $pciaddr;
2191
2192     if ($q35 && $vgaid eq 'vga') {
2193         # the first display uses pcie.0 bus on q35 machines
2194         $pciaddr = print_pcie_addr($vgaid, $bridges, $arch, $machine);
2195     } else {
2196         $pciaddr = print_pci_addr($vgaid, $bridges, $arch, $machine);
2197     }
2198
2199     return "$type,id=${vgaid}${memory}${max_outputs}${pciaddr}";
2200 }
2201
2202 sub drive_is_cloudinit {
2203     my ($drive) = @_;
2204     return $drive->{file} =~ m@[:/]vm-\d+-cloudinit(?:\.$QEMU_FORMAT_RE)?$@;
2205 }
2206
2207 sub drive_is_cdrom {
2208     my ($drive, $exclude_cloudinit) = @_;
2209
2210     return 0 if $exclude_cloudinit && drive_is_cloudinit($drive);
2211
2212     return $drive && $drive->{media} && ($drive->{media} eq 'cdrom');
2213
2214 }
2215
2216 sub parse_number_sets {
2217     my ($set) = @_;
2218     my $res = [];
2219     foreach my $part (split(/;/, $set)) {
2220         if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
2221             die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
2222             push @$res, [ $1, $2 ];
2223         } else {
2224             die "invalid range: $part\n";
2225         }
2226     }
2227     return $res;
2228 }
2229
2230 sub parse_numa {
2231     my ($data) = @_;
2232
2233     my $res = PVE::JSONSchema::parse_property_string($numa_fmt, $data);
2234     $res->{cpus} = parse_number_sets($res->{cpus}) if defined($res->{cpus});
2235     $res->{hostnodes} = parse_number_sets($res->{hostnodes}) if defined($res->{hostnodes});
2236     return $res;
2237 }
2238
2239 sub parse_hostpci {
2240     my ($value) = @_;
2241
2242     return undef if !$value;
2243
2244     my $res = PVE::JSONSchema::parse_property_string($hostpci_fmt, $value);
2245
2246     my @idlist = split(/;/, $res->{host});
2247     delete $res->{host};
2248     foreach my $id (@idlist) {
2249         if ($id =~ m/\./) { # full id 00:00.1
2250             push @{$res->{pciid}}, {
2251                 id => $id,
2252             };
2253         } else { # partial id 00:00
2254             $res->{pciid} = PVE::SysFSTools::lspci($id);
2255         }
2256     }
2257     return $res;
2258 }
2259
2260 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
2261 sub parse_net {
2262     my ($data) = @_;
2263
2264     my $res = eval { PVE::JSONSchema::parse_property_string($net_fmt, $data) };
2265     if ($@) {
2266         warn $@;
2267         return undef;
2268     }
2269     if (!defined($res->{macaddr})) {
2270         my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
2271         $res->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix});
2272     }
2273     return $res;
2274 }
2275
2276 # ipconfigX ip=cidr,gw=ip,ip6=cidr,gw6=ip
2277 sub parse_ipconfig {
2278     my ($data) = @_;
2279
2280     my $res = eval { PVE::JSONSchema::parse_property_string($ipconfig_fmt, $data) };
2281     if ($@) {
2282         warn $@;
2283         return undef;
2284     }
2285
2286     if ($res->{gw} && !$res->{ip}) {
2287         warn 'gateway specified without specifying an IP address';
2288         return undef;
2289     }
2290     if ($res->{gw6} && !$res->{ip6}) {
2291         warn 'IPv6 gateway specified without specifying an IPv6 address';
2292         return undef;
2293     }
2294     if ($res->{gw} && $res->{ip} eq 'dhcp') {
2295         warn 'gateway specified together with DHCP';
2296         return undef;
2297     }
2298     if ($res->{gw6} && $res->{ip6} !~ /^$IPV6RE/) {
2299         # gw6 + auto/dhcp
2300         warn "IPv6 gateway specified together with $res->{ip6} address";
2301         return undef;
2302     }
2303
2304     if (!$res->{ip} && !$res->{ip6}) {
2305         return { ip => 'dhcp', ip6 => 'dhcp' };
2306     }
2307
2308     return $res;
2309 }
2310
2311 sub print_net {
2312     my $net = shift;
2313
2314     return PVE::JSONSchema::print_property_string($net, $net_fmt);
2315 }
2316
2317 sub add_random_macs {
2318     my ($settings) = @_;
2319
2320     foreach my $opt (keys %$settings) {
2321         next if $opt !~ m/^net(\d+)$/;
2322         my $net = parse_net($settings->{$opt});
2323         next if !$net;
2324         $settings->{$opt} = print_net($net);
2325     }
2326 }
2327
2328 sub vm_is_volid_owner {
2329     my ($storecfg, $vmid, $volid) = @_;
2330
2331     if ($volid !~  m|^/|) {
2332         my ($path, $owner);
2333         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
2334         if ($owner && ($owner == $vmid)) {
2335             return 1;
2336         }
2337     }
2338
2339     return undef;
2340 }
2341
2342 sub vmconfig_register_unused_drive {
2343     my ($storecfg, $vmid, $conf, $drive) = @_;
2344
2345     if (drive_is_cloudinit($drive)) {
2346         eval { PVE::Storage::vdisk_free($storecfg, $drive->{file}) };
2347         warn $@ if $@;
2348     } elsif (!drive_is_cdrom($drive)) {
2349         my $volid = $drive->{file};
2350         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
2351             PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
2352         }
2353     }
2354 }
2355
2356 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str][,base64=bool]
2357 my $smbios1_fmt = {
2358     uuid => {
2359         type => 'string',
2360         pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
2361         format_description => 'UUID',
2362         description => "Set SMBIOS1 UUID.",
2363         optional => 1,
2364     },
2365     version => {
2366         type => 'string',
2367         pattern => '[A-Za-z0-9+\/]+={0,2}',
2368         format_description => 'Base64 encoded string',
2369         description => "Set SMBIOS1 version.",
2370         optional => 1,
2371     },
2372     serial => {
2373         type => 'string',
2374         pattern => '[A-Za-z0-9+\/]+={0,2}',
2375         format_description => 'Base64 encoded string',
2376         description => "Set SMBIOS1 serial number.",
2377         optional => 1,
2378     },
2379     manufacturer => {
2380         type => 'string',
2381         pattern => '[A-Za-z0-9+\/]+={0,2}',
2382         format_description => 'Base64 encoded string',
2383         description => "Set SMBIOS1 manufacturer.",
2384         optional => 1,
2385     },
2386     product => {
2387         type => 'string',
2388         pattern => '[A-Za-z0-9+\/]+={0,2}',
2389         format_description => 'Base64 encoded string',
2390         description => "Set SMBIOS1 product ID.",
2391         optional => 1,
2392     },
2393     sku => {
2394         type => 'string',
2395         pattern => '[A-Za-z0-9+\/]+={0,2}',
2396         format_description => 'Base64 encoded string',
2397         description => "Set SMBIOS1 SKU string.",
2398         optional => 1,
2399     },
2400     family => {
2401         type => 'string',
2402         pattern => '[A-Za-z0-9+\/]+={0,2}',
2403         format_description => 'Base64 encoded string',
2404         description => "Set SMBIOS1 family string.",
2405         optional => 1,
2406     },
2407     base64 => {
2408         type => 'boolean',
2409         description => 'Flag to indicate that the SMBIOS values are base64 encoded',
2410         optional => 1,
2411     },
2412 };
2413
2414 sub parse_smbios1 {
2415     my ($data) = @_;
2416
2417     my $res = eval { PVE::JSONSchema::parse_property_string($smbios1_fmt, $data) };
2418     warn $@ if $@;
2419     return $res;
2420 }
2421
2422 sub print_smbios1 {
2423     my ($smbios1) = @_;
2424     return PVE::JSONSchema::print_property_string($smbios1, $smbios1_fmt);
2425 }
2426
2427 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_fmt);
2428
2429 PVE::JSONSchema::register_format('pve-qm-bootdisk', \&verify_bootdisk);
2430 sub verify_bootdisk {
2431     my ($value, $noerr) = @_;
2432
2433     return $value if is_valid_drivename($value);
2434
2435     return undef if $noerr;
2436
2437     die "invalid boot disk '$value'\n";
2438 }
2439
2440 sub parse_watchdog {
2441     my ($value) = @_;
2442
2443     return undef if !$value;
2444
2445     my $res = eval { PVE::JSONSchema::parse_property_string($watchdog_fmt, $value) };
2446     warn $@ if $@;
2447     return $res;
2448 }
2449
2450 sub parse_guest_agent {
2451     my ($value) = @_;
2452
2453     return {} if !defined($value->{agent});
2454
2455     my $res = eval { PVE::JSONSchema::parse_property_string($agent_fmt, $value->{agent}) };
2456     warn $@ if $@;
2457
2458     # if the agent is disabled ignore the other potentially set properties
2459     return {} if !$res->{enabled};
2460     return $res;
2461 }
2462
2463 sub parse_vga {
2464     my ($value) = @_;
2465
2466     return {} if !$value;
2467     my $res = eval { PVE::JSONSchema::parse_property_string($vga_fmt, $value) };
2468     warn $@ if $@;
2469     return $res;
2470 }
2471
2472 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
2473 sub verify_usb_device {
2474     my ($value, $noerr) = @_;
2475
2476     return $value if parse_usb_device($value);
2477
2478     return undef if $noerr;
2479
2480     die "unable to parse usb device\n";
2481 }
2482
2483 # add JSON properties for create and set function
2484 sub json_config_properties {
2485     my $prop = shift;
2486
2487     foreach my $opt (keys %$confdesc) {
2488         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate' || $opt eq 'runningmachine';
2489         $prop->{$opt} = $confdesc->{$opt};
2490     }
2491
2492     return $prop;
2493 }
2494
2495 # return copy of $confdesc_cloudinit to generate documentation
2496 sub cloudinit_config_properties {
2497
2498     return dclone($confdesc_cloudinit);
2499 }
2500
2501 sub check_type {
2502     my ($key, $value) = @_;
2503
2504     die "unknown setting '$key'\n" if !$confdesc->{$key};
2505
2506     my $type = $confdesc->{$key}->{type};
2507
2508     if (!defined($value)) {
2509         die "got undefined value\n";
2510     }
2511
2512     if ($value =~ m/[\n\r]/) {
2513         die "property contains a line feed\n";
2514     }
2515
2516     if ($type eq 'boolean') {
2517         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
2518         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
2519         die "type check ('boolean') failed - got '$value'\n";
2520     } elsif ($type eq 'integer') {
2521         return int($1) if $value =~ m/^(\d+)$/;
2522         die "type check ('integer') failed - got '$value'\n";
2523     } elsif ($type eq 'number') {
2524         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
2525         die "type check ('number') failed - got '$value'\n";
2526     } elsif ($type eq 'string') {
2527         if (my $fmt = $confdesc->{$key}->{format}) {
2528             PVE::JSONSchema::check_format($fmt, $value);
2529             return $value;
2530         }
2531         $value =~ s/^\"(.*)\"$/$1/;
2532         return $value;
2533     } else {
2534         die "internal error"
2535     }
2536 }
2537
2538 sub destroy_vm {
2539     my ($storecfg, $vmid, $skiplock, $replacement_conf) = @_;
2540
2541     my $conf = PVE::QemuConfig->load_config($vmid);
2542
2543     PVE::QemuConfig->check_lock($conf) if !$skiplock;
2544
2545     if ($conf->{template}) {
2546         # check if any base image is still used by a linked clone
2547         foreach_drive($conf, sub {
2548                 my ($ds, $drive) = @_;
2549                 return if drive_is_cdrom($drive);
2550
2551                 my $volid = $drive->{file};
2552                 return if !$volid || $volid =~ m|^/|;
2553
2554                 die "base volume '$volid' is still in use by linked cloned\n"
2555                     if PVE::Storage::volume_is_base_and_used($storecfg, $volid);
2556
2557         });
2558     }
2559
2560     # only remove disks owned by this VM
2561     foreach_drive($conf, sub {
2562         my ($ds, $drive) = @_;
2563         return if drive_is_cdrom($drive, 1);
2564
2565         my $volid = $drive->{file};
2566         return if !$volid || $volid =~ m|^/|;
2567
2568         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
2569         return if !$path || !$owner || ($owner != $vmid);
2570
2571         eval { PVE::Storage::vdisk_free($storecfg, $volid) };
2572         warn "Could not remove disk '$volid', check manually: $@" if $@;
2573     });
2574
2575     # also remove unused disk
2576     my $vmdisks = PVE::Storage::vdisk_list($storecfg, undef, $vmid);
2577     PVE::Storage::foreach_volid($vmdisks, sub {
2578         my ($volid, $sid, $volname, $d) = @_;
2579         eval { PVE::Storage::vdisk_free($storecfg, $volid) };
2580         warn $@ if $@;
2581     });
2582
2583     if (defined $replacement_conf) {
2584         PVE::QemuConfig->write_config($vmid, $replacement_conf);
2585     } else {
2586         PVE::QemuConfig->destroy_config($vmid);
2587     }
2588 }
2589
2590 sub parse_vm_config {
2591     my ($filename, $raw) = @_;
2592
2593     return undef if !defined($raw);
2594
2595     my $res = {
2596         digest => Digest::SHA::sha1_hex($raw),
2597         snapshots => {},
2598         pending => {},
2599     };
2600
2601     $filename =~ m|/qemu-server/(\d+)\.conf$|
2602         || die "got strange filename '$filename'";
2603
2604     my $vmid = $1;
2605
2606     my $conf = $res;
2607     my $descr;
2608     my $section = '';
2609
2610     my @lines = split(/\n/, $raw);
2611     foreach my $line (@lines) {
2612         next if $line =~ m/^\s*$/;
2613
2614         if ($line =~ m/^\[PENDING\]\s*$/i) {
2615             $section = 'pending';
2616             if (defined($descr)) {
2617                 $descr =~ s/\s+$//;
2618                 $conf->{description} = $descr;
2619             }
2620             $descr = undef;
2621             $conf = $res->{$section} = {};
2622             next;
2623
2624         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2625             $section = $1;
2626             if (defined($descr)) {
2627                 $descr =~ s/\s+$//;
2628                 $conf->{description} = $descr;
2629             }
2630             $descr = undef;
2631             $conf = $res->{snapshots}->{$section} = {};
2632             next;
2633         }
2634
2635         if ($line =~ m/^\#(.*)\s*$/) {
2636             $descr = '' if !defined($descr);
2637             $descr .= PVE::Tools::decode_text($1) . "\n";
2638             next;
2639         }
2640
2641         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2642             $descr = '' if !defined($descr);
2643             $descr .= PVE::Tools::decode_text($2);
2644         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2645             $conf->{snapstate} = $1;
2646         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2647             my $key = $1;
2648             my $value = $2;
2649             $conf->{$key} = $value;
2650         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2651             my $value = $1;
2652             if ($section eq 'pending') {
2653                 $conf->{delete} = $value; # we parse this later
2654             } else {
2655                 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
2656             }
2657         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(.+?)\s*$/) {
2658             my $key = $1;
2659             my $value = $2;
2660             eval { $value = check_type($key, $value); };
2661             if ($@) {
2662                 warn "vm $vmid - unable to parse value of '$key' - $@";
2663             } else {
2664                 $key = 'ide2' if $key eq 'cdrom';
2665                 my $fmt = $confdesc->{$key}->{format};
2666                 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2667                     my $v = parse_drive($key, $value);
2668                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2669                         $v->{file} = $volid;
2670                         $value = print_drive($vmid, $v);
2671                     } else {
2672                         warn "vm $vmid - unable to parse value of '$key'\n";
2673                         next;
2674                     }
2675                 }
2676
2677                 $conf->{$key} = $value;
2678             }
2679         }
2680     }
2681
2682     if (defined($descr)) {
2683         $descr =~ s/\s+$//;
2684         $conf->{description} = $descr;
2685     }
2686     delete $res->{snapstate}; # just to be sure
2687
2688     return $res;
2689 }
2690
2691 sub write_vm_config {
2692     my ($filename, $conf) = @_;
2693
2694     delete $conf->{snapstate}; # just to be sure
2695
2696     if ($conf->{cdrom}) {
2697         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2698         $conf->{ide2} = $conf->{cdrom};
2699         delete $conf->{cdrom};
2700     }
2701
2702     # we do not use 'smp' any longer
2703     if ($conf->{sockets}) {
2704         delete $conf->{smp};
2705     } elsif ($conf->{smp}) {
2706         $conf->{sockets} = $conf->{smp};
2707         delete $conf->{cores};
2708         delete $conf->{smp};
2709     }
2710
2711     my $used_volids = {};
2712
2713     my $cleanup_config = sub {
2714         my ($cref, $pending, $snapname) = @_;
2715
2716         foreach my $key (keys %$cref) {
2717             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2718                 $key eq 'snapstate' || $key eq 'pending';
2719             my $value = $cref->{$key};
2720             if ($key eq 'delete') {
2721                 die "propertry 'delete' is only allowed in [PENDING]\n"
2722                     if !$pending;
2723                 # fixme: check syntax?
2724                 next;
2725             }
2726             eval { $value = check_type($key, $value); };
2727             die "unable to parse value of '$key' - $@" if $@;
2728
2729             $cref->{$key} = $value;
2730
2731             if (!$snapname && is_valid_drivename($key)) {
2732                 my $drive = parse_drive($key, $value);
2733                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2734             }
2735         }
2736     };
2737
2738     &$cleanup_config($conf);
2739
2740     &$cleanup_config($conf->{pending}, 1);
2741
2742     foreach my $snapname (keys %{$conf->{snapshots}}) {
2743         die "internal error: snapshot name '$snapname' is forbidden" if lc($snapname) eq 'pending';
2744         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2745     }
2746
2747     # remove 'unusedX' settings if we re-add a volume
2748     foreach my $key (keys %$conf) {
2749         my $value = $conf->{$key};
2750         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2751             delete $conf->{$key};
2752         }
2753     }
2754
2755     my $generate_raw_config = sub {
2756         my ($conf, $pending) = @_;
2757
2758         my $raw = '';
2759
2760         # add description as comment to top of file
2761         if (defined(my $descr = $conf->{description})) {
2762             if ($descr) {
2763                 foreach my $cl (split(/\n/, $descr)) {
2764                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2765                 }
2766             } else {
2767                 $raw .= "#\n" if $pending;
2768             }
2769         }
2770
2771         foreach my $key (sort keys %$conf) {
2772             next if $key eq 'digest' || $key eq 'description' || $key eq 'pending' || $key eq 'snapshots';
2773             $raw .= "$key: $conf->{$key}\n";
2774         }
2775         return $raw;
2776     };
2777
2778     my $raw = &$generate_raw_config($conf);
2779
2780     if (scalar(keys %{$conf->{pending}})){
2781         $raw .= "\n[PENDING]\n";
2782         $raw .= &$generate_raw_config($conf->{pending}, 1);
2783     }
2784
2785     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2786         $raw .= "\n[$snapname]\n";
2787         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2788     }
2789
2790     return $raw;
2791 }
2792
2793 sub load_defaults {
2794
2795     my $res = {};
2796
2797     # we use static defaults from our JSON schema configuration
2798     foreach my $key (keys %$confdesc) {
2799         if (defined(my $default = $confdesc->{$key}->{default})) {
2800             $res->{$key} = $default;
2801         }
2802     }
2803
2804     return $res;
2805 }
2806
2807 sub config_list {
2808     my $vmlist = PVE::Cluster::get_vmlist();
2809     my $res = {};
2810     return $res if !$vmlist || !$vmlist->{ids};
2811     my $ids = $vmlist->{ids};
2812
2813     foreach my $vmid (keys %$ids) {
2814         my $d = $ids->{$vmid};
2815         next if !$d->{node} || $d->{node} ne $nodename;
2816         next if !$d->{type} || $d->{type} ne 'qemu';
2817         $res->{$vmid}->{exists} = 1;
2818     }
2819     return $res;
2820 }
2821
2822 # test if VM uses local resources (to prevent migration)
2823 sub check_local_resources {
2824     my ($conf, $noerr) = @_;
2825
2826     my @loc_res = ();
2827
2828     push @loc_res, "hostusb" if $conf->{hostusb}; # old syntax
2829     push @loc_res, "hostpci" if $conf->{hostpci}; # old syntax
2830
2831     push @loc_res, "ivshmem" if $conf->{ivshmem};
2832
2833     foreach my $k (keys %$conf) {
2834         next if $k =~ m/^usb/ && ($conf->{$k} =~ m/^spice(?![^,])/);
2835         # sockets are safe: they will recreated be on the target side post-migrate
2836         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2837         push @loc_res, $k if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2838     }
2839
2840     die "VM uses local resources\n" if scalar @loc_res && !$noerr;
2841
2842     return \@loc_res;
2843 }
2844
2845 # check if used storages are available on all nodes (use by migrate)
2846 sub check_storage_availability {
2847     my ($storecfg, $conf, $node) = @_;
2848
2849     foreach_drive($conf, sub {
2850         my ($ds, $drive) = @_;
2851
2852         my $volid = $drive->{file};
2853         return if !$volid;
2854
2855         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2856         return if !$sid;
2857
2858         # check if storage is available on both nodes
2859         my $scfg = PVE::Storage::storage_check_node($storecfg, $sid);
2860         PVE::Storage::storage_check_node($storecfg, $sid, $node);
2861    });
2862 }
2863
2864 # list nodes where all VM images are available (used by has_feature API)
2865 sub shared_nodes {
2866     my ($conf, $storecfg) = @_;
2867
2868     my $nodelist = PVE::Cluster::get_nodelist();
2869     my $nodehash = { map { $_ => 1 } @$nodelist };
2870     my $nodename = PVE::INotify::nodename();
2871
2872     foreach_drive($conf, sub {
2873         my ($ds, $drive) = @_;
2874
2875         my $volid = $drive->{file};
2876         return if !$volid;
2877
2878         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2879         if ($storeid) {
2880             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2881             if ($scfg->{disable}) {
2882                 $nodehash = {};
2883             } elsif (my $avail = $scfg->{nodes}) {
2884                 foreach my $node (keys %$nodehash) {
2885                     delete $nodehash->{$node} if !$avail->{$node};
2886                 }
2887             } elsif (!$scfg->{shared}) {
2888                 foreach my $node (keys %$nodehash) {
2889                     delete $nodehash->{$node} if $node ne $nodename
2890                 }
2891             }
2892         }
2893     });
2894
2895     return $nodehash
2896 }
2897
2898 sub check_local_storage_availability {
2899     my ($conf, $storecfg) = @_;
2900
2901     my $nodelist = PVE::Cluster::get_nodelist();
2902     my $nodehash = { map { $_ => {} } @$nodelist };
2903
2904     foreach_drive($conf, sub {
2905         my ($ds, $drive) = @_;
2906
2907         my $volid = $drive->{file};
2908         return if !$volid;
2909
2910         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2911         if ($storeid) {
2912             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2913
2914             if ($scfg->{disable}) {
2915                 foreach my $node (keys %$nodehash) {
2916                     $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2917                 }
2918             } elsif (my $avail = $scfg->{nodes}) {
2919                 foreach my $node (keys %$nodehash) {
2920                     if (!$avail->{$node}) {
2921                         $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2922                     }
2923                 }
2924             }
2925         }
2926     });
2927
2928     foreach my $node (values %$nodehash) {
2929         if (my $unavail = $node->{unavailable_storages}) {
2930             $node->{unavailable_storages} = [ sort keys %$unavail ];
2931         }
2932     }
2933
2934     return $nodehash
2935 }
2936
2937 # Compat only, use assert_config_exists_on_node and vm_running_locally where possible
2938 sub check_running {
2939     my ($vmid, $nocheck, $node) = @_;
2940
2941     PVE::QemuConfig::assert_config_exists_on_node($vmid, $node) if !$nocheck;
2942     return PVE::QemuServer::Helpers::vm_running_locally($vmid);
2943 }
2944
2945 sub vzlist {
2946
2947     my $vzlist = config_list();
2948
2949     my $fd = IO::Dir->new($PVE::QemuServer::Helpers::var_run_tmpdir) || return $vzlist;
2950
2951     while (defined(my $de = $fd->read)) {
2952         next if $de !~ m/^(\d+)\.pid$/;
2953         my $vmid = $1;
2954         next if !defined($vzlist->{$vmid});
2955         if (my $pid = check_running($vmid)) {
2956             $vzlist->{$vmid}->{pid} = $pid;
2957         }
2958     }
2959
2960     return $vzlist;
2961 }
2962
2963 sub disksize {
2964     my ($storecfg, $conf) = @_;
2965
2966     my $bootdisk = $conf->{bootdisk};
2967     return undef if !$bootdisk;
2968     return undef if !is_valid_drivename($bootdisk);
2969
2970     return undef if !$conf->{$bootdisk};
2971
2972     my $drive = parse_drive($bootdisk, $conf->{$bootdisk});
2973     return undef if !defined($drive);
2974
2975     return undef if drive_is_cdrom($drive);
2976
2977     my $volid = $drive->{file};
2978     return undef if !$volid;
2979
2980     return $drive->{size};
2981 }
2982
2983 our $vmstatus_return_properties = {
2984     vmid => get_standard_option('pve-vmid'),
2985     status => {
2986         description => "Qemu process status.",
2987         type => 'string',
2988         enum => ['stopped', 'running'],
2989     },
2990     maxmem => {
2991         description => "Maximum memory in bytes.",
2992         type => 'integer',
2993         optional => 1,
2994         renderer => 'bytes',
2995     },
2996     maxdisk => {
2997         description => "Root disk size in bytes.",
2998         type => 'integer',
2999         optional => 1,
3000         renderer => 'bytes',
3001     },
3002     name => {
3003         description => "VM name.",
3004         type => 'string',
3005         optional => 1,
3006     },
3007     qmpstatus => {
3008         description => "Qemu QMP agent status.",
3009         type => 'string',
3010         optional => 1,
3011     },
3012     pid => {
3013         description => "PID of running qemu process.",
3014         type => 'integer',
3015         optional => 1,
3016     },
3017     uptime => {
3018         description => "Uptime.",
3019         type => 'integer',
3020         optional => 1,
3021         renderer => 'duration',
3022     },
3023     cpus => {
3024         description => "Maximum usable CPUs.",
3025         type => 'number',
3026         optional => 1,
3027     },
3028     lock => {
3029         description => "The current config lock, if any.",
3030         type => 'string',
3031         optional => 1,
3032     },
3033     tags => {
3034         description  => "The current configured tags, if any",
3035         type => 'string',
3036         optional => 1,
3037     },
3038 };
3039
3040 my $last_proc_pid_stat;
3041
3042 # get VM status information
3043 # This must be fast and should not block ($full == false)
3044 # We only query KVM using QMP if $full == true (this can be slow)
3045 sub vmstatus {
3046     my ($opt_vmid, $full) = @_;
3047
3048     my $res = {};
3049
3050     my $storecfg = PVE::Storage::config();
3051
3052     my $list = vzlist();
3053     my $defaults = load_defaults();
3054
3055     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
3056
3057     my $cpucount = $cpuinfo->{cpus} || 1;
3058
3059     foreach my $vmid (keys %$list) {
3060         next if $opt_vmid && ($vmid ne $opt_vmid);
3061
3062         my $conf = PVE::QemuConfig->load_config($vmid);
3063
3064         my $d = { vmid => $vmid };
3065         $d->{pid} = $list->{$vmid}->{pid};
3066
3067         # fixme: better status?
3068         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
3069
3070         my $size = disksize($storecfg, $conf);
3071         if (defined($size)) {
3072             $d->{disk} = 0; # no info available
3073             $d->{maxdisk} = $size;
3074         } else {
3075             $d->{disk} = 0;
3076             $d->{maxdisk} = 0;
3077         }
3078
3079         $d->{cpus} = ($conf->{sockets} || $defaults->{sockets})
3080             * ($conf->{cores} || $defaults->{cores});
3081         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
3082         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
3083
3084         $d->{name} = $conf->{name} || "VM $vmid";
3085         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024)
3086             : $defaults->{memory}*(1024*1024);
3087
3088         if ($conf->{balloon}) {
3089             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
3090             $d->{shares} = defined($conf->{shares}) ? $conf->{shares}
3091                 : $defaults->{shares};
3092         }
3093
3094         $d->{uptime} = 0;
3095         $d->{cpu} = 0;
3096         $d->{mem} = 0;
3097
3098         $d->{netout} = 0;
3099         $d->{netin} = 0;
3100
3101         $d->{diskread} = 0;
3102         $d->{diskwrite} = 0;
3103
3104         $d->{template} = PVE::QemuConfig->is_template($conf);
3105
3106         $d->{serial} = 1 if conf_has_serial($conf);
3107         $d->{lock} = $conf->{lock} if $conf->{lock};
3108         $d->{tags} = $conf->{tags} if defined($conf->{tags});
3109
3110         $res->{$vmid} = $d;
3111     }
3112
3113     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
3114     foreach my $dev (keys %$netdev) {
3115         next if $dev !~ m/^tap([1-9]\d*)i/;
3116         my $vmid = $1;
3117         my $d = $res->{$vmid};
3118         next if !$d;
3119
3120         $d->{netout} += $netdev->{$dev}->{receive};
3121         $d->{netin} += $netdev->{$dev}->{transmit};
3122
3123         if ($full) {
3124             $d->{nics}->{$dev}->{netout} = $netdev->{$dev}->{receive};
3125             $d->{nics}->{$dev}->{netin} = $netdev->{$dev}->{transmit};
3126         }
3127
3128     }
3129
3130     my $ctime = gettimeofday;
3131
3132     foreach my $vmid (keys %$list) {
3133
3134         my $d = $res->{$vmid};
3135         my $pid = $d->{pid};
3136         next if !$pid;
3137
3138         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
3139         next if !$pstat; # not running
3140
3141         my $used = $pstat->{utime} + $pstat->{stime};
3142
3143         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
3144
3145         if ($pstat->{vsize}) {
3146             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
3147         }
3148
3149         my $old = $last_proc_pid_stat->{$pid};
3150         if (!$old) {
3151             $last_proc_pid_stat->{$pid} = {
3152                 time => $ctime,
3153                 used => $used,
3154                 cpu => 0,
3155             };
3156             next;
3157         }
3158
3159         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
3160
3161         if ($dtime > 1000) {
3162             my $dutime = $used -  $old->{used};
3163
3164             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
3165             $last_proc_pid_stat->{$pid} = {
3166                 time => $ctime,
3167                 used => $used,
3168                 cpu => $d->{cpu},
3169             };
3170         } else {
3171             $d->{cpu} = $old->{cpu};
3172         }
3173     }
3174
3175     return $res if !$full;
3176
3177     my $qmpclient = PVE::QMPClient->new();
3178
3179     my $ballooncb = sub {
3180         my ($vmid, $resp) = @_;
3181
3182         my $info = $resp->{'return'};
3183         return if !$info->{max_mem};
3184
3185         my $d = $res->{$vmid};
3186
3187         # use memory assigned to VM
3188         $d->{maxmem} = $info->{max_mem};
3189         $d->{balloon} = $info->{actual};
3190
3191         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
3192             $d->{mem} = $info->{total_mem} - $info->{free_mem};
3193             $d->{freemem} = $info->{free_mem};
3194         }
3195
3196         $d->{ballooninfo} = $info;
3197     };
3198
3199     my $blockstatscb = sub {
3200         my ($vmid, $resp) = @_;
3201         my $data = $resp->{'return'} || [];
3202         my $totalrdbytes = 0;
3203         my $totalwrbytes = 0;
3204
3205         for my $blockstat (@$data) {
3206             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
3207             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
3208
3209             $blockstat->{device} =~ s/drive-//;
3210             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
3211         }
3212         $res->{$vmid}->{diskread} = $totalrdbytes;
3213         $res->{$vmid}->{diskwrite} = $totalwrbytes;
3214     };
3215
3216     my $statuscb = sub {
3217         my ($vmid, $resp) = @_;
3218
3219         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
3220         # this fails if ballon driver is not loaded, so this must be
3221         # the last commnand (following command are aborted if this fails).
3222         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
3223
3224         my $status = 'unknown';
3225         if (!defined($status = $resp->{'return'}->{status})) {
3226             warn "unable to get VM status\n";
3227             return;
3228         }
3229
3230         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
3231     };
3232
3233     foreach my $vmid (keys %$list) {
3234         next if $opt_vmid && ($vmid ne $opt_vmid);
3235         next if !$res->{$vmid}->{pid}; # not running
3236         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
3237     }
3238
3239     $qmpclient->queue_execute(undef, 2);
3240
3241     foreach my $vmid (keys %$list) {
3242         next if $opt_vmid && ($vmid ne $opt_vmid);
3243         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
3244     }
3245
3246     return $res;
3247 }
3248
3249 sub foreach_drive {
3250     my ($conf, $func, @param) = @_;
3251
3252     foreach my $ds (valid_drive_names()) {
3253         next if !defined($conf->{$ds});
3254
3255         my $drive = parse_drive($ds, $conf->{$ds});
3256         next if !$drive;
3257
3258         &$func($ds, $drive, @param);
3259     }
3260 }
3261
3262 sub foreach_volid {
3263     my ($conf, $func, @param) = @_;
3264
3265     my $volhash = {};
3266
3267     my $test_volid = sub {
3268         my ($volid, $is_cdrom, $replicate, $shared, $snapname, $size) = @_;
3269
3270         return if !$volid;
3271
3272         $volhash->{$volid}->{cdrom} //= 1;
3273         $volhash->{$volid}->{cdrom} = 0 if !$is_cdrom;
3274
3275         $volhash->{$volid}->{replicate} //= 0;
3276         $volhash->{$volid}->{replicate} = 1 if $replicate;
3277
3278         $volhash->{$volid}->{shared} //= 0;
3279         $volhash->{$volid}->{shared} = 1 if $shared;
3280
3281         $volhash->{$volid}->{referenced_in_config} //= 0;
3282         $volhash->{$volid}->{referenced_in_config} = 1 if !defined($snapname);
3283
3284         $volhash->{$volid}->{referenced_in_snapshot}->{$snapname} = 1
3285             if defined($snapname);
3286         $volhash->{$volid}->{size} = $size if $size;
3287     };
3288
3289     foreach_drive($conf, sub {
3290         my ($ds, $drive) = @_;
3291         $test_volid->($drive->{file}, drive_is_cdrom($drive), $drive->{replicate} // 1, $drive->{shared}, undef, $drive->{size});
3292     });
3293
3294     foreach my $snapname (keys %{$conf->{snapshots}}) {
3295         my $snap = $conf->{snapshots}->{$snapname};
3296         $test_volid->($snap->{vmstate}, 0, 1, $snapname);
3297         foreach_drive($snap, sub {
3298             my ($ds, $drive) = @_;
3299             $test_volid->($drive->{file}, drive_is_cdrom($drive), $drive->{replicate} // 1, $drive->{shared}, $snapname);
3300         });
3301     }
3302
3303     foreach my $volid (keys %$volhash) {
3304         &$func($volid, $volhash->{$volid}, @param);
3305     }
3306 }
3307
3308 sub conf_has_serial {
3309     my ($conf) = @_;
3310
3311     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3312         if ($conf->{"serial$i"}) {
3313             return 1;
3314         }
3315     }
3316
3317     return 0;
3318 }
3319
3320 sub conf_has_audio {
3321     my ($conf, $id) = @_;
3322
3323     $id //= 0;
3324     my $audio = $conf->{"audio$id"};
3325     return undef if !defined($audio);
3326
3327     my $audioproperties = PVE::JSONSchema::parse_property_string($audio_fmt, $audio);
3328     my $audiodriver = $audioproperties->{driver} // 'spice';
3329
3330     return {
3331         dev => $audioproperties->{device},
3332         dev_id => "audiodev$id",
3333         backend => $audiodriver,
3334         backend_id => "$audiodriver-backend${id}",
3335     };
3336 }
3337
3338 sub vga_conf_has_spice {
3339     my ($vga) = @_;
3340
3341     my $vgaconf = parse_vga($vga);
3342     my $vgatype = $vgaconf->{type};
3343     return 0 if !$vgatype || $vgatype !~ m/^qxl([234])?$/;
3344
3345     return $1 || 1;
3346 }
3347
3348 sub is_native($) {
3349     my ($arch) = @_;
3350     return get_host_arch() eq $arch;
3351 }
3352
3353 sub get_vm_arch {
3354     my ($conf) = @_;
3355     return $conf->{arch} // get_host_arch();
3356 }
3357
3358 my $default_machines = {
3359     x86_64 => 'pc',
3360     aarch64 => 'virt',
3361 };
3362
3363 sub get_vm_machine {
3364     my ($conf, $forcemachine, $arch, $add_pve_version) = @_;
3365
3366     my $machine = $forcemachine || $conf->{machine};
3367
3368     if (!$machine || $machine =~ m/^(?:pc|q35|virt)$/) {
3369         $arch //= 'x86_64';
3370         $machine ||= $default_machines->{$arch};
3371         $machine .= "+pve$PVE::QemuServer::Machine::PVE_MACHINE_VERSION" if $add_pve_version;
3372     }
3373
3374     return $machine;
3375 }
3376
3377 sub get_ovmf_files($) {
3378     my ($arch) = @_;
3379
3380     my $ovmf = $OVMF->{$arch}
3381         or die "no OVMF images known for architecture '$arch'\n";
3382
3383     return @$ovmf;
3384 }
3385
3386 my $Arch2Qemu = {
3387     aarch64 => '/usr/bin/qemu-system-aarch64',
3388     x86_64 => '/usr/bin/qemu-system-x86_64',
3389 };
3390 sub get_command_for_arch($) {
3391     my ($arch) = @_;
3392     return '/usr/bin/kvm' if is_native($arch);
3393
3394     my $cmd = $Arch2Qemu->{$arch}
3395         or die "don't know how to emulate architecture '$arch'\n";
3396     return $cmd;
3397 }
3398
3399 sub get_cpu_options {
3400     my ($conf, $arch, $kvm, $kvm_off, $machine_version, $winversion, $gpu_passthrough) = @_;
3401
3402     my $cpuFlags = [];
3403     my $ostype = $conf->{ostype};
3404
3405     my $cpu = $kvm ? "kvm64" : "qemu64";
3406     if ($arch eq 'aarch64') {
3407         $cpu = 'cortex-a57';
3408     }
3409     my $hv_vendor_id;
3410     if (my $cputype = $conf->{cpu}) {
3411         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
3412             or die "Cannot parse cpu description: $cputype\n";
3413         $cpu = $cpuconf->{cputype};
3414         $kvm_off = 1 if $cpuconf->{hidden};
3415         $hv_vendor_id = $cpuconf->{'hv-vendor-id'};
3416
3417         if (defined(my $flags = $cpuconf->{flags})) {
3418             push @$cpuFlags, split(";", $flags);
3419         }
3420     }
3421
3422     push @$cpuFlags , '+lahf_lm' if $cpu eq 'kvm64' && $arch eq 'x86_64';
3423
3424     push @$cpuFlags , '-x2apic' if $ostype && $ostype eq 'solaris';
3425
3426     push @$cpuFlags, '+sep' if $cpu eq 'kvm64' || $cpu eq 'kvm32';
3427
3428     push @$cpuFlags, '-rdtscp' if $cpu =~ m/^Opteron/;
3429
3430     if (min_version($machine_version, 2, 3) && $arch eq 'x86_64') {
3431
3432         push @$cpuFlags , '+kvm_pv_unhalt' if $kvm;
3433         push @$cpuFlags , '+kvm_pv_eoi' if $kvm;
3434     }
3435
3436     add_hyperv_enlightenments($cpuFlags, $winversion, $machine_version, $conf->{bios}, $gpu_passthrough, $hv_vendor_id) if $kvm;
3437
3438     push @$cpuFlags, 'enforce' if $cpu ne 'host' && $kvm && $arch eq 'x86_64';
3439
3440     push @$cpuFlags, 'kvm=off' if $kvm_off;
3441
3442     if (my $cpu_vendor = $cpu_vendor_list->{$cpu}) {
3443         push @$cpuFlags, "vendor=${cpu_vendor}"
3444             if $cpu_vendor ne 'default';
3445     } elsif ($arch ne 'aarch64') {
3446         die "internal error"; # should not happen
3447     }
3448
3449     $cpu .= "," . join(',', @$cpuFlags) if scalar(@$cpuFlags);
3450
3451     return ('-cpu', $cpu);
3452 }
3453
3454 sub config_to_command {
3455     my ($storecfg, $vmid, $conf, $defaults, $forcemachine) = @_;
3456
3457     my $cmd = [];
3458     my $globalFlags = [];
3459     my $machineFlags = [];
3460     my $rtcFlags = [];
3461     my $devices = [];
3462     my $pciaddr = '';
3463     my $bridges = {};
3464     my $vernum = 0; # unknown
3465     my $ostype = $conf->{ostype};
3466     my $winversion = windows_version($ostype);
3467     my $kvm = $conf->{kvm};
3468
3469     my $arch = get_vm_arch($conf);
3470     my $kvm_binary = get_command_for_arch($arch);
3471     my $kvmver = kvm_user_version($kvm_binary);
3472
3473     my $add_pve_version = min_version($kvmver, 4, 1);
3474
3475     my $machine_type = get_vm_machine($conf, $forcemachine, $arch, $add_pve_version);
3476     my $machine_version = PVE::QemuServer::Machine::extract_version($machine_type, $kvmver);
3477     $kvm //= 1 if is_native($arch);
3478
3479     if ($kvm) {
3480         die "KVM virtualisation configured, but not available. Either disable in VM configuration or enable in BIOS.\n"
3481             if !defined kvm_version();
3482     }
3483
3484     if ($kvmver =~ m/^(\d+)\.(\d+)$/) {
3485         $vernum = $1*1000000+$2*1000;
3486     } elsif ($kvmver =~ m/^(\d+)\.(\d+)\.(\d+)$/) {
3487         $vernum = $1*1000000+$2*1000+$3;
3488     }
3489
3490     die "detected old qemu-kvm binary ($kvmver)\n" if $vernum < 15000;
3491
3492     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
3493     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
3494     my $use_old_bios_files = undef;
3495     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3496
3497     my $cpuunits = defined($conf->{cpuunits}) ?
3498             $conf->{cpuunits} : $defaults->{cpuunits};
3499
3500     push @$cmd, $kvm_binary;
3501
3502     push @$cmd, '-id', $vmid;
3503
3504     my $vmname = $conf->{name} || "vm$vmid";
3505
3506     push @$cmd, '-name', $vmname;
3507
3508     my $use_virtio = 0;
3509
3510     my $qmpsocket = PVE::QemuServer::Helpers::qmp_socket($vmid);
3511     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server,nowait";
3512     push @$cmd, '-mon', "chardev=qmp,mode=control";
3513
3514     if (min_version($machine_version, 2, 12)) {
3515         push @$cmd, '-chardev', "socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5";
3516         push @$cmd, '-mon', "chardev=qmp-event,mode=control";
3517     }
3518
3519     push @$cmd, '-pidfile' , PVE::QemuServer::Helpers::pidfile_name($vmid);
3520
3521     push @$cmd, '-daemonize';
3522
3523     if ($conf->{smbios1}) {
3524         my $smbios_conf = parse_smbios1($conf->{smbios1});
3525         if ($smbios_conf->{base64}) {
3526             # Do not pass base64 flag to qemu
3527             delete $smbios_conf->{base64};
3528             my $smbios_string = "";
3529             foreach my $key (keys %$smbios_conf) {
3530                 my $value;
3531                 if ($key eq "uuid") {
3532                     $value = $smbios_conf->{uuid}
3533                 } else {
3534                     $value = decode_base64($smbios_conf->{$key});
3535                 }
3536                 # qemu accepts any binary data, only commas need escaping by double comma
3537                 $value =~ s/,/,,/g;
3538                 $smbios_string .= "," . $key . "=" . $value if $value;
3539             }
3540             push @$cmd, '-smbios', "type=1" . $smbios_string;
3541         } else {
3542             push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
3543         }
3544     }
3545
3546     if ($conf->{vmgenid}) {
3547         push @$devices, '-device', 'vmgenid,guid='.$conf->{vmgenid};
3548     }
3549
3550     my ($ovmf_code, $ovmf_vars) = get_ovmf_files($arch);
3551     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3552         die "uefi base image not found\n" if ! -f $ovmf_code;
3553
3554         my $path;
3555         my $format;
3556         if (my $efidisk = $conf->{efidisk0}) {
3557             my $d = PVE::JSONSchema::parse_property_string($efidisk_fmt, $efidisk);
3558             my ($storeid, $volname) = PVE::Storage::parse_volume_id($d->{file}, 1);
3559             $format = $d->{format};
3560             if ($storeid) {
3561                 $path = PVE::Storage::path($storecfg, $d->{file});
3562                 if (!defined($format)) {
3563                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
3564                     $format = qemu_img_format($scfg, $volname);
3565                 }
3566             } else {
3567                 $path = $d->{file};
3568                 die "efidisk format must be specified\n"
3569                     if !defined($format);
3570             }
3571         } else {
3572             warn "no efidisk configured! Using temporary efivars disk.\n";
3573             $path = "/tmp/$vmid-ovmf.fd";
3574             PVE::Tools::file_copy($ovmf_vars, $path, -s $ovmf_vars);
3575             $format = 'raw';
3576         }
3577
3578         push @$cmd, '-drive', "if=pflash,unit=0,format=raw,readonly,file=$ovmf_code";
3579         push @$cmd, '-drive', "if=pflash,unit=1,format=$format,id=drive-efidisk0,file=$path";
3580     }
3581
3582     # load q35 config
3583     if ($q35) {
3584         # we use different pcie-port hardware for qemu >= 4.0 for passthrough
3585         if (min_version($machine_version, 4, 0)) {
3586             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35-4.0.cfg';
3587         } else {
3588             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35.cfg';
3589         }
3590     }
3591
3592     # add usb controllers
3593     my @usbcontrollers = PVE::QemuServer::USB::get_usb_controllers($conf, $bridges, $arch, $machine_type, $usbdesc->{format}, $MAX_USB_DEVICES);
3594     push @$devices, @usbcontrollers if @usbcontrollers;
3595     my $vga = parse_vga($conf->{vga});
3596
3597     my $qxlnum = vga_conf_has_spice($conf->{vga});
3598     $vga->{type} = 'qxl' if $qxlnum;
3599
3600     if (!$vga->{type}) {
3601         if ($arch eq 'aarch64') {
3602             $vga->{type} = 'virtio';
3603         } elsif (min_version($machine_version, 2, 9)) {
3604             $vga->{type} = (!$winversion || $winversion >= 6) ? 'std' : 'cirrus';
3605         } else {
3606             $vga->{type} = ($winversion >= 6) ? 'std' : 'cirrus';
3607         }
3608     }
3609
3610     # enable absolute mouse coordinates (needed by vnc)
3611     my $tablet;
3612     if (defined($conf->{tablet})) {
3613         $tablet = $conf->{tablet};
3614     } else {
3615         $tablet = $defaults->{tablet};
3616         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
3617         $tablet = 0 if $vga->{type} =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
3618     }
3619
3620     if ($tablet) {
3621         push @$devices, '-device', print_tabletdevice_full($conf, $arch) if $tablet;
3622         my $kbd = print_keyboarddevice_full($conf, $arch);
3623         push @$devices, '-device', $kbd if defined($kbd);
3624     }
3625
3626     my $kvm_off = 0;
3627     my $gpu_passthrough;
3628
3629     # host pci devices
3630     for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
3631         my $id = "hostpci$i";
3632         my $d = parse_hostpci($conf->{$id});
3633         next if !$d;
3634
3635         if (my $pcie = $d->{pcie}) {
3636             die "q35 machine model is not enabled" if !$q35;
3637             # win7 wants to have the pcie devices directly on the pcie bus
3638             # instead of in the root port
3639             if ($winversion == 7) {
3640                 $pciaddr = print_pcie_addr("${id}bus0");
3641             } else {
3642                 # add more root ports if needed, 4 are present by default
3643                 # by pve-q35 cfgs, rest added here on demand.
3644                 if ($i > 3) {
3645                     push @$devices, '-device', print_pcie_root_port($i);
3646                 }
3647                 $pciaddr = print_pcie_addr($id);
3648             }
3649         } else {
3650             $pciaddr = print_pci_addr($id, $bridges, $arch, $machine_type);
3651         }
3652
3653         my $xvga = '';
3654         if ($d->{'x-vga'}) {
3655             $xvga = ',x-vga=on' if !($conf->{bios} && $conf->{bios} eq 'ovmf');
3656             $kvm_off = 1;
3657             $vga->{type} = 'none' if !defined($conf->{vga});
3658             $gpu_passthrough = 1;
3659         }
3660
3661         my $pcidevices = $d->{pciid};
3662         my $multifunction = 1 if @$pcidevices > 1;
3663
3664         my $sysfspath;
3665         if ($d->{mdev} && scalar(@$pcidevices) == 1) {
3666             my $pci_id = $pcidevices->[0]->{id};
3667             my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $i);
3668             $sysfspath = "/sys/bus/pci/devices/$pci_id/$uuid";
3669         } elsif ($d->{mdev}) {
3670             warn "ignoring mediated device '$id' with multifunction device\n";
3671         }
3672
3673         my $j=0;
3674         foreach my $pcidevice (@$pcidevices) {
3675             my $devicestr = "vfio-pci";
3676
3677             if ($sysfspath) {
3678                 $devicestr .= ",sysfsdev=$sysfspath";
3679             } else {
3680                 $devicestr .= ",host=$pcidevice->{id}";
3681             }
3682
3683             my $mf_addr = $multifunction ? ".$j" : '';
3684             $devicestr .= ",id=${id}${mf_addr}${pciaddr}${mf_addr}";
3685
3686             if ($j == 0) {
3687                 $devicestr .= ',rombar=0' if defined($d->{rombar}) && !$d->{rombar};
3688                 $devicestr .= "$xvga";
3689                 $devicestr .= ",multifunction=on" if $multifunction;
3690                 $devicestr .= ",romfile=/usr/share/kvm/$d->{romfile}" if $d->{romfile};
3691             }
3692
3693             push @$devices, '-device', $devicestr;
3694             $j++;
3695         }
3696     }
3697
3698     # usb devices
3699     my $usb_dev_features = {};
3700     $usb_dev_features->{spice_usb3} = 1 if min_version($machine_version, 4, 0);
3701
3702     my @usbdevices = PVE::QemuServer::USB::get_usb_devices($conf, $usbdesc->{format}, $MAX_USB_DEVICES, $usb_dev_features);
3703     push @$devices, @usbdevices if @usbdevices;
3704     # serial devices
3705     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3706         if (my $path = $conf->{"serial$i"}) {
3707             if ($path eq 'socket') {
3708                 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
3709                 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server,nowait";
3710                 # On aarch64, serial0 is the UART device. Qemu only allows
3711                 # connecting UART devices via the '-serial' command line, as
3712                 # the device has a fixed slot on the hardware...
3713                 if ($arch eq 'aarch64' && $i == 0) {
3714                     push @$devices, '-serial', "chardev:serial$i";
3715                 } else {
3716                     push @$devices, '-device', "isa-serial,chardev=serial$i";
3717                 }
3718             } else {
3719                 die "no such serial device\n" if ! -c $path;
3720                 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
3721                 push @$devices, '-device', "isa-serial,chardev=serial$i";
3722             }
3723         }
3724     }
3725
3726     # parallel devices
3727     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
3728         if (my $path = $conf->{"parallel$i"}) {
3729             die "no such parallel device\n" if ! -c $path;
3730             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
3731             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
3732             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
3733         }
3734     }
3735
3736     if (my $audio = conf_has_audio($conf)) {
3737
3738         my $audiopciaddr = print_pci_addr("audio0", $bridges, $arch, $machine_type);
3739
3740         my $id = $audio->{dev_id};
3741         if ($audio->{dev} eq 'AC97') {
3742             push @$devices, '-device', "AC97,id=${id}${audiopciaddr}";
3743         } elsif ($audio->{dev} =~ /intel\-hda$/) {
3744             push @$devices, '-device', "$audio->{dev},id=${id}${audiopciaddr}";
3745             push @$devices, '-device', "hda-micro,id=${id}-codec0,bus=${id}.0,cad=0";
3746             push @$devices, '-device', "hda-duplex,id=${id}-codec1,bus=${id}.0,cad=1";
3747         } else {
3748             die "unkown audio device '$audio->{dev}', implement me!";
3749         }
3750
3751         push @$devices, '-audiodev', "$audio->{backend},id=$audio->{backend_id}";
3752     }
3753
3754     my $sockets = 1;
3755     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3756     $sockets = $conf->{sockets} if  $conf->{sockets};
3757
3758     my $cores = $conf->{cores} || 1;
3759
3760     my $maxcpus = $sockets * $cores;
3761
3762     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
3763
3764     my $allowed_vcpus = $cpuinfo->{cpus};
3765
3766     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n"
3767         if ($allowed_vcpus < $maxcpus);
3768
3769     if($hotplug_features->{cpu} && min_version($machine_version, 2, 7)) {
3770
3771         push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3772         for (my $i = 2; $i <= $vcpus; $i++)  {
3773             my $cpustr = print_cpu_device($conf,$i);
3774             push @$cmd, '-device', $cpustr;
3775         }
3776
3777     } else {
3778
3779         push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3780     }
3781     push @$cmd, '-nodefaults';
3782
3783     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
3784
3785     my $bootindex_hash = {};
3786     my $i = 1;
3787     foreach my $o (split(//, $bootorder)) {
3788         $bootindex_hash->{$o} = $i*100;
3789         $i++;
3790     }
3791
3792     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
3793
3794     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
3795
3796     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
3797
3798     if ($vga->{type} && $vga->{type} !~ m/^serial\d+$/ && $vga->{type} ne 'none'){
3799         push @$devices, '-device', print_vga_device($conf, $vga, $arch, $machine_version, $machine_type, undef, $qxlnum, $bridges);
3800         my $socket = PVE::QemuServer::Helpers::vnc_socket($vmid);
3801         push @$cmd,  '-vnc', "unix:$socket,password";
3802     } else {
3803         push @$cmd, '-vga', 'none' if $vga->{type} eq 'none';
3804         push @$cmd, '-nographic';
3805     }
3806
3807     # time drift fix
3808     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
3809
3810     my $useLocaltime = $conf->{localtime};
3811
3812     if ($winversion >= 5) { # windows
3813         $useLocaltime = 1 if !defined($conf->{localtime});
3814
3815         # use time drift fix when acpi is enabled
3816         if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
3817             $tdf = 1 if !defined($conf->{tdf});
3818         }
3819     }
3820
3821     if ($winversion >= 6) {
3822         push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3823         push @$cmd, '-no-hpet';
3824     }
3825
3826     push @$rtcFlags, 'driftfix=slew' if $tdf;
3827
3828     if (!$kvm) {
3829         push @$machineFlags, 'accel=tcg';
3830     }
3831
3832     if ($machine_type) {
3833         push @$machineFlags, "type=${machine_type}";
3834     }
3835
3836     if (($conf->{startdate}) && ($conf->{startdate} ne 'now')) {
3837         push @$rtcFlags, "base=$conf->{startdate}";
3838     } elsif ($useLocaltime) {
3839         push @$rtcFlags, 'base=localtime';
3840     }
3841
3842     push @$cmd, get_cpu_options($conf, $arch, $kvm, $kvm_off, $machine_version, $winversion, $gpu_passthrough);
3843
3844     PVE::QemuServer::Memory::config($conf, $vmid, $sockets, $cores, $defaults, $hotplug_features, $cmd);
3845
3846     push @$cmd, '-S' if $conf->{freeze};
3847
3848     push @$cmd, '-k', $conf->{keyboard} if defined($conf->{keyboard});
3849
3850     my $guest_agent = parse_guest_agent($conf);
3851
3852     if ($guest_agent->{enabled}) {
3853         my $qgasocket = PVE::QemuServer::Helpers::qmp_socket($vmid, 1);
3854         push @$devices, '-chardev', "socket,path=$qgasocket,server,nowait,id=qga0";
3855
3856         if (!$guest_agent->{type} || $guest_agent->{type} eq 'virtio') {
3857             my $pciaddr = print_pci_addr("qga0", $bridges, $arch, $machine_type);
3858             push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3859             push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3860         } elsif ($guest_agent->{type} eq 'isa') {
3861             push @$devices, '-device', "isa-serial,chardev=qga0";
3862         }
3863     }
3864
3865     my $spice_port;
3866
3867     if ($qxlnum) {
3868         if ($qxlnum > 1) {
3869             if ($winversion){
3870                 for(my $i = 1; $i < $qxlnum; $i++){
3871                     push @$devices, '-device', print_vga_device($conf, $vga, $arch, $machine_version, $machine_type, $i, $qxlnum, $bridges);
3872                 }
3873             } else {
3874                 # assume other OS works like Linux
3875                 my ($ram, $vram) = ("134217728", "67108864");
3876                 if ($vga->{memory}) {
3877                     $ram = PVE::Tools::convert_size($qxlnum*4*$vga->{memory}, 'mb' => 'b');
3878                     $vram = PVE::Tools::convert_size($qxlnum*2*$vga->{memory}, 'mb' => 'b');
3879                 }
3880                 push @$cmd, '-global', "qxl-vga.ram_size=$ram";
3881                 push @$cmd, '-global', "qxl-vga.vram_size=$vram";
3882             }
3883         }
3884
3885         my $pciaddr = print_pci_addr("spice", $bridges, $arch, $machine_type);
3886
3887         my $nodename = PVE::INotify::nodename();
3888         my $pfamily = PVE::Tools::get_host_address_family($nodename);
3889         my @nodeaddrs = PVE::Tools::getaddrinfo_all('localhost', family => $pfamily);
3890         die "failed to get an ip address of type $pfamily for 'localhost'\n" if !@nodeaddrs;
3891
3892         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3893         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3894         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3895
3896         my $localhost = PVE::Network::addr_to_ip($nodeaddrs[0]->{addr});
3897         $spice_port = PVE::Tools::next_spice_port($pfamily, $localhost);
3898
3899         my $spice_enhancement = PVE::JSONSchema::parse_property_string($spice_enhancements_fmt, $conf->{spice_enhancements} // '');
3900         if ($spice_enhancement->{foldersharing}) {
3901             push @$devices, '-chardev', "spiceport,id=foldershare,name=org.spice-space.webdav.0";
3902             push @$devices, '-device', "virtserialport,chardev=foldershare,name=org.spice-space.webdav.0";
3903         }
3904
3905         my $spice_opts = "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";
3906         $spice_opts .= ",streaming-video=$spice_enhancement->{videostreaming}" if $spice_enhancement->{videostreaming};
3907         push @$devices, '-spice', "$spice_opts";
3908     }
3909
3910     # enable balloon by default, unless explicitly disabled
3911     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3912         $pciaddr = print_pci_addr("balloon0", $bridges, $arch, $machine_type);
3913         push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
3914     }
3915
3916     if ($conf->{watchdog}) {
3917         my $wdopts = parse_watchdog($conf->{watchdog});
3918         $pciaddr = print_pci_addr("watchdog", $bridges, $arch, $machine_type);
3919         my $watchdog = $wdopts->{model} || 'i6300esb';
3920         push @$devices, '-device', "$watchdog$pciaddr";
3921         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3922     }
3923
3924     my $vollist = [];
3925     my $scsicontroller = {};
3926     my $ahcicontroller = {};
3927     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3928
3929     # Add iscsi initiator name if available
3930     if (my $initiator = get_initiator_name()) {
3931         push @$devices, '-iscsi', "initiator-name=$initiator";
3932     }
3933
3934     foreach_drive($conf, sub {
3935         my ($ds, $drive) = @_;
3936
3937         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3938             push @$vollist, $drive->{file};
3939         }
3940
3941         # ignore efidisk here, already added in bios/fw handling code above
3942         return if $drive->{interface} eq 'efidisk';
3943
3944         $use_virtio = 1 if $ds =~ m/^virtio/;
3945
3946         if (drive_is_cdrom ($drive)) {
3947             if ($bootindex_hash->{d}) {
3948                 $drive->{bootindex} = $bootindex_hash->{d};
3949                 $bootindex_hash->{d} += 1;
3950             }
3951         } else {
3952             if ($bootindex_hash->{c}) {
3953                 $drive->{bootindex} = $bootindex_hash->{c} if $conf->{bootdisk} && ($conf->{bootdisk} eq $ds);
3954                 $bootindex_hash->{c} += 1;
3955             }
3956         }
3957
3958         if($drive->{interface} eq 'virtio'){
3959            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3960         }
3961
3962         if ($drive->{interface} eq 'scsi') {
3963
3964             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
3965
3966             $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges, $arch, $machine_type);
3967             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
3968
3969             my $iothread = '';
3970             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
3971                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3972                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3973             } elsif ($drive->{iothread}) {
3974                 warn "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n";
3975             }
3976
3977             my $queues = '';
3978             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
3979                 $queues = ",num_queues=$drive->{queues}";
3980             }
3981
3982             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues" if !$scsicontroller->{$controller};
3983             $scsicontroller->{$controller}=1;
3984         }
3985
3986         if ($drive->{interface} eq 'sata') {
3987            my $controller = int($drive->{index} / $MAX_SATA_DISKS);
3988            $pciaddr = print_pci_addr("ahci$controller", $bridges, $arch, $machine_type);
3989            push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr" if !$ahcicontroller->{$controller};
3990            $ahcicontroller->{$controller}=1;
3991         }
3992
3993         my $drive_cmd = print_drive_full($storecfg, $vmid, $drive);
3994         push @$devices, '-drive',$drive_cmd;
3995         push @$devices, '-device', print_drivedevice_full($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type);
3996     });
3997
3998     for (my $i = 0; $i < $MAX_NETS; $i++) {
3999          next if !$conf->{"net$i"};
4000          my $d = parse_net($conf->{"net$i"});
4001          next if !$d;
4002
4003          $use_virtio = 1 if $d->{model} eq 'virtio';
4004
4005          if ($bootindex_hash->{n}) {
4006             $d->{bootindex} = $bootindex_hash->{n};
4007             $bootindex_hash->{n} += 1;
4008          }
4009
4010          my $netdevfull = print_netdev_full($vmid, $conf, $arch, $d, "net$i");
4011          push @$devices, '-netdev', $netdevfull;
4012
4013          my $netdevicefull = print_netdevice_full($vmid, $conf, $d, "net$i", $bridges, $use_old_bios_files, $arch, $machine_type);
4014          push @$devices, '-device', $netdevicefull;
4015     }
4016
4017     if ($conf->{ivshmem}) {
4018         my $ivshmem = PVE::JSONSchema::parse_property_string($ivshmem_fmt, $conf->{ivshmem});
4019
4020         my $bus;
4021         if ($q35) {
4022             $bus = print_pcie_addr("ivshmem");
4023         } else {
4024             $bus = print_pci_addr("ivshmem", $bridges, $arch, $machine_type);
4025         }
4026
4027         my $ivshmem_name = $ivshmem->{name} // $vmid;
4028         my $path = '/dev/shm/pve-shm-' . $ivshmem_name;
4029
4030         push @$devices, '-device', "ivshmem-plain,memdev=ivshmem$bus,";
4031         push @$devices, '-object', "memory-backend-file,id=ivshmem,share=on,mem-path=$path,size=$ivshmem->{size}M";
4032     }
4033
4034     if (!$q35) {
4035         # add pci bridges
4036         if (min_version($machine_version, 2, 3)) {
4037            $bridges->{1} = 1;
4038            $bridges->{2} = 1;
4039         }
4040
4041         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
4042
4043         for my $k (sort {$b cmp $a} keys %$bridges) {
4044             $pciaddr = print_pci_addr("pci.$k", undef, $arch, $machine_type);
4045             unshift @$devices, '-device', "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr" if $k > 0;
4046         }
4047     }
4048
4049     push @$cmd, @$devices;
4050     push @$cmd, '-rtc', join(',', @$rtcFlags)
4051         if scalar(@$rtcFlags);
4052     push @$cmd, '-machine', join(',', @$machineFlags)
4053         if scalar(@$machineFlags);
4054     push @$cmd, '-global', join(',', @$globalFlags)
4055         if scalar(@$globalFlags);
4056
4057     if (my $vmstate = $conf->{vmstate}) {
4058         my $statepath = PVE::Storage::path($storecfg, $vmstate);
4059         push @$vollist, $vmstate;
4060         push @$cmd, '-loadstate', $statepath;
4061     }
4062
4063     # add custom args
4064     if ($conf->{args}) {
4065         my $aa = PVE::Tools::split_args($conf->{args});
4066         push @$cmd, @$aa;
4067     }
4068
4069     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
4070 }
4071
4072 sub spice_port {
4073     my ($vmid) = @_;
4074
4075     my $res = mon_cmd($vmid, 'query-spice');
4076
4077     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
4078 }
4079
4080 sub vm_devices_list {
4081     my ($vmid) = @_;
4082
4083     my $res = mon_cmd($vmid, 'query-pci');
4084     my $devices_to_check = [];
4085     my $devices = {};
4086     foreach my $pcibus (@$res) {
4087         push @$devices_to_check, @{$pcibus->{devices}},
4088     }
4089
4090     while (@$devices_to_check) {
4091         my $to_check = [];
4092         for my $d (@$devices_to_check) {
4093             $devices->{$d->{'qdev_id'}} = 1 if $d->{'qdev_id'};
4094             next if !$d->{'pci_bridge'};
4095
4096             $devices->{$d->{'qdev_id'}} += scalar(@{$d->{'pci_bridge'}->{devices}});
4097             push @$to_check, @{$d->{'pci_bridge'}->{devices}};
4098         }
4099         $devices_to_check = $to_check;
4100     }
4101
4102     my $resblock = mon_cmd($vmid, 'query-block');
4103     foreach my $block (@$resblock) {
4104         if($block->{device} =~ m/^drive-(\S+)/){
4105                 $devices->{$1} = 1;
4106         }
4107     }
4108
4109     my $resmice = mon_cmd($vmid, 'query-mice');
4110     foreach my $mice (@$resmice) {
4111         if ($mice->{name} eq 'QEMU HID Tablet') {
4112             $devices->{tablet} = 1;
4113             last;
4114         }
4115     }
4116
4117     # for usb devices there is no query-usb
4118     # but we can iterate over the entries in
4119     # qom-list path=/machine/peripheral
4120     my $resperipheral = mon_cmd($vmid, 'qom-list', path => '/machine/peripheral');
4121     foreach my $per (@$resperipheral) {
4122         if ($per->{name} =~ m/^usb\d+$/) {
4123             $devices->{$per->{name}} = 1;
4124         }
4125     }
4126
4127     return $devices;
4128 }
4129
4130 sub vm_deviceplug {
4131     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4132
4133     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
4134
4135     my $devices_list = vm_devices_list($vmid);
4136     return 1 if defined($devices_list->{$deviceid});
4137
4138     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid, $arch, $machine_type); # add PCI bridge if we need it for the device
4139
4140     if ($deviceid eq 'tablet') {
4141
4142         qemu_deviceadd($vmid, print_tabletdevice_full($conf, $arch));
4143
4144     } elsif ($deviceid eq 'keyboard') {
4145
4146         qemu_deviceadd($vmid, print_keyboarddevice_full($conf, $arch));
4147
4148     } elsif ($deviceid =~ m/^usb(\d+)$/) {
4149
4150         die "usb hotplug currently not reliable\n";
4151         # since we can't reliably hot unplug all added usb devices
4152         # and usb passthrough disables live migration
4153         # we disable usb hotplugging for now
4154         qemu_deviceadd($vmid, PVE::QemuServer::USB::print_usbdevice_full($conf, $deviceid, $device));
4155
4156     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4157
4158         qemu_iothread_add($vmid, $deviceid, $device);
4159
4160         qemu_driveadd($storecfg, $vmid, $device);
4161         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, $arch, $machine_type);
4162
4163         qemu_deviceadd($vmid, $devicefull);
4164         eval { qemu_deviceaddverify($vmid, $deviceid); };
4165         if (my $err = $@) {
4166             eval { qemu_drivedel($vmid, $deviceid); };
4167             warn $@ if $@;
4168             die $err;
4169         }
4170
4171     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4172
4173
4174         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
4175         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
4176         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
4177
4178         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
4179
4180         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
4181             qemu_iothread_add($vmid, $deviceid, $device);
4182             $devicefull .= ",iothread=iothread-$deviceid";
4183         }
4184
4185         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
4186             $devicefull .= ",num_queues=$device->{queues}";
4187         }
4188
4189         qemu_deviceadd($vmid, $devicefull);
4190         qemu_deviceaddverify($vmid, $deviceid);
4191
4192     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4193
4194         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device, $arch, $machine_type);
4195         qemu_driveadd($storecfg, $vmid, $device);
4196
4197         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, $arch, $machine_type);
4198         eval { qemu_deviceadd($vmid, $devicefull); };
4199         if (my $err = $@) {
4200             eval { qemu_drivedel($vmid, $deviceid); };
4201             warn $@ if $@;
4202             die $err;
4203         }
4204
4205     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4206
4207         return undef if !qemu_netdevadd($vmid, $conf, $arch, $device, $deviceid);
4208
4209         my $machine_type = PVE::QemuServer::Machine::qemu_machine_pxe($vmid, $conf);
4210         my $use_old_bios_files = undef;
4211         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
4212
4213         my $netdevicefull = print_netdevice_full($vmid, $conf, $device, $deviceid, undef, $use_old_bios_files, $arch, $machine_type);
4214         qemu_deviceadd($vmid, $netdevicefull);
4215         eval {
4216             qemu_deviceaddverify($vmid, $deviceid);
4217             qemu_set_link_status($vmid, $deviceid, !$device->{link_down});
4218         };
4219         if (my $err = $@) {
4220             eval { qemu_netdevdel($vmid, $deviceid); };
4221             warn $@ if $@;
4222             die $err;
4223         }
4224
4225     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
4226
4227         my $bridgeid = $2;
4228         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
4229         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
4230
4231         qemu_deviceadd($vmid, $devicefull);
4232         qemu_deviceaddverify($vmid, $deviceid);
4233
4234     } else {
4235         die "can't hotplug device '$deviceid'\n";
4236     }
4237
4238     return 1;
4239 }
4240
4241 # fixme: this should raise exceptions on error!
4242 sub vm_deviceunplug {
4243     my ($vmid, $conf, $deviceid) = @_;
4244
4245     my $devices_list = vm_devices_list($vmid);
4246     return 1 if !defined($devices_list->{$deviceid});
4247
4248     die "can't unplug bootdisk" if $conf->{bootdisk} && $conf->{bootdisk} eq $deviceid;
4249
4250     if ($deviceid eq 'tablet' || $deviceid eq 'keyboard') {
4251
4252         qemu_devicedel($vmid, $deviceid);
4253
4254     } elsif ($deviceid =~ m/^usb\d+$/) {
4255
4256         die "usb hotplug currently not reliable\n";
4257         # when unplugging usb devices this way,
4258         # there may be remaining usb controllers/hubs
4259         # so we disable it for now
4260         qemu_devicedel($vmid, $deviceid);
4261         qemu_devicedelverify($vmid, $deviceid);
4262
4263     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4264
4265         qemu_devicedel($vmid, $deviceid);
4266         qemu_devicedelverify($vmid, $deviceid);
4267         qemu_drivedel($vmid, $deviceid);
4268         qemu_iothread_del($conf, $vmid, $deviceid);
4269
4270     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4271
4272         qemu_devicedel($vmid, $deviceid);
4273         qemu_devicedelverify($vmid, $deviceid);
4274         qemu_iothread_del($conf, $vmid, $deviceid);
4275
4276     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4277
4278         qemu_devicedel($vmid, $deviceid);
4279         qemu_drivedel($vmid, $deviceid);
4280         qemu_deletescsihw($conf, $vmid, $deviceid);
4281
4282     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4283
4284         qemu_devicedel($vmid, $deviceid);
4285         qemu_devicedelverify($vmid, $deviceid);
4286         qemu_netdevdel($vmid, $deviceid);
4287
4288     } else {
4289         die "can't unplug device '$deviceid'\n";
4290     }
4291
4292     return 1;
4293 }
4294
4295 sub qemu_deviceadd {
4296     my ($vmid, $devicefull) = @_;
4297
4298     $devicefull = "driver=".$devicefull;
4299     my %options =  split(/[=,]/, $devicefull);
4300
4301     mon_cmd($vmid, "device_add" , %options);
4302 }
4303
4304 sub qemu_devicedel {
4305     my ($vmid, $deviceid) = @_;
4306
4307     my $ret = mon_cmd($vmid, "device_del", id => $deviceid);
4308 }
4309
4310 sub qemu_iothread_add {
4311     my($vmid, $deviceid, $device) = @_;
4312
4313     if ($device->{iothread}) {
4314         my $iothreads = vm_iothreads_list($vmid);
4315         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
4316     }
4317 }
4318
4319 sub qemu_iothread_del {
4320     my($conf, $vmid, $deviceid) = @_;
4321
4322     my $confid = $deviceid;
4323     if ($deviceid =~ m/^(?:virtioscsi|scsihw)(\d+)$/) {
4324         $confid = 'scsi' . $1;
4325     }
4326     my $device = parse_drive($confid, $conf->{$confid});
4327     if ($device->{iothread}) {
4328         my $iothreads = vm_iothreads_list($vmid);
4329         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
4330     }
4331 }
4332
4333 sub qemu_objectadd {
4334     my($vmid, $objectid, $qomtype) = @_;
4335
4336     mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
4337
4338     return 1;
4339 }
4340
4341 sub qemu_objectdel {
4342     my($vmid, $objectid) = @_;
4343
4344     mon_cmd($vmid, "object-del", id => $objectid);
4345
4346     return 1;
4347 }
4348
4349 sub qemu_driveadd {
4350     my ($storecfg, $vmid, $device) = @_;
4351
4352     my $drive = print_drive_full($storecfg, $vmid, $device);
4353     $drive =~ s/\\/\\\\/g;
4354     my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_add auto \"$drive\"");
4355
4356     # If the command succeeds qemu prints: "OK"
4357     return 1 if $ret =~ m/OK/s;
4358
4359     die "adding drive failed: $ret\n";
4360 }
4361
4362 sub qemu_drivedel {
4363     my($vmid, $deviceid) = @_;
4364
4365     my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_del drive-$deviceid");
4366     $ret =~ s/^\s+//;
4367
4368     return 1 if $ret eq "";
4369
4370     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
4371     return 1 if $ret =~ m/Device \'.*?\' not found/s;
4372
4373     die "deleting drive $deviceid failed : $ret\n";
4374 }
4375
4376 sub qemu_deviceaddverify {
4377     my ($vmid, $deviceid) = @_;
4378
4379     for (my $i = 0; $i <= 5; $i++) {
4380          my $devices_list = vm_devices_list($vmid);
4381          return 1 if defined($devices_list->{$deviceid});
4382          sleep 1;
4383     }
4384
4385     die "error on hotplug device '$deviceid'\n";
4386 }
4387
4388
4389 sub qemu_devicedelverify {
4390     my ($vmid, $deviceid) = @_;
4391
4392     # need to verify that the device is correctly removed as device_del
4393     # is async and empty return is not reliable
4394
4395     for (my $i = 0; $i <= 5; $i++) {
4396          my $devices_list = vm_devices_list($vmid);
4397          return 1 if !defined($devices_list->{$deviceid});
4398          sleep 1;
4399     }
4400
4401     die "error on hot-unplugging device '$deviceid'\n";
4402 }
4403
4404 sub qemu_findorcreatescsihw {
4405     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4406
4407     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4408
4409     my $scsihwid="$controller_prefix$controller";
4410     my $devices_list = vm_devices_list($vmid);
4411
4412     if(!defined($devices_list->{$scsihwid})) {
4413         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device, $arch, $machine_type);
4414     }
4415
4416     return 1;
4417 }
4418
4419 sub qemu_deletescsihw {
4420     my ($conf, $vmid, $opt) = @_;
4421
4422     my $device = parse_drive($opt, $conf->{$opt});
4423
4424     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
4425         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
4426         return 1;
4427     }
4428
4429     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4430
4431     my $devices_list = vm_devices_list($vmid);
4432     foreach my $opt (keys %{$devices_list}) {
4433         if (PVE::QemuServer::is_valid_drivename($opt)) {
4434             my $drive = PVE::QemuServer::parse_drive($opt, $conf->{$opt});
4435             if($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
4436                 return 1;
4437             }
4438         }
4439     }
4440
4441     my $scsihwid="scsihw$controller";
4442
4443     vm_deviceunplug($vmid, $conf, $scsihwid);
4444
4445     return 1;
4446 }
4447
4448 sub qemu_add_pci_bridge {
4449     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4450
4451     my $bridges = {};
4452
4453     my $bridgeid;
4454
4455     print_pci_addr($device, $bridges, $arch, $machine_type);
4456
4457     while (my ($k, $v) = each %$bridges) {
4458         $bridgeid = $k;
4459     }
4460     return 1 if !defined($bridgeid) || $bridgeid < 1;
4461
4462     my $bridge = "pci.$bridgeid";
4463     my $devices_list = vm_devices_list($vmid);
4464
4465     if (!defined($devices_list->{$bridge})) {
4466         vm_deviceplug($storecfg, $conf, $vmid, $bridge, $arch, $machine_type);
4467     }
4468
4469     return 1;
4470 }
4471
4472 sub qemu_set_link_status {
4473     my ($vmid, $device, $up) = @_;
4474
4475     mon_cmd($vmid, "set_link", name => $device,
4476                up => $up ? JSON::true : JSON::false);
4477 }
4478
4479 sub qemu_netdevadd {
4480     my ($vmid, $conf, $arch, $device, $deviceid) = @_;
4481
4482     my $netdev = print_netdev_full($vmid, $conf, $arch, $device, $deviceid, 1);
4483     my %options =  split(/[=,]/, $netdev);
4484
4485     mon_cmd($vmid, "netdev_add",  %options);
4486     return 1;
4487 }
4488
4489 sub qemu_netdevdel {
4490     my ($vmid, $deviceid) = @_;
4491
4492     mon_cmd($vmid, "netdev_del", id => $deviceid);
4493 }
4494
4495 sub qemu_usb_hotplug {
4496     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4497
4498     return if !$device;
4499
4500     # remove the old one first
4501     vm_deviceunplug($vmid, $conf, $deviceid);
4502
4503     # check if xhci controller is necessary and available
4504     if ($device->{usb3}) {
4505
4506         my $devicelist = vm_devices_list($vmid);
4507
4508         if (!$devicelist->{xhci}) {
4509             my $pciaddr = print_pci_addr("xhci", undef, $arch, $machine_type);
4510             qemu_deviceadd($vmid, "nec-usb-xhci,id=xhci$pciaddr");
4511         }
4512     }
4513     my $d = parse_usb_device($device->{host});
4514     $d->{usb3} = $device->{usb3};
4515
4516     # add the new one
4517     vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $d, $arch, $machine_type);
4518 }
4519
4520 sub qemu_cpu_hotplug {
4521     my ($vmid, $conf, $vcpus) = @_;
4522
4523     my $machine_type = PVE::QemuServer::Machine::get_current_qemu_machine($vmid);
4524
4525     my $sockets = 1;
4526     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
4527     $sockets = $conf->{sockets} if  $conf->{sockets};
4528     my $cores = $conf->{cores} || 1;
4529     my $maxcpus = $sockets * $cores;
4530
4531     $vcpus = $maxcpus if !$vcpus;
4532
4533     die "you can't add more vcpus than maxcpus\n"
4534         if $vcpus > $maxcpus;
4535
4536     my $currentvcpus = $conf->{vcpus} || $maxcpus;
4537
4538     if ($vcpus < $currentvcpus) {
4539
4540         if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4541
4542             for (my $i = $currentvcpus; $i > $vcpus; $i--) {
4543                 qemu_devicedel($vmid, "cpu$i");
4544                 my $retry = 0;
4545                 my $currentrunningvcpus = undef;
4546                 while (1) {
4547                     $currentrunningvcpus = mon_cmd($vmid, "query-cpus");
4548                     last if scalar(@{$currentrunningvcpus}) == $i-1;
4549                     raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
4550                     $retry++;
4551                     sleep 1;
4552                 }
4553                 #update conf after each succesfull cpu unplug
4554                 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4555                 PVE::QemuConfig->write_config($vmid, $conf);
4556             }
4557         } else {
4558             die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
4559         }
4560
4561         return;
4562     }
4563
4564     my $currentrunningvcpus = mon_cmd($vmid, "query-cpus");
4565     die "vcpus in running vm does not match its configuration\n"
4566         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
4567
4568     if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4569
4570         for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
4571             my $cpustr = print_cpu_device($conf, $i);
4572             qemu_deviceadd($vmid, $cpustr);
4573
4574             my $retry = 0;
4575             my $currentrunningvcpus = undef;
4576             while (1) {
4577                 $currentrunningvcpus = mon_cmd($vmid, "query-cpus");
4578                 last if scalar(@{$currentrunningvcpus}) == $i;
4579                 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
4580                 sleep 1;
4581                 $retry++;
4582             }
4583             #update conf after each succesfull cpu hotplug
4584             $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4585             PVE::QemuConfig->write_config($vmid, $conf);
4586         }
4587     } else {
4588
4589         for (my $i = $currentvcpus; $i < $vcpus; $i++) {
4590             mon_cmd($vmid, "cpu-add", id => int($i));
4591         }
4592     }
4593 }
4594
4595 sub qemu_block_set_io_throttle {
4596     my ($vmid, $deviceid,
4597         $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
4598         $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max,
4599         $bps_max_length, $bps_rd_max_length, $bps_wr_max_length,
4600         $iops_max_length, $iops_rd_max_length, $iops_wr_max_length) = @_;
4601
4602     return if !check_running($vmid) ;
4603
4604     mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
4605         bps => int($bps),
4606         bps_rd => int($bps_rd),
4607         bps_wr => int($bps_wr),
4608         iops => int($iops),
4609         iops_rd => int($iops_rd),
4610         iops_wr => int($iops_wr),
4611         bps_max => int($bps_max),
4612         bps_rd_max => int($bps_rd_max),
4613         bps_wr_max => int($bps_wr_max),
4614         iops_max => int($iops_max),
4615         iops_rd_max => int($iops_rd_max),
4616         iops_wr_max => int($iops_wr_max),
4617         bps_max_length => int($bps_max_length),
4618         bps_rd_max_length => int($bps_rd_max_length),
4619         bps_wr_max_length => int($bps_wr_max_length),
4620         iops_max_length => int($iops_max_length),
4621         iops_rd_max_length => int($iops_rd_max_length),
4622         iops_wr_max_length => int($iops_wr_max_length),
4623     );
4624
4625 }
4626
4627 # old code, only used to shutdown old VM after update
4628 sub __read_avail {
4629     my ($fh, $timeout) = @_;
4630
4631     my $sel = new IO::Select;
4632     $sel->add($fh);
4633
4634     my $res = '';
4635     my $buf;
4636
4637     my @ready;
4638     while (scalar (@ready = $sel->can_read($timeout))) {
4639         my $count;
4640         if ($count = $fh->sysread($buf, 8192)) {
4641             if ($buf =~ /^(.*)\(qemu\) $/s) {
4642                 $res .= $1;
4643                 last;
4644             } else {
4645                 $res .= $buf;
4646             }
4647         } else {
4648             if (!defined($count)) {
4649                 die "$!\n";
4650             }
4651             last;
4652         }
4653     }
4654
4655     die "monitor read timeout\n" if !scalar(@ready);
4656
4657     return $res;
4658 }
4659
4660 sub qemu_block_resize {
4661     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4662
4663     my $running = check_running($vmid);
4664
4665     $size = 0 if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4666
4667     return if !$running;
4668
4669     mon_cmd($vmid, "block_resize", device => $deviceid, size => int($size));
4670
4671 }
4672
4673 sub qemu_volume_snapshot {
4674     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4675
4676     my $running = check_running($vmid);
4677
4678     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4679         mon_cmd($vmid, 'blockdev-snapshot-internal-sync', device => $deviceid, name => $snap);
4680     } else {
4681         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4682     }
4683 }
4684
4685 sub qemu_volume_snapshot_delete {
4686     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4687
4688     my $running = check_running($vmid);
4689
4690     if($running) {
4691
4692         $running = undef;
4693         my $conf = PVE::QemuConfig->load_config($vmid);
4694         foreach_drive($conf, sub {
4695             my ($ds, $drive) = @_;
4696             $running = 1 if $drive->{file} eq $volid;
4697         });
4698     }
4699
4700     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4701         mon_cmd($vmid, 'blockdev-snapshot-delete-internal-sync', device => $deviceid, name => $snap);
4702     } else {
4703         PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4704     }
4705 }
4706
4707 sub set_migration_caps {
4708     my ($vmid) = @_;
4709
4710     my $cap_ref = [];
4711
4712     my $enabled_cap = {
4713         "auto-converge" => 1,
4714         "xbzrle" => 1,
4715         "x-rdma-pin-all" => 0,
4716         "zero-blocks" => 0,
4717         "compress" => 0
4718     };
4719
4720     my $supported_capabilities = mon_cmd($vmid, "query-migrate-capabilities");
4721
4722     for my $supported_capability (@$supported_capabilities) {
4723         push @$cap_ref, {
4724             capability => $supported_capability->{capability},
4725             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4726         };
4727     }
4728
4729     mon_cmd($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4730 }
4731
4732 my $fast_plug_option = {
4733     'lock' => 1,
4734     'name' => 1,
4735     'onboot' => 1,
4736     'shares' => 1,
4737     'startup' => 1,
4738     'description' => 1,
4739     'protection' => 1,
4740     'vmstatestorage' => 1,
4741     'hookscript' => 1,
4742     'tags' => 1,
4743 };
4744
4745 # hotplug changes in [PENDING]
4746 # $selection hash can be used to only apply specified options, for
4747 # example: { cores => 1 } (only apply changed 'cores')
4748 # $errors ref is used to return error messages
4749 sub vmconfig_hotplug_pending {
4750     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4751
4752     my $defaults = load_defaults();
4753     my $arch = get_vm_arch($conf);
4754     my $machine_type = get_vm_machine($conf, undef, $arch);
4755
4756     # commit values which do not have any impact on running VM first
4757     # Note: those option cannot raise errors, we we do not care about
4758     # $selection and always apply them.
4759
4760     my $add_error = sub {
4761         my ($opt, $msg) = @_;
4762         $errors->{$opt} = "hotplug problem - $msg";
4763     };
4764
4765     my $changes = 0;
4766     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4767         if ($fast_plug_option->{$opt}) {
4768             $conf->{$opt} = $conf->{pending}->{$opt};
4769             delete $conf->{pending}->{$opt};
4770             $changes = 1;
4771         }
4772     }
4773
4774     if ($changes) {
4775         PVE::QemuConfig->write_config($vmid, $conf);
4776         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4777     }
4778
4779     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4780
4781     my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4782     foreach my $opt (sort keys %$pending_delete_hash) {
4783         next if $selection && !$selection->{$opt};
4784         my $force = $pending_delete_hash->{$opt}->{force};
4785         eval {
4786             if ($opt eq 'hotplug') {
4787                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
4788             } elsif ($opt eq 'tablet') {
4789                 die "skip\n" if !$hotplug_features->{usb};
4790                 if ($defaults->{tablet}) {
4791                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4792                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4793                         if $arch eq 'aarch64';
4794                 } else {
4795                     vm_deviceunplug($vmid, $conf, 'tablet');
4796                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4797                 }
4798             } elsif ($opt =~ m/^usb\d+/) {
4799                 die "skip\n";
4800                 # since we cannot reliably hot unplug usb devices
4801                 # we are disabling it
4802                 die "skip\n" if !$hotplug_features->{usb} || $conf->{$opt} =~ m/spice/i;
4803                 vm_deviceunplug($vmid, $conf, $opt);
4804             } elsif ($opt eq 'vcpus') {
4805                 die "skip\n" if !$hotplug_features->{cpu};
4806                 qemu_cpu_hotplug($vmid, $conf, undef);
4807             } elsif ($opt eq 'balloon') {
4808                 # enable balloon device is not hotpluggable
4809                 die "skip\n" if defined($conf->{balloon}) && $conf->{balloon} == 0;
4810                 # here we reset the ballooning value to memory
4811                 my $balloon = $conf->{memory} || $defaults->{memory};
4812                 mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4813             } elsif ($fast_plug_option->{$opt}) {
4814                 # do nothing
4815             } elsif ($opt =~ m/^net(\d+)$/) {
4816                 die "skip\n" if !$hotplug_features->{network};
4817                 vm_deviceunplug($vmid, $conf, $opt);
4818             } elsif (is_valid_drivename($opt)) {
4819                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4820                 vm_deviceunplug($vmid, $conf, $opt);
4821                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4822             } elsif ($opt =~ m/^memory$/) {
4823                 die "skip\n" if !$hotplug_features->{memory};
4824                 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
4825             } elsif ($opt eq 'cpuunits') {
4826                 cgroups_write("cpu", $vmid, "cpu.shares", $defaults->{cpuunits});
4827             } elsif ($opt eq 'cpulimit') {
4828                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", -1);
4829             } else {
4830                 die "skip\n";
4831             }
4832         };
4833         if (my $err = $@) {
4834             &$add_error($opt, $err) if $err ne "skip\n";
4835         } else {
4836             # save new config if hotplug was successful
4837             delete $conf->{$opt};
4838             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4839             PVE::QemuConfig->write_config($vmid, $conf);
4840             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4841         }
4842     }
4843
4844     my ($apply_pending_cloudinit, $apply_pending_cloudinit_done);
4845     $apply_pending_cloudinit = sub {
4846         return if $apply_pending_cloudinit_done; # once is enough
4847         $apply_pending_cloudinit_done = 1; # once is enough
4848
4849         my ($key, $value) = @_;
4850
4851         my @cloudinit_opts = keys %$confdesc_cloudinit;
4852         foreach my $opt (keys %{$conf->{pending}}) {
4853             next if !grep { $_ eq $opt } @cloudinit_opts;
4854             $conf->{$opt} = delete $conf->{pending}->{$opt};
4855         }
4856
4857         my $new_conf = { %$conf };
4858         $new_conf->{$key} = $value;
4859         PVE::QemuServer::Cloudinit::generate_cloudinitconfig($new_conf, $vmid);
4860     };
4861
4862     foreach my $opt (keys %{$conf->{pending}}) {
4863         next if $selection && !$selection->{$opt};
4864         my $value = $conf->{pending}->{$opt};
4865         eval {
4866             if ($opt eq 'hotplug') {
4867                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
4868             } elsif ($opt eq 'tablet') {
4869                 die "skip\n" if !$hotplug_features->{usb};
4870                 if ($value == 1) {
4871                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4872                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4873                         if $arch eq 'aarch64';
4874                 } elsif ($value == 0) {
4875                     vm_deviceunplug($vmid, $conf, 'tablet');
4876                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4877                 }
4878             } elsif ($opt =~ m/^usb\d+$/) {
4879                 die "skip\n";
4880                 # since we cannot reliably hot unplug usb devices
4881                 # we are disabling it
4882                 die "skip\n" if !$hotplug_features->{usb} || $value =~ m/spice/i;
4883                 my $d = eval { PVE::JSONSchema::parse_property_string($usbdesc->{format}, $value) };
4884                 die "skip\n" if !$d;
4885                 qemu_usb_hotplug($storecfg, $conf, $vmid, $opt, $d, $arch, $machine_type);
4886             } elsif ($opt eq 'vcpus') {
4887                 die "skip\n" if !$hotplug_features->{cpu};
4888                 qemu_cpu_hotplug($vmid, $conf, $value);
4889             } elsif ($opt eq 'balloon') {
4890                 # enable/disable balloning device is not hotpluggable
4891                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
4892                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
4893                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
4894
4895                 # allow manual ballooning if shares is set to zero
4896                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
4897                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
4898                     mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4899                 }
4900             } elsif ($opt =~ m/^net(\d+)$/) {
4901                 # some changes can be done without hotplug
4902                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
4903                                     $vmid, $opt, $value, $arch, $machine_type);
4904             } elsif (is_valid_drivename($opt)) {
4905                 die "skip\n" if $opt eq 'efidisk0';
4906                 # some changes can be done without hotplug
4907                 my $drive = parse_drive($opt, $value);
4908                 if (drive_is_cloudinit($drive)) {
4909                     &$apply_pending_cloudinit($opt, $value);
4910                 }
4911                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
4912                                      $vmid, $opt, $value, 1, $arch, $machine_type);
4913             } elsif ($opt =~ m/^memory$/) { #dimms
4914                 die "skip\n" if !$hotplug_features->{memory};
4915                 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
4916             } elsif ($opt eq 'cpuunits') {
4917                 cgroups_write("cpu", $vmid, "cpu.shares", $conf->{pending}->{$opt});
4918             } elsif ($opt eq 'cpulimit') {
4919                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
4920                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", $cpulimit);
4921             } else {
4922                 die "skip\n";  # skip non-hot-pluggable options
4923             }
4924         };
4925         if (my $err = $@) {
4926             &$add_error($opt, $err) if $err ne "skip\n";
4927         } else {
4928             # save new config if hotplug was successful
4929             $conf->{$opt} = $value;
4930             delete $conf->{pending}->{$opt};
4931             PVE::QemuConfig->write_config($vmid, $conf);
4932             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4933         }
4934     }
4935 }
4936
4937 sub try_deallocate_drive {
4938     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
4939
4940     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
4941         my $volid = $drive->{file};
4942         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
4943             my $sid = PVE::Storage::parse_volume_id($volid);
4944             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
4945
4946             # check if the disk is really unused
4947             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
4948                 if is_volume_in_use($storecfg, $conf, $key, $volid);
4949             PVE::Storage::vdisk_free($storecfg, $volid);
4950             return 1;
4951         } else {
4952             # If vm is not owner of this disk remove from config
4953             return 1;
4954         }
4955     }
4956
4957     return undef;
4958 }
4959
4960 sub vmconfig_delete_or_detach_drive {
4961     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
4962
4963     my $drive = parse_drive($opt, $conf->{$opt});
4964
4965     my $rpcenv = PVE::RPCEnvironment::get();
4966     my $authuser = $rpcenv->get_user();
4967
4968     if ($force) {
4969         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
4970         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
4971     } else {
4972         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
4973     }
4974 }
4975
4976
4977
4978 sub vmconfig_apply_pending {
4979     my ($vmid, $conf, $storecfg) = @_;
4980
4981     # cold plug
4982
4983     my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4984     foreach my $opt (sort keys %$pending_delete_hash) {
4985         die "internal error" if $opt =~ m/^unused/;
4986         my $force = $pending_delete_hash->{$opt}->{force};
4987         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4988         if (!defined($conf->{$opt})) {
4989             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4990             PVE::QemuConfig->write_config($vmid, $conf);
4991         } elsif (is_valid_drivename($opt)) {
4992             vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4993             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4994             delete $conf->{$opt};
4995             PVE::QemuConfig->write_config($vmid, $conf);
4996         } else {
4997             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4998             delete $conf->{$opt};
4999             PVE::QemuConfig->write_config($vmid, $conf);
5000         }
5001     }
5002
5003     $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5004
5005     foreach my $opt (keys %{$conf->{pending}}) { # add/change
5006         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5007
5008         if (defined($conf->{$opt}) && ($conf->{$opt} eq $conf->{pending}->{$opt})) {
5009             # skip if nothing changed
5010         } elsif (is_valid_drivename($opt)) {
5011             vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
5012                 if defined($conf->{$opt});
5013             $conf->{$opt} = $conf->{pending}->{$opt};
5014         } else {
5015             $conf->{$opt} = $conf->{pending}->{$opt};
5016         }
5017
5018         delete $conf->{pending}->{$opt};
5019         PVE::QemuConfig->write_config($vmid, $conf);
5020     }
5021 }
5022
5023 my $safe_num_ne = sub {
5024     my ($a, $b) = @_;
5025
5026     return 0 if !defined($a) && !defined($b);
5027     return 1 if !defined($a);
5028     return 1 if !defined($b);
5029
5030     return $a != $b;
5031 };
5032
5033 my $safe_string_ne = sub {
5034     my ($a, $b) = @_;
5035
5036     return 0 if !defined($a) && !defined($b);
5037     return 1 if !defined($a);
5038     return 1 if !defined($b);
5039
5040     return $a ne $b;
5041 };
5042
5043 sub vmconfig_update_net {
5044     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
5045
5046     my $newnet = parse_net($value);
5047
5048     if ($conf->{$opt}) {
5049         my $oldnet = parse_net($conf->{$opt});
5050
5051         if (&$safe_string_ne($oldnet->{model}, $newnet->{model}) ||
5052             &$safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
5053             &$safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
5054             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
5055
5056             # for non online change, we try to hot-unplug
5057             die "skip\n" if !$hotplug;
5058             vm_deviceunplug($vmid, $conf, $opt);
5059         } else {
5060
5061             die "internal error" if $opt !~ m/net(\d+)/;
5062             my $iface = "tap${vmid}i$1";
5063
5064             if (&$safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
5065                 &$safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
5066                 &$safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
5067                 &$safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
5068                 PVE::Network::tap_unplug($iface);
5069                 PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
5070             } elsif (&$safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
5071                 # Rate can be applied on its own but any change above needs to
5072                 # include the rate in tap_plug since OVS resets everything.
5073                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
5074             }
5075
5076             if (&$safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
5077                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
5078             }
5079
5080             return 1;
5081         }
5082     }
5083
5084     if ($hotplug) {
5085         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet, $arch, $machine_type);
5086     } else {
5087         die "skip\n";
5088     }
5089 }
5090
5091 sub vmconfig_update_disk {
5092     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $force, $arch, $machine_type) = @_;
5093
5094     # fixme: do we need force?
5095
5096     my $drive = parse_drive($opt, $value);
5097
5098     if ($conf->{$opt}) {
5099
5100         if (my $old_drive = parse_drive($opt, $conf->{$opt}))  {
5101
5102             my $media = $drive->{media} || 'disk';
5103             my $oldmedia = $old_drive->{media} || 'disk';
5104             die "unable to change media type\n" if $media ne $oldmedia;
5105
5106             if (!drive_is_cdrom($old_drive)) {
5107
5108                 if ($drive->{file} ne $old_drive->{file}) {
5109
5110                     die "skip\n" if !$hotplug;
5111
5112                     # unplug and register as unused
5113                     vm_deviceunplug($vmid, $conf, $opt);
5114                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
5115
5116                 } else {
5117                     # update existing disk
5118
5119                     # skip non hotpluggable value
5120                     if (&$safe_string_ne($drive->{discard}, $old_drive->{discard}) ||
5121                         &$safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
5122                         &$safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
5123                         &$safe_string_ne($drive->{cache}, $old_drive->{cache})) {
5124                         die "skip\n";
5125                     }
5126
5127                     # apply throttle
5128                     if (&$safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
5129                         &$safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
5130                         &$safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
5131                         &$safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
5132                         &$safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
5133                         &$safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
5134                         &$safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
5135                         &$safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
5136                         &$safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
5137                         &$safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
5138                         &$safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
5139                         &$safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max}) ||
5140                         &$safe_num_ne($drive->{bps_max_length}, $old_drive->{bps_max_length}) ||
5141                         &$safe_num_ne($drive->{bps_rd_max_length}, $old_drive->{bps_rd_max_length}) ||
5142                         &$safe_num_ne($drive->{bps_wr_max_length}, $old_drive->{bps_wr_max_length}) ||
5143                         &$safe_num_ne($drive->{iops_max_length}, $old_drive->{iops_max_length}) ||
5144                         &$safe_num_ne($drive->{iops_rd_max_length}, $old_drive->{iops_rd_max_length}) ||
5145                         &$safe_num_ne($drive->{iops_wr_max_length}, $old_drive->{iops_wr_max_length})) {
5146
5147                         qemu_block_set_io_throttle($vmid,"drive-$opt",
5148                                                    ($drive->{mbps} || 0)*1024*1024,
5149                                                    ($drive->{mbps_rd} || 0)*1024*1024,
5150                                                    ($drive->{mbps_wr} || 0)*1024*1024,
5151                                                    $drive->{iops} || 0,
5152                                                    $drive->{iops_rd} || 0,
5153                                                    $drive->{iops_wr} || 0,
5154                                                    ($drive->{mbps_max} || 0)*1024*1024,
5155                                                    ($drive->{mbps_rd_max} || 0)*1024*1024,
5156                                                    ($drive->{mbps_wr_max} || 0)*1024*1024,
5157                                                    $drive->{iops_max} || 0,
5158                                                    $drive->{iops_rd_max} || 0,
5159                                                    $drive->{iops_wr_max} || 0,
5160                                                    $drive->{bps_max_length} || 1,
5161                                                    $drive->{bps_rd_max_length} || 1,
5162                                                    $drive->{bps_wr_max_length} || 1,
5163                                                    $drive->{iops_max_length} || 1,
5164                                                    $drive->{iops_rd_max_length} || 1,
5165                                                    $drive->{iops_wr_max_length} || 1);
5166
5167                     }
5168
5169                     return 1;
5170                 }
5171
5172             } else { # cdrom
5173
5174                 if ($drive->{file} eq 'none') {
5175                     mon_cmd($vmid, "eject",force => JSON::true,device => "drive-$opt");
5176                     if (drive_is_cloudinit($old_drive)) {
5177                         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive);
5178                     }
5179                 } else {
5180                     my $path = get_iso_path($storecfg, $vmid, $drive->{file});
5181                     mon_cmd($vmid, "eject", force => JSON::true,device => "drive-$opt"); # force eject if locked
5182                     mon_cmd($vmid, "change", device => "drive-$opt",target => "$path") if $path;
5183                 }
5184
5185                 return 1;
5186             }
5187         }
5188     }
5189
5190     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
5191     # hotplug new disks
5192     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
5193     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive, $arch, $machine_type);
5194 }
5195
5196 sub vm_start {
5197     my ($storecfg, $vmid, $statefile, $skiplock, $migratedfrom, $paused,
5198         $forcemachine, $spice_ticket, $migration_network, $migration_type, $targetstorage) = @_;
5199
5200     PVE::QemuConfig->lock_config($vmid, sub {
5201         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
5202
5203         die "you can't start a vm if it's a template\n" if PVE::QemuConfig->is_template($conf);
5204
5205         my $is_suspended = PVE::QemuConfig->has_lock($conf, 'suspended');
5206
5207         PVE::QemuConfig->check_lock($conf)
5208             if !($skiplock || $is_suspended);
5209
5210         die "VM $vmid already running\n" if check_running($vmid, undef, $migratedfrom);
5211
5212         # clean up leftover reboot request files
5213         eval { clear_reboot_request($vmid); };
5214         warn $@ if $@;
5215
5216         if (!$statefile && scalar(keys %{$conf->{pending}})) {
5217             vmconfig_apply_pending($vmid, $conf, $storecfg);
5218             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5219         }
5220
5221         PVE::QemuServer::Cloudinit::generate_cloudinitconfig($conf, $vmid);
5222
5223         my $defaults = load_defaults();
5224
5225         # set environment variable useful inside network script
5226         $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
5227
5228         my $local_volumes = {};
5229
5230         if ($targetstorage) {
5231             foreach_drive($conf, sub {
5232                 my ($ds, $drive) = @_;
5233
5234                 return if drive_is_cdrom($drive);
5235
5236                 my $volid = $drive->{file};
5237
5238                 return if !$volid;
5239
5240                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
5241
5242                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5243                 return if $scfg->{shared};
5244                 $local_volumes->{$ds} = [$volid, $storeid, $volname];
5245             });
5246
5247             my $format = undef;
5248
5249             foreach my $opt (sort keys %$local_volumes) {
5250
5251                 my ($volid, $storeid, $volname) = @{$local_volumes->{$opt}};
5252                 my $drive = parse_drive($opt, $conf->{$opt});
5253
5254                 #if remote storage is specified, use default format
5255                 if ($targetstorage && $targetstorage ne "1") {
5256                     $storeid = $targetstorage;
5257                     my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
5258                     $format = $defFormat;
5259                 } else {
5260                     #else we use same format than original
5261                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5262                     $format = qemu_img_format($scfg, $volid);
5263                 }
5264
5265                 my $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $format, undef, ($drive->{size}/1024));
5266                 my $newdrive = $drive;
5267                 $newdrive->{format} = $format;
5268                 $newdrive->{file} = $newvolid;
5269                 my $drivestr = PVE::QemuServer::print_drive($vmid, $newdrive);
5270                 $local_volumes->{$opt} = $drivestr;
5271                 #pass drive to conf for command line
5272                 $conf->{$opt} = $drivestr;
5273             }
5274         }
5275
5276         PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-start', 1);
5277
5278         if ($is_suspended) {
5279             # enforce machine type on suspended vm to ensure HW compatibility
5280             $forcemachine = $conf->{runningmachine};
5281             print "Resuming suspended VM\n";
5282         }
5283
5284         my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine);
5285
5286         my $migration_ip;
5287         my $get_migration_ip = sub {
5288             my ($cidr, $nodename) = @_;
5289
5290             return $migration_ip if defined($migration_ip);
5291
5292             if (!defined($cidr)) {
5293                 my $dc_conf = PVE::Cluster::cfs_read_file('datacenter.cfg');
5294                 $cidr = $dc_conf->{migration}->{network};
5295             }
5296
5297             if (defined($cidr)) {
5298                 my $ips = PVE::Network::get_local_ip_from_cidr($cidr);
5299
5300                 die "could not get IP: no address configured on local " .
5301                     "node for network '$cidr'\n" if scalar(@$ips) == 0;
5302
5303                 die "could not get IP: multiple addresses configured on local " .
5304                     "node for network '$cidr'\n" if scalar(@$ips) > 1;
5305
5306                 $migration_ip = @$ips[0];
5307             }
5308
5309             $migration_ip = PVE::Cluster::remote_node_ip($nodename, 1)
5310                 if !defined($migration_ip);
5311
5312             return $migration_ip;
5313         };
5314
5315         my $migrate_uri;
5316         if ($statefile) {
5317             if ($statefile eq 'tcp') {
5318                 my $localip = "localhost";
5319                 my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
5320                 my $nodename = PVE::INotify::nodename();
5321
5322                 if (!defined($migration_type)) {
5323                     if (defined($datacenterconf->{migration}->{type})) {
5324                         $migration_type = $datacenterconf->{migration}->{type};
5325                     } else {
5326                         $migration_type = 'secure';
5327                     }
5328                 }
5329
5330                 if ($migration_type eq 'insecure') {
5331                     $localip = $get_migration_ip->($migration_network, $nodename);
5332                     $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5333                 }
5334
5335                 my $pfamily = PVE::Tools::get_host_address_family($nodename);
5336                 my $migrate_port = PVE::Tools::next_migrate_port($pfamily);
5337                 $migrate_uri = "tcp:${localip}:${migrate_port}";
5338                 push @$cmd, '-incoming', $migrate_uri;
5339                 push @$cmd, '-S';
5340
5341             } elsif ($statefile eq 'unix') {
5342                 # should be default for secure migrations as a ssh TCP forward
5343                 # tunnel is not deterministic reliable ready and fails regurarly
5344                 # to set up in time, so use UNIX socket forwards
5345                 my $socket_addr = "/run/qemu-server/$vmid.migrate";
5346                 unlink $socket_addr;
5347
5348                 $migrate_uri = "unix:$socket_addr";
5349
5350                 push @$cmd, '-incoming', $migrate_uri;
5351                 push @$cmd, '-S';
5352
5353             } elsif (-e $statefile) {
5354                 push @$cmd, '-loadstate', $statefile;
5355             } else {
5356                 my $statepath = PVE::Storage::path($storecfg, $statefile);
5357                 push @$vollist, $statefile;
5358                 push @$cmd, '-loadstate', $statepath;
5359             }
5360         } elsif ($paused) {
5361             push @$cmd, '-S';
5362         }
5363
5364         # host pci devices
5365         for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
5366           my $d = parse_hostpci($conf->{"hostpci$i"});
5367           next if !$d;
5368           my $pcidevices = $d->{pciid};
5369           foreach my $pcidevice (@$pcidevices) {
5370                 my $pciid = $pcidevice->{id};
5371                 $pciid = "0000:$pciid" if $pciid !~ m/^[0-9a-f]{4}:/;
5372
5373                 my $info = PVE::SysFSTools::pci_device_info("$pciid");
5374                 die "IOMMU not present\n" if !PVE::SysFSTools::check_iommu_support();
5375                 die "no pci device info for device '$pciid'\n" if !$info;
5376
5377                 if ($d->{mdev}) {
5378                     my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $i);
5379                     PVE::SysFSTools::pci_create_mdev_device($pciid, $uuid, $d->{mdev});
5380                 } else {
5381                     die "can't unbind/bind pci group to vfio '$pciid'\n"
5382                         if !PVE::SysFSTools::pci_dev_group_bind_to_vfio($pciid);
5383                     die "can't reset pci device '$pciid'\n"
5384                         if $info->{has_fl_reset} and !PVE::SysFSTools::pci_dev_reset($info);
5385                 }
5386           }
5387         }
5388
5389         PVE::Storage::activate_volumes($storecfg, $vollist);
5390
5391         eval {
5392             run_command(['/bin/systemctl', 'stop', "$vmid.scope"],
5393                 outfunc => sub {}, errfunc => sub {});
5394         };
5395         # Issues with the above 'stop' not being fully completed are extremely rare, a very low
5396         # timeout should be more than enough here...
5397         PVE::Systemd::wait_for_unit_removed("$vmid.scope", 5);
5398
5399         my $cpuunits = defined($conf->{cpuunits}) ? $conf->{cpuunits}
5400                                                   : $defaults->{cpuunits};
5401
5402         my $start_timeout = ($conf->{hugepages} || $is_suspended) ? 300 : 30;
5403         my %run_params = (timeout => $statefile ? undef : $start_timeout, umask => 0077);
5404
5405         my %properties = (
5406             Slice => 'qemu.slice',
5407             KillMode => 'none',
5408             CPUShares => $cpuunits
5409         );
5410
5411         if (my $cpulimit = $conf->{cpulimit}) {
5412             $properties{CPUQuota} = int($cpulimit * 100);
5413         }
5414         $properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
5415
5416         my $run_qemu = sub {
5417             PVE::Tools::run_fork sub {
5418                 PVE::Systemd::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
5419                 run_command($cmd, %run_params);
5420             };
5421         };
5422
5423         if ($conf->{hugepages}) {
5424
5425             my $code = sub {
5426                 my $hugepages_topology = PVE::QemuServer::Memory::hugepages_topology($conf);
5427                 my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
5428
5429                 PVE::QemuServer::Memory::hugepages_mount();
5430                 PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
5431
5432                 eval { $run_qemu->() };
5433                 if (my $err = $@) {
5434                     PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology);
5435                     die $err;
5436                 }
5437
5438                 PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology);
5439             };
5440             eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
5441
5442         } else {
5443             eval { $run_qemu->() };
5444         }
5445
5446         if (my $err = $@) {
5447             # deactivate volumes if start fails
5448             eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
5449             die "start failed: $err";
5450         }
5451
5452         print "migration listens on $migrate_uri\n" if $migrate_uri;
5453
5454         if ($statefile && $statefile ne 'tcp' && $statefile ne 'unix')  {
5455             eval { mon_cmd($vmid, "cont"); };
5456             warn $@ if $@;
5457         }
5458
5459         #start nbd server for storage migration
5460         if ($targetstorage) {
5461             my $nodename = PVE::INotify::nodename();
5462             my $localip = $get_migration_ip->($migration_network, $nodename);
5463             my $pfamily = PVE::Tools::get_host_address_family($nodename);
5464             my $storage_migrate_port = PVE::Tools::next_migrate_port($pfamily);
5465
5466             mon_cmd($vmid, "nbd-server-start", addr => { type => 'inet', data => { host => "${localip}", port => "${storage_migrate_port}" } } );
5467
5468             $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5469
5470             foreach my $opt (sort keys %$local_volumes) {
5471                 my $volid = $local_volumes->{$opt};
5472                 mon_cmd($vmid, "nbd-server-add", device => "drive-$opt", writable => JSON::true );
5473                 my $migrate_storage_uri = "nbd:${localip}:${storage_migrate_port}:exportname=drive-$opt";
5474                 print "storage migration listens on $migrate_storage_uri volume:$volid\n";
5475             }
5476         }
5477
5478         if ($migratedfrom) {
5479             eval {
5480                 set_migration_caps($vmid);
5481             };
5482             warn $@ if $@;
5483
5484             if ($spice_port) {
5485                 print "spice listens on port $spice_port\n";
5486                 if ($spice_ticket) {
5487                     mon_cmd($vmid, "set_password", protocol => 'spice', password => $spice_ticket);
5488                     mon_cmd($vmid, "expire_password", protocol => 'spice', time => "+30");
5489                 }
5490             }
5491
5492         } else {
5493             mon_cmd($vmid, "balloon", value => $conf->{balloon}*1024*1024)
5494                 if !$statefile && $conf->{balloon};
5495
5496             foreach my $opt (keys %$conf) {
5497                 next if $opt !~  m/^net\d+$/;
5498                 my $nicconf = parse_net($conf->{$opt});
5499                 qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
5500             }
5501         }
5502
5503         mon_cmd($vmid, 'qom-set',
5504                     path => "machine/peripheral/balloon0",
5505                     property => "guest-stats-polling-interval",
5506                     value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
5507
5508         if ($is_suspended && (my $vmstate = $conf->{vmstate})) {
5509             print "Resumed VM, removing state\n";
5510             delete $conf->@{qw(lock vmstate runningmachine)};
5511             PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
5512             PVE::Storage::vdisk_free($storecfg, $vmstate);
5513             PVE::QemuConfig->write_config($vmid, $conf);
5514         }
5515
5516         PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'post-start');
5517     });
5518 }
5519
5520 sub vm_commandline {
5521     my ($storecfg, $vmid, $snapname) = @_;
5522
5523     my $conf = PVE::QemuConfig->load_config($vmid);
5524     my $forcemachine;
5525
5526     if ($snapname) {
5527         my $snapshot = $conf->{snapshots}->{$snapname};
5528         die "snapshot '$snapname' does not exist\n" if !defined($snapshot);
5529
5530         # check for a 'runningmachine' in snapshot
5531         $forcemachine = $snapshot->{runningmachine} if $snapshot->{runningmachine};
5532
5533         $snapshot->{digest} = $conf->{digest}; # keep file digest for API
5534
5535         $conf = $snapshot;
5536     }
5537
5538     my $defaults = load_defaults();
5539
5540     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine);
5541
5542     return PVE::Tools::cmd2string($cmd);
5543 }
5544
5545 sub vm_reset {
5546     my ($vmid, $skiplock) = @_;
5547
5548     PVE::QemuConfig->lock_config($vmid, sub {
5549
5550         my $conf = PVE::QemuConfig->load_config($vmid);
5551
5552         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5553
5554         mon_cmd($vmid, "system_reset");
5555     });
5556 }
5557
5558 sub get_vm_volumes {
5559     my ($conf) = @_;
5560
5561     my $vollist = [];
5562     foreach_volid($conf, sub {
5563         my ($volid, $attr) = @_;
5564
5565         return if $volid =~ m|^/|;
5566
5567         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5568         return if !$sid;
5569
5570         push @$vollist, $volid;
5571     });
5572
5573     return $vollist;
5574 }
5575
5576 sub vm_stop_cleanup {
5577     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
5578
5579     eval {
5580
5581         if (!$keepActive) {
5582             my $vollist = get_vm_volumes($conf);
5583             PVE::Storage::deactivate_volumes($storecfg, $vollist);
5584         }
5585
5586         foreach my $ext (qw(mon qmp pid vnc qga)) {
5587             unlink "/var/run/qemu-server/${vmid}.$ext";
5588         }
5589
5590         if ($conf->{ivshmem}) {
5591             my $ivshmem = PVE::JSONSchema::parse_property_string($ivshmem_fmt, $conf->{ivshmem});
5592             # just delete it for now, VMs which have this already open do not
5593             # are affected, but new VMs will get a separated one. If this
5594             # becomes an issue we either add some sort of ref-counting or just
5595             # add a "don't delete on stop" flag to the ivshmem format.
5596             unlink '/dev/shm/pve-shm-' . ($ivshmem->{name} // $vmid);
5597         }
5598
5599         foreach my $key (keys %$conf) {
5600             next if $key !~ m/^hostpci(\d+)$/;
5601             my $hostpciindex = $1;
5602             my $d = parse_hostpci($conf->{$key});
5603             my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $hostpciindex);
5604
5605             foreach my $pci (@{$d->{pciid}}) {
5606                 my $pciid = $pci->{id};
5607                 PVE::SysFSTools::pci_cleanup_mdev_device($pciid, $uuid);
5608             }
5609         }
5610
5611         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
5612     };
5613     warn $@ if $@; # avoid errors - just warn
5614 }
5615
5616 # call only in locked context
5617 sub _do_vm_stop {
5618     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive) = @_;
5619
5620     my $pid = check_running($vmid, $nocheck);
5621     return if !$pid;
5622
5623     my $conf;
5624     if (!$nocheck) {
5625         $conf = PVE::QemuConfig->load_config($vmid);
5626         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5627         if (!defined($timeout) && $shutdown && $conf->{startup}) {
5628             my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
5629             $timeout = $opts->{down} if $opts->{down};
5630         }
5631         PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-stop');
5632     }
5633
5634     eval {
5635         if ($shutdown) {
5636             if (defined($conf) && parse_guest_agent($conf)->{enabled}) {
5637                 mon_cmd($vmid, "guest-shutdown", timeout => $timeout);
5638             } else {
5639                 mon_cmd($vmid, "system_powerdown");
5640             }
5641         } else {
5642             mon_cmd($vmid, "quit");
5643         }
5644     };
5645     my $err = $@;
5646
5647     if (!$err) {
5648         $timeout = 60 if !defined($timeout);
5649
5650         my $count = 0;
5651         while (($count < $timeout) && check_running($vmid, $nocheck)) {
5652             $count++;
5653             sleep 1;
5654         }
5655
5656         if ($count >= $timeout) {
5657             if ($force) {
5658                 warn "VM still running - terminating now with SIGTERM\n";
5659                 kill 15, $pid;
5660             } else {
5661                 die "VM quit/powerdown failed - got timeout\n";
5662             }
5663         } else {
5664             vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5665             return;
5666         }
5667     } else {
5668         if ($force) {
5669             warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
5670             kill 15, $pid;
5671         } else {
5672             die "VM quit/powerdown failed\n";
5673         }
5674     }
5675
5676     # wait again
5677     $timeout = 10;
5678
5679     my $count = 0;
5680     while (($count < $timeout) && check_running($vmid, $nocheck)) {
5681         $count++;
5682         sleep 1;
5683     }
5684
5685     if ($count >= $timeout) {
5686         warn "VM still running - terminating now with SIGKILL\n";
5687         kill 9, $pid;
5688         sleep 1;
5689     }
5690
5691     vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5692 }
5693
5694 # Note: use $nocheck to skip tests if VM configuration file exists.
5695 # We need that when migration VMs to other nodes (files already moved)
5696 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
5697 sub vm_stop {
5698     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
5699
5700     $force = 1 if !defined($force) && !$shutdown;
5701
5702     if ($migratedfrom){
5703         my $pid = check_running($vmid, $nocheck, $migratedfrom);
5704         kill 15, $pid if $pid;
5705         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
5706         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
5707         return;
5708     }
5709
5710     PVE::QemuConfig->lock_config($vmid, sub {
5711         _do_vm_stop($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive);
5712    });
5713 }
5714
5715 sub vm_reboot {
5716     my ($vmid, $timeout) = @_;
5717
5718     PVE::QemuConfig->lock_config($vmid, sub {
5719         eval {
5720
5721             # only reboot if running, as qmeventd starts it again on a stop event
5722             return if !check_running($vmid);
5723
5724             create_reboot_request($vmid);
5725
5726             my $storecfg = PVE::Storage::config();
5727             _do_vm_stop($storecfg, $vmid, undef, undef, $timeout, 1);
5728
5729         };
5730         if (my $err = $@) {
5731             # avoid that the next normal shutdown will be confused for a reboot
5732             clear_reboot_request($vmid);
5733             die $err;
5734         }
5735    });
5736 }
5737
5738 sub vm_suspend {
5739     my ($vmid, $skiplock, $includestate, $statestorage) = @_;
5740
5741     my $conf;
5742     my $path;
5743     my $storecfg;
5744     my $vmstate;
5745
5746     PVE::QemuConfig->lock_config($vmid, sub {
5747
5748         $conf = PVE::QemuConfig->load_config($vmid);
5749
5750         my $is_backing_up = PVE::QemuConfig->has_lock($conf, 'backup');
5751         PVE::QemuConfig->check_lock($conf)
5752             if !($skiplock || $is_backing_up);
5753
5754         die "cannot suspend to disk during backup\n"
5755             if $is_backing_up && $includestate;
5756
5757         if ($includestate) {
5758             $conf->{lock} = 'suspending';
5759             my $date = strftime("%Y-%m-%d", localtime(time()));
5760             $storecfg = PVE::Storage::config();
5761             $vmstate = PVE::QemuConfig->__snapshot_save_vmstate($vmid, $conf, "suspend-$date", $storecfg, $statestorage, 1);
5762             $path = PVE::Storage::path($storecfg, $vmstate);
5763             PVE::QemuConfig->write_config($vmid, $conf);
5764         } else {
5765             mon_cmd($vmid, "stop");
5766         }
5767     });
5768
5769     if ($includestate) {
5770         # save vm state
5771         PVE::Storage::activate_volumes($storecfg, [$vmstate]);
5772
5773         eval {
5774             mon_cmd($vmid, "savevm-start", statefile => $path);
5775             for(;;) {
5776                 my $state = mon_cmd($vmid, "query-savevm");
5777                 if (!$state->{status}) {
5778                     die "savevm not active\n";
5779                 } elsif ($state->{status} eq 'active') {
5780                     sleep(1);
5781                     next;
5782                 } elsif ($state->{status} eq 'completed') {
5783                     print "State saved, quitting\n";
5784                     last;
5785                 } elsif ($state->{status} eq 'failed' && $state->{error}) {
5786                     die "query-savevm failed with error '$state->{error}'\n"
5787                 } else {
5788                     die "query-savevm returned status '$state->{status}'\n";
5789                 }
5790             }
5791         };
5792         my $err = $@;
5793
5794         PVE::QemuConfig->lock_config($vmid, sub {
5795             $conf = PVE::QemuConfig->load_config($vmid);
5796             if ($err) {
5797                 # cleanup, but leave suspending lock, to indicate something went wrong
5798                 eval {
5799                     mon_cmd($vmid, "savevm-end");
5800                     PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
5801                     PVE::Storage::vdisk_free($storecfg, $vmstate);
5802                     delete $conf->@{qw(vmstate runningmachine)};
5803                     PVE::QemuConfig->write_config($vmid, $conf);
5804                 };
5805                 warn $@ if $@;
5806                 die $err;
5807             }
5808
5809             die "lock changed unexpectedly\n"
5810                 if !PVE::QemuConfig->has_lock($conf, 'suspending');
5811
5812             mon_cmd($vmid, "quit");
5813             $conf->{lock} = 'suspended';
5814             PVE::QemuConfig->write_config($vmid, $conf);
5815         });
5816     }
5817 }
5818
5819 sub vm_resume {
5820     my ($vmid, $skiplock, $nocheck) = @_;
5821
5822     PVE::QemuConfig->lock_config($vmid, sub {
5823         my $res = mon_cmd($vmid, 'query-status');
5824         my $resume_cmd = 'cont';
5825
5826         if ($res->{status} && $res->{status} eq 'suspended') {
5827             $resume_cmd = 'system_wakeup';
5828         }
5829
5830         if (!$nocheck) {
5831
5832             my $conf = PVE::QemuConfig->load_config($vmid);
5833
5834             PVE::QemuConfig->check_lock($conf)
5835                 if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
5836         }
5837
5838         mon_cmd($vmid, $resume_cmd);
5839     });
5840 }
5841
5842 sub vm_sendkey {
5843     my ($vmid, $skiplock, $key) = @_;
5844
5845     PVE::QemuConfig->lock_config($vmid, sub {
5846
5847         my $conf = PVE::QemuConfig->load_config($vmid);
5848
5849         # there is no qmp command, so we use the human monitor command
5850         my $res = PVE::QemuServer::Monitor::hmp_cmd($vmid, "sendkey $key");
5851         die $res if $res ne '';
5852     });
5853 }
5854
5855 # vzdump restore implementaion
5856
5857 sub tar_archive_read_firstfile {
5858     my $archive = shift;
5859
5860     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
5861
5862     # try to detect archive type first
5863     my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
5864         die "unable to open file '$archive'\n";
5865     my $firstfile = <$fh>;
5866     kill 15, $pid;
5867     close $fh;
5868
5869     die "ERROR: archive contaions no data\n" if !$firstfile;
5870     chomp $firstfile;
5871
5872     return $firstfile;
5873 }
5874
5875 sub tar_restore_cleanup {
5876     my ($storecfg, $statfile) = @_;
5877
5878     print STDERR "starting cleanup\n";
5879
5880     if (my $fd = IO::File->new($statfile, "r")) {
5881         while (defined(my $line = <$fd>)) {
5882             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5883                 my $volid = $2;
5884                 eval {
5885                     if ($volid =~ m|^/|) {
5886                         unlink $volid || die 'unlink failed\n';
5887                     } else {
5888                         PVE::Storage::vdisk_free($storecfg, $volid);
5889                     }
5890                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
5891                 };
5892                 print STDERR "unable to cleanup '$volid' - $@" if $@;
5893             } else {
5894                 print STDERR "unable to parse line in statfile - $line";
5895             }
5896         }
5897         $fd->close();
5898     }
5899 }
5900
5901 sub restore_archive {
5902     my ($archive, $vmid, $user, $opts) = @_;
5903
5904     my $format = $opts->{format};
5905     my $comp;
5906
5907     if ($archive =~ m/\.tgz$/ || $archive =~ m/\.tar\.gz$/) {
5908         $format = 'tar' if !$format;
5909         $comp = 'gzip';
5910     } elsif ($archive =~ m/\.tar$/) {
5911         $format = 'tar' if !$format;
5912     } elsif ($archive =~ m/.tar.lzo$/) {
5913         $format = 'tar' if !$format;
5914         $comp = 'lzop';
5915     } elsif ($archive =~ m/\.vma$/) {
5916         $format = 'vma' if !$format;
5917     } elsif ($archive =~ m/\.vma\.gz$/) {
5918         $format = 'vma' if !$format;
5919         $comp = 'gzip';
5920     } elsif ($archive =~ m/\.vma\.lzo$/) {
5921         $format = 'vma' if !$format;
5922         $comp = 'lzop';
5923     } else {
5924         $format = 'vma' if !$format; # default
5925     }
5926
5927     # try to detect archive format
5928     if ($format eq 'tar') {
5929         return restore_tar_archive($archive, $vmid, $user, $opts);
5930     } else {
5931         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
5932     }
5933 }
5934
5935 sub restore_update_config_line {
5936     my ($outfd, $cookie, $vmid, $map, $line, $unique) = @_;
5937
5938     return if $line =~ m/^\#qmdump\#/;
5939     return if $line =~ m/^\#vzdump\#/;
5940     return if $line =~ m/^lock:/;
5941     return if $line =~ m/^unused\d+:/;
5942     return if $line =~ m/^parent:/;
5943
5944     my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
5945     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
5946         # try to convert old 1.X settings
5947         my ($id, $ind, $ethcfg) = ($1, $2, $3);
5948         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
5949             my ($model, $macaddr) = split(/\=/, $devconfig);
5950             $macaddr = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if !$macaddr || $unique;
5951             my $net = {
5952                 model => $model,
5953                 bridge => "vmbr$ind",
5954                 macaddr => $macaddr,
5955             };
5956             my $netstr = print_net($net);
5957
5958             print $outfd "net$cookie->{netcount}: $netstr\n";
5959             $cookie->{netcount}++;
5960         }
5961     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
5962         my ($id, $netstr) = ($1, $2);
5963         my $net = parse_net($netstr);
5964         $net->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if $net->{macaddr};
5965         $netstr = print_net($net);
5966         print $outfd "$id: $netstr\n";
5967     } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk)\d+):\s*(\S+)\s*$/) {
5968         my $virtdev = $1;
5969         my $value = $3;
5970         my $di = parse_drive($virtdev, $value);
5971         if (defined($di->{backup}) && !$di->{backup}) {
5972             print $outfd "#$line";
5973         } elsif ($map->{$virtdev}) {
5974             delete $di->{format}; # format can change on restore
5975             $di->{file} = $map->{$virtdev};
5976             $value = print_drive($vmid, $di);
5977             print $outfd "$virtdev: $value\n";
5978         } else {
5979             print $outfd $line;
5980         }
5981     } elsif (($line =~ m/^vmgenid: (.*)/)) {
5982         my $vmgenid = $1;
5983         if ($vmgenid ne '0') {
5984             # always generate a new vmgenid if there was a valid one setup
5985             $vmgenid = generate_uuid();
5986         }
5987         print $outfd "vmgenid: $vmgenid\n";
5988     } elsif (($line =~ m/^(smbios1: )(.*)/) && $unique) {
5989         my ($uuid, $uuid_str);
5990         UUID::generate($uuid);
5991         UUID::unparse($uuid, $uuid_str);
5992         my $smbios1 = parse_smbios1($2);
5993         $smbios1->{uuid} = $uuid_str;
5994         print $outfd $1.print_smbios1($smbios1)."\n";
5995     } else {
5996         print $outfd $line;
5997     }
5998 }
5999
6000 sub scan_volids {
6001     my ($cfg, $vmid) = @_;
6002
6003     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid);
6004
6005     my $volid_hash = {};
6006     foreach my $storeid (keys %$info) {
6007         foreach my $item (@{$info->{$storeid}}) {
6008             next if !($item->{volid} && $item->{size});
6009             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
6010             $volid_hash->{$item->{volid}} = $item;
6011         }
6012     }
6013
6014     return $volid_hash;
6015 }
6016
6017 sub is_volume_in_use {
6018     my ($storecfg, $conf, $skip_drive, $volid) = @_;
6019
6020     my $path = PVE::Storage::path($storecfg, $volid);
6021
6022     my $scan_config = sub {
6023         my ($cref, $snapname) = @_;
6024
6025         foreach my $key (keys %$cref) {
6026             my $value = $cref->{$key};
6027             if (is_valid_drivename($key)) {
6028                 next if $skip_drive && $key eq $skip_drive;
6029                 my $drive = parse_drive($key, $value);
6030                 next if !$drive || !$drive->{file} || drive_is_cdrom($drive);
6031                 return 1 if $volid eq $drive->{file};
6032                 if ($drive->{file} =~ m!^/!) {
6033                     return 1 if $drive->{file} eq $path;
6034                 } else {
6035                     my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file}, 1);
6036                     next if !$storeid;
6037                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid, 1);
6038                     next if !$scfg;
6039                     return 1 if $path eq PVE::Storage::path($storecfg, $drive->{file}, $snapname);
6040                 }
6041             }
6042         }
6043
6044         return 0;
6045     };
6046
6047     return 1 if &$scan_config($conf);
6048
6049     undef $skip_drive;
6050
6051     foreach my $snapname (keys %{$conf->{snapshots}}) {
6052         return 1 if &$scan_config($conf->{snapshots}->{$snapname}, $snapname);
6053     }
6054
6055     return 0;
6056 }
6057
6058 sub update_disksize {
6059     my ($vmid, $conf, $volid_hash) = @_;
6060
6061     my $changes;
6062     my $prefix = "VM $vmid:";
6063
6064     # used and unused disks
6065     my $referenced = {};
6066
6067     # Note: it is allowed to define multiple storages with same path (alias), so
6068     # we need to check both 'volid' and real 'path' (two different volid can point
6069     # to the same path).
6070
6071     my $referencedpath = {};
6072
6073     # update size info
6074     foreach my $opt (keys %$conf) {
6075         if (is_valid_drivename($opt)) {
6076             my $drive = parse_drive($opt, $conf->{$opt});
6077             my $volid = $drive->{file};
6078             next if !$volid;
6079
6080             $referenced->{$volid} = 1;
6081             if ($volid_hash->{$volid} &&
6082                 (my $path = $volid_hash->{$volid}->{path})) {
6083                 $referencedpath->{$path} = 1;
6084             }
6085
6086             next if drive_is_cdrom($drive);
6087             next if !$volid_hash->{$volid};
6088
6089             $drive->{size} = $volid_hash->{$volid}->{size};
6090             my $new = print_drive($vmid, $drive);
6091             if ($new ne $conf->{$opt}) {
6092                 $changes = 1;
6093                 $conf->{$opt} = $new;
6094                 print "$prefix update disk '$opt' information.\n";
6095             }
6096         }
6097     }
6098
6099     # remove 'unusedX' entry if volume is used
6100     foreach my $opt (keys %$conf) {
6101         next if $opt !~ m/^unused\d+$/;
6102         my $volid = $conf->{$opt};
6103         my $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
6104         if ($referenced->{$volid} || ($path && $referencedpath->{$path})) {
6105             print "$prefix remove entry '$opt', its volume '$volid' is in use.\n";
6106             $changes = 1;
6107             delete $conf->{$opt};
6108         }
6109
6110         $referenced->{$volid} = 1;
6111         $referencedpath->{$path} = 1 if $path;
6112     }
6113
6114     foreach my $volid (sort keys %$volid_hash) {
6115         next if $volid =~ m/vm-$vmid-state-/;
6116         next if $referenced->{$volid};
6117         my $path = $volid_hash->{$volid}->{path};
6118         next if !$path; # just to be sure
6119         next if $referencedpath->{$path};
6120         $changes = 1;
6121         my $key = PVE::QemuConfig->add_unused_volume($conf, $volid);
6122         print "$prefix add unreferenced volume '$volid' as '$key' to config.\n";
6123         $referencedpath->{$path} = 1; # avoid to add more than once (aliases)
6124     }
6125
6126     return $changes;
6127 }
6128
6129 sub rescan {
6130     my ($vmid, $nolock, $dryrun) = @_;
6131
6132     my $cfg = PVE::Storage::config();
6133
6134     # FIXME: Remove once our RBD plugin can handle CT and VM on a single storage
6135     # see: https://pve.proxmox.com/pipermail/pve-devel/2018-July/032900.html
6136     foreach my $stor (keys %{$cfg->{ids}}) {
6137         delete($cfg->{ids}->{$stor}) if ! $cfg->{ids}->{$stor}->{content}->{images};
6138     }
6139
6140     print "rescan volumes...\n";
6141     my $volid_hash = scan_volids($cfg, $vmid);
6142
6143     my $updatefn =  sub {
6144         my ($vmid) = @_;
6145
6146         my $conf = PVE::QemuConfig->load_config($vmid);
6147
6148         PVE::QemuConfig->check_lock($conf);
6149
6150         my $vm_volids = {};
6151         foreach my $volid (keys %$volid_hash) {
6152             my $info = $volid_hash->{$volid};
6153             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
6154         }
6155
6156         my $changes = update_disksize($vmid, $conf, $vm_volids);
6157
6158         PVE::QemuConfig->write_config($vmid, $conf) if $changes && !$dryrun;
6159     };
6160
6161     if (defined($vmid)) {
6162         if ($nolock) {
6163             &$updatefn($vmid);
6164         } else {
6165             PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6166         }
6167     } else {
6168         my $vmlist = config_list();
6169         foreach my $vmid (keys %$vmlist) {
6170             if ($nolock) {
6171                 &$updatefn($vmid);
6172             } else {
6173                 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6174             }
6175         }
6176     }
6177 }
6178
6179 sub restore_vma_archive {
6180     my ($archive, $vmid, $user, $opts, $comp) = @_;
6181
6182     my $readfrom = $archive;
6183
6184     my $cfg = PVE::Storage::config();
6185     my $commands = [];
6186     my $bwlimit = $opts->{bwlimit};
6187
6188     my $dbg_cmdstring = '';
6189     my $add_pipe = sub {
6190         my ($cmd) = @_;
6191         push @$commands, $cmd;
6192         $dbg_cmdstring .= ' | ' if length($dbg_cmdstring);
6193         $dbg_cmdstring .= PVE::Tools::cmd2string($cmd);
6194         $readfrom = '-';
6195     };
6196
6197     my $input = undef;
6198     if ($archive eq '-') {
6199         $input = '<&STDIN';
6200     } else {
6201         # If we use a backup from a PVE defined storage we also consider that
6202         # storage's rate limit:
6203         my (undef, $volid) = PVE::Storage::path_to_volume_id($cfg, $archive);
6204         if (defined($volid)) {
6205             my ($sid, undef) = PVE::Storage::parse_volume_id($volid);
6206             my $readlimit = PVE::Storage::get_bandwidth_limit('restore', [$sid], $bwlimit);
6207             if ($readlimit) {
6208                 print STDERR "applying read rate limit: $readlimit\n";
6209                 my $cstream = ['cstream', '-t', $readlimit*1024, '--', $readfrom];
6210                 $add_pipe->($cstream);
6211             }
6212         }
6213     }
6214
6215     if ($comp) {
6216         my $cmd;
6217         if ($comp eq 'gzip') {
6218             $cmd = ['zcat', $readfrom];
6219         } elsif ($comp eq 'lzop') {
6220             $cmd = ['lzop', '-d', '-c', $readfrom];
6221         } else {
6222             die "unknown compression method '$comp'\n";
6223         }
6224         $add_pipe->($cmd);
6225     }
6226
6227     my $tmpdir = "/var/tmp/vzdumptmp$$";
6228     rmtree $tmpdir;
6229
6230     # disable interrupts (always do cleanups)
6231     local $SIG{INT} =
6232         local $SIG{TERM} =
6233         local $SIG{QUIT} =
6234         local $SIG{HUP} = sub { warn "got interrupt - ignored\n"; };
6235
6236     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
6237     POSIX::mkfifo($mapfifo, 0600);
6238     my $fifofh;
6239
6240     my $openfifo = sub {
6241         open($fifofh, '>', $mapfifo) || die $!;
6242     };
6243
6244     $add_pipe->(['vma', 'extract', '-v', '-r', $mapfifo, $readfrom, $tmpdir]);
6245
6246     my $oldtimeout;
6247     my $timeout = 5;
6248
6249     my $devinfo = {};
6250
6251     my $rpcenv = PVE::RPCEnvironment::get();
6252
6253     my $conffile = PVE::QemuConfig->config_file($vmid);
6254     my $tmpfn = "$conffile.$$.tmp";
6255
6256     # Note: $oldconf is undef if VM does not exists
6257     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
6258     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
6259
6260     my %storage_limits;
6261
6262     my $print_devmap = sub {
6263         my $virtdev_hash = {};
6264
6265         my $cfgfn = "$tmpdir/qemu-server.conf";
6266
6267         # we can read the config - that is already extracted
6268         my $fh = IO::File->new($cfgfn, "r") ||
6269             "unable to read qemu-server.conf - $!\n";
6270
6271         my $fwcfgfn = "$tmpdir/qemu-server.fw";
6272         if (-f $fwcfgfn) {
6273             my $pve_firewall_dir = '/etc/pve/firewall';
6274             mkdir $pve_firewall_dir; # make sure the dir exists
6275             PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
6276         }
6277
6278         while (defined(my $line = <$fh>)) {
6279             if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
6280                 my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
6281                 die "archive does not contain data for drive '$virtdev'\n"
6282                     if !$devinfo->{$devname};
6283                 if (defined($opts->{storage})) {
6284                     $storeid = $opts->{storage} || 'local';
6285                 } elsif (!$storeid) {
6286                     $storeid = 'local';
6287                 }
6288                 $format = 'raw' if !$format;
6289                 $devinfo->{$devname}->{devname} = $devname;
6290                 $devinfo->{$devname}->{virtdev} = $virtdev;
6291                 $devinfo->{$devname}->{format} = $format;
6292                 $devinfo->{$devname}->{storeid} = $storeid;
6293
6294                 # check permission on storage
6295                 my $pool = $opts->{pool}; # todo: do we need that?
6296                 if ($user ne 'root@pam') {
6297                     $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
6298                 }
6299
6300                 $storage_limits{$storeid} = $bwlimit;
6301
6302                 $virtdev_hash->{$virtdev} = $devinfo->{$devname};
6303             } elsif ($line =~ m/^((?:ide|sata|scsi)\d+):\s*(.*)\s*$/) {
6304                 my $virtdev = $1;
6305                 my $drive = parse_drive($virtdev, $2);
6306                 if (drive_is_cloudinit($drive)) {
6307                     my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6308                     my $scfg = PVE::Storage::storage_config($cfg, $storeid);
6309                     my $format = qemu_img_format($scfg, $volname); # has 'raw' fallback
6310
6311                     my $d = {
6312                         format => $format,
6313                         storeid => $opts->{storage} // $storeid,
6314                         size => PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE,
6315                         file => $drive->{file}, # to make drive_is_cloudinit check possible
6316                         name => "vm-$vmid-cloudinit",
6317                         is_cloudinit => 1,
6318                     };
6319                     $virtdev_hash->{$virtdev} = $d;
6320                 }
6321             }
6322         }
6323
6324         foreach my $key (keys %storage_limits) {
6325             my $limit = PVE::Storage::get_bandwidth_limit('restore', [$key], $bwlimit);
6326             next if !$limit;
6327             print STDERR "rate limit for storage $key: $limit KiB/s\n";
6328             $storage_limits{$key} = $limit * 1024;
6329         }
6330
6331         foreach my $devname (keys %$devinfo) {
6332             die "found no device mapping information for device '$devname'\n"
6333                 if !$devinfo->{$devname}->{virtdev};
6334         }
6335
6336         # create empty/temp config
6337         if ($oldconf) {
6338             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
6339             foreach_drive($oldconf, sub {
6340                 my ($ds, $drive) = @_;
6341
6342                 return if drive_is_cdrom($drive, 1);
6343
6344                 my $volid = $drive->{file};
6345                 return if !$volid || $volid =~ m|^/|;
6346
6347                 my ($path, $owner) = PVE::Storage::path($cfg, $volid);
6348                 return if !$path || !$owner || ($owner != $vmid);
6349
6350                 # Note: only delete disk we want to restore
6351                 # other volumes will become unused
6352                 if ($virtdev_hash->{$ds}) {
6353                     eval { PVE::Storage::vdisk_free($cfg, $volid); };
6354                     if (my $err = $@) {
6355                         warn $err;
6356                     }
6357                 }
6358             });
6359
6360             # delete vmstate files, after the restore we have no snapshots anymore
6361             foreach my $snapname (keys %{$oldconf->{snapshots}}) {
6362                 my $snap = $oldconf->{snapshots}->{$snapname};
6363                 if ($snap->{vmstate}) {
6364                     eval { PVE::Storage::vdisk_free($cfg, $snap->{vmstate}); };
6365                     if (my $err = $@) {
6366                         warn $err;
6367                     }
6368                 }
6369             }
6370         }
6371
6372         my $map = {};
6373         foreach my $virtdev (sort keys %$virtdev_hash) {
6374             my $d = $virtdev_hash->{$virtdev};
6375             my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
6376             my $storeid = $d->{storeid};
6377             my $scfg = PVE::Storage::storage_config($cfg, $storeid);
6378
6379             my $map_opts = '';
6380             if (my $limit = $storage_limits{$storeid}) {
6381                 $map_opts .= "throttling.bps=$limit:throttling.group=$storeid:";
6382             }
6383
6384             # test if requested format is supported
6385             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($cfg, $storeid);
6386             my $supported = grep { $_ eq $d->{format} } @$validFormats;
6387             $d->{format} = $defFormat if !$supported;
6388
6389             my $name;
6390             if ($d->{is_cloudinit}) {
6391                 $name = $d->{name};
6392                 $name .= ".$d->{format}" if $d->{format} ne 'raw';
6393             }
6394
6395             my $volid = PVE::Storage::vdisk_alloc($cfg, $storeid, $vmid, $d->{format}, $name, $alloc_size);
6396             print STDERR "new volume ID is '$volid'\n";
6397             $d->{volid} = $volid;
6398
6399             PVE::Storage::activate_volumes($cfg, [$volid]);
6400
6401             my $write_zeros = 1;
6402             if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
6403                 $write_zeros = 0;
6404             }
6405
6406             if (!$d->{is_cloudinit}) {
6407                 my $path = PVE::Storage::path($cfg, $volid);
6408
6409                 print $fifofh "${map_opts}format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
6410
6411                 print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
6412             }
6413             $map->{$virtdev} = $volid;
6414         }
6415
6416         $fh->seek(0, 0) || die "seek failed - $!\n";
6417
6418         my $outfd = new IO::File ($tmpfn, "w") ||
6419             die "unable to write config for VM $vmid\n";
6420
6421         my $cookie = { netcount => 0 };
6422         while (defined(my $line = <$fh>)) {
6423             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
6424         }
6425
6426         $fh->close();
6427         $outfd->close();
6428     };
6429
6430     eval {
6431         # enable interrupts
6432         local $SIG{INT} =
6433             local $SIG{TERM} =
6434             local $SIG{QUIT} =
6435             local $SIG{HUP} =
6436             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6437         local $SIG{ALRM} = sub { die "got timeout\n"; };
6438
6439         $oldtimeout = alarm($timeout);
6440
6441         my $parser = sub {
6442             my $line = shift;
6443
6444             print "$line\n";
6445
6446             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
6447                 my ($dev_id, $size, $devname) = ($1, $2, $3);
6448                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
6449             } elsif ($line =~ m/^CTIME: /) {
6450                 # we correctly received the vma config, so we can disable
6451                 # the timeout now for disk allocation (set to 10 minutes, so
6452                 # that we always timeout if something goes wrong)
6453                 alarm(600);
6454                 &$print_devmap();
6455                 print $fifofh "done\n";
6456                 my $tmp = $oldtimeout || 0;
6457                 $oldtimeout = undef;
6458                 alarm($tmp);
6459                 close($fifofh);
6460             }
6461         };
6462
6463         print "restore vma archive: $dbg_cmdstring\n";
6464         run_command($commands, input => $input, outfunc => $parser, afterfork => $openfifo);
6465     };
6466     my $err = $@;
6467
6468     alarm($oldtimeout) if $oldtimeout;
6469
6470     my $vollist = [];
6471     foreach my $devname (keys %$devinfo) {
6472         my $volid = $devinfo->{$devname}->{volid};
6473         push @$vollist, $volid if $volid;
6474     }
6475
6476     PVE::Storage::deactivate_volumes($cfg, $vollist);
6477
6478     unlink $mapfifo;
6479
6480     if ($err) {
6481         rmtree $tmpdir;
6482         unlink $tmpfn;
6483
6484         foreach my $devname (keys %$devinfo) {
6485             my $volid = $devinfo->{$devname}->{volid};
6486             next if !$volid;
6487             eval {
6488                 if ($volid =~ m|^/|) {
6489                     unlink $volid || die 'unlink failed\n';
6490                 } else {
6491                     PVE::Storage::vdisk_free($cfg, $volid);
6492                 }
6493                 print STDERR "temporary volume '$volid' sucessfuly removed\n";
6494             };
6495             print STDERR "unable to cleanup '$volid' - $@" if $@;
6496         }
6497         die $err;
6498     }
6499
6500     rmtree $tmpdir;
6501
6502     rename($tmpfn, $conffile) ||
6503         die "unable to commit configuration file '$conffile'\n";
6504
6505     PVE::Cluster::cfs_update(); # make sure we read new file
6506
6507     eval { rescan($vmid, 1); };
6508     warn $@ if $@;
6509 }
6510
6511 sub restore_tar_archive {
6512     my ($archive, $vmid, $user, $opts) = @_;
6513
6514     if ($archive ne '-') {
6515         my $firstfile = tar_archive_read_firstfile($archive);
6516         die "ERROR: file '$archive' dos not lock like a QemuServer vzdump backup\n"
6517             if $firstfile ne 'qemu-server.conf';
6518     }
6519
6520     my $storecfg = PVE::Storage::config();
6521
6522     # avoid zombie disks when restoring over an existing VM -> cleanup first
6523     # pass keep_empty_config=1 to keep the config (thus VMID) reserved for us
6524     # skiplock=1 because qmrestore has set the 'create' lock itself already
6525     my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
6526     destroy_vm($storecfg, $vmid, 1, { lock => 'restore' }) if -f $vmcfgfn;
6527
6528     my $tocmd = "/usr/lib/qemu-server/qmextract";
6529
6530     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
6531     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
6532     $tocmd .= ' --prealloc' if $opts->{prealloc};
6533     $tocmd .= ' --info' if $opts->{info};
6534
6535     # tar option "xf" does not autodetect compression when read from STDIN,
6536     # so we pipe to zcat
6537     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
6538         PVE::Tools::shellquote("--to-command=$tocmd");
6539
6540     my $tmpdir = "/var/tmp/vzdumptmp$$";
6541     mkpath $tmpdir;
6542
6543     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
6544     local $ENV{VZDUMP_VMID} = $vmid;
6545     local $ENV{VZDUMP_USER} = $user;
6546
6547     my $conffile = PVE::QemuConfig->config_file($vmid);
6548     my $tmpfn = "$conffile.$$.tmp";
6549
6550     # disable interrupts (always do cleanups)
6551     local $SIG{INT} =
6552         local $SIG{TERM} =
6553         local $SIG{QUIT} =
6554         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
6555
6556     eval {
6557         # enable interrupts
6558         local $SIG{INT} =
6559             local $SIG{TERM} =
6560             local $SIG{QUIT} =
6561             local $SIG{HUP} =
6562             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6563
6564         if ($archive eq '-') {
6565             print "extracting archive from STDIN\n";
6566             run_command($cmd, input => "<&STDIN");
6567         } else {
6568             print "extracting archive '$archive'\n";
6569             run_command($cmd);
6570         }
6571
6572         return if $opts->{info};
6573
6574         # read new mapping
6575         my $map = {};
6576         my $statfile = "$tmpdir/qmrestore.stat";
6577         if (my $fd = IO::File->new($statfile, "r")) {
6578             while (defined (my $line = <$fd>)) {
6579                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
6580                     $map->{$1} = $2 if $1;
6581                 } else {
6582                     print STDERR "unable to parse line in statfile - $line\n";
6583                 }
6584             }
6585             $fd->close();
6586         }
6587
6588         my $confsrc = "$tmpdir/qemu-server.conf";
6589
6590         my $srcfd = new IO::File($confsrc, "r") ||
6591             die "unable to open file '$confsrc'\n";
6592
6593         my $outfd = new IO::File ($tmpfn, "w") ||
6594             die "unable to write config for VM $vmid\n";
6595
6596         my $cookie = { netcount => 0 };
6597         while (defined (my $line = <$srcfd>)) {
6598             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
6599         }
6600
6601         $srcfd->close();
6602         $outfd->close();
6603     };
6604     if (my $err = $@) {
6605         unlink $tmpfn;
6606         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
6607         die $err;
6608     }
6609
6610     rmtree $tmpdir;
6611
6612     rename $tmpfn, $conffile ||
6613         die "unable to commit configuration file '$conffile'\n";
6614
6615     PVE::Cluster::cfs_update(); # make sure we read new file
6616
6617     eval { rescan($vmid, 1); };
6618     warn $@ if $@;
6619 };
6620
6621 sub foreach_storage_used_by_vm {
6622     my ($conf, $func) = @_;
6623
6624     my $sidhash = {};
6625
6626     foreach_drive($conf, sub {
6627         my ($ds, $drive) = @_;
6628         return if drive_is_cdrom($drive);
6629
6630         my $volid = $drive->{file};
6631
6632         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
6633         $sidhash->{$sid} = $sid if $sid;
6634     });
6635
6636     foreach my $sid (sort keys %$sidhash) {
6637         &$func($sid);
6638     }
6639 }
6640
6641 my $qemu_snap_storage = {
6642     rbd => 1,
6643 };
6644 sub do_snapshots_with_qemu {
6645     my ($storecfg, $volid) = @_;
6646
6647     my $storage_name = PVE::Storage::parse_volume_id($volid);
6648     my $scfg = $storecfg->{ids}->{$storage_name};
6649
6650     if ($qemu_snap_storage->{$scfg->{type}} && !$scfg->{krbd}){
6651         return 1;
6652     }
6653
6654     if ($volid =~ m/\.(qcow2|qed)$/){
6655         return 1;
6656     }
6657
6658     return undef;
6659 }
6660
6661 sub qga_check_running {
6662     my ($vmid, $nowarn) = @_;
6663
6664     eval { mon_cmd($vmid, "guest-ping", timeout => 3); };
6665     if ($@) {
6666         warn "Qemu Guest Agent is not running - $@" if !$nowarn;
6667         return 0;
6668     }
6669     return 1;
6670 }
6671
6672 sub template_create {
6673     my ($vmid, $conf, $disk) = @_;
6674
6675     my $storecfg = PVE::Storage::config();
6676
6677     foreach_drive($conf, sub {
6678         my ($ds, $drive) = @_;
6679
6680         return if drive_is_cdrom($drive);
6681         return if $disk && $ds ne $disk;
6682
6683         my $volid = $drive->{file};
6684         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
6685
6686         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
6687         $drive->{file} = $voliddst;
6688         $conf->{$ds} = print_drive($vmid, $drive);
6689         PVE::QemuConfig->write_config($vmid, $conf);
6690     });
6691 }
6692
6693 sub convert_iscsi_path {
6694     my ($path) = @_;
6695
6696     if ($path =~ m|^iscsi://([^/]+)/([^/]+)/(.+)$|) {
6697         my $portal = $1;
6698         my $target = $2;
6699         my $lun = $3;
6700
6701         my $initiator_name = get_initiator_name();
6702
6703         return "file.driver=iscsi,file.transport=tcp,file.initiator-name=$initiator_name,".
6704                "file.portal=$portal,file.target=$target,file.lun=$lun,driver=raw";
6705     }
6706
6707     die "cannot convert iscsi path '$path', unkown format\n";
6708 }
6709
6710 sub qemu_img_convert {
6711     my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized) = @_;
6712
6713     my $storecfg = PVE::Storage::config();
6714     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
6715     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
6716
6717     die "destination '$dst_volid' is not a valid volid form qemu-img convert\n" if !$dst_storeid;
6718
6719     my $cachemode;
6720     my $src_path;
6721     my $src_is_iscsi = 0;
6722     my $src_format = 'raw';
6723
6724     if ($src_storeid) {
6725         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
6726         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
6727         $src_format = qemu_img_format($src_scfg, $src_volname);
6728         $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
6729         $src_is_iscsi = ($src_path =~ m|^iscsi://|);
6730         $cachemode = 'none' if $src_scfg->{type} eq 'zfspool';
6731     } elsif (-f $src_volid) {
6732         $src_path = $src_volid;
6733         if ($src_path =~ m/\.($QEMU_FORMAT_RE)$/) {
6734             $src_format = $1;
6735         }
6736     }
6737
6738     die "source '$src_volid' is not a valid volid nor path for qemu-img convert\n" if !$src_path;
6739
6740     my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6741     my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
6742     my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6743     my $dst_is_iscsi = ($dst_path =~ m|^iscsi://|);
6744
6745     my $cmd = [];
6746     push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
6747     push @$cmd, '-l', "snapshot.name=$snapname" if($snapname && $src_format eq "qcow2");
6748     push @$cmd, '-t', 'none' if $dst_scfg->{type} eq 'zfspool';
6749     push @$cmd, '-T', $cachemode if defined($cachemode);
6750
6751     if ($src_is_iscsi) {
6752         push @$cmd, '--image-opts';
6753         $src_path = convert_iscsi_path($src_path);
6754     } else {
6755         push @$cmd, '-f', $src_format;
6756     }
6757
6758     if ($dst_is_iscsi) {
6759         push @$cmd, '--target-image-opts';
6760         $dst_path = convert_iscsi_path($dst_path);
6761     } else {
6762         push @$cmd, '-O', $dst_format;
6763     }
6764
6765     push @$cmd, $src_path;
6766
6767     if (!$dst_is_iscsi && $is_zero_initialized) {
6768         push @$cmd, "zeroinit:$dst_path";
6769     } else {
6770         push @$cmd, $dst_path;
6771     }
6772
6773     my $parser = sub {
6774         my $line = shift;
6775         if($line =~ m/\((\S+)\/100\%\)/){
6776             my $percent = $1;
6777             my $transferred = int($size * $percent / 100);
6778             my $remaining = $size - $transferred;
6779
6780             print "transferred: $transferred bytes remaining: $remaining bytes total: $size bytes progression: $percent %\n";
6781         }
6782
6783     };
6784
6785     eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
6786     my $err = $@;
6787     die "copy failed: $err" if $err;
6788 }
6789
6790 sub qemu_img_format {
6791     my ($scfg, $volname) = @_;
6792
6793     if ($scfg->{path} && $volname =~ m/\.($QEMU_FORMAT_RE)$/) {
6794         return $1;
6795     } else {
6796         return "raw";
6797     }
6798 }
6799
6800 sub qemu_drive_mirror {
6801     my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $skipcomplete, $qga, $bwlimit) = @_;
6802
6803     $jobs = {} if !$jobs;
6804
6805     my $qemu_target;
6806     my $format;
6807     $jobs->{"drive-$drive"} = {};
6808
6809     if ($dst_volid =~ /^nbd:/) {
6810         $qemu_target = $dst_volid;
6811         $format = "nbd";
6812     } else {
6813         my $storecfg = PVE::Storage::config();
6814         my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
6815
6816         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6817
6818         $format = qemu_img_format($dst_scfg, $dst_volname);
6819
6820         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6821
6822         $qemu_target = $is_zero_initialized ? "zeroinit:$dst_path" : $dst_path;
6823     }
6824
6825     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $qemu_target };
6826     $opts->{format} = $format if $format;
6827
6828     if (defined($bwlimit)) {
6829         $opts->{speed} = $bwlimit * 1024;
6830         print "drive mirror is starting for drive-$drive with bandwidth limit: ${bwlimit} KB/s\n";
6831     } else {
6832         print "drive mirror is starting for drive-$drive\n";
6833     }
6834
6835     # if a job already runs for this device we get an error, catch it for cleanup
6836     eval { mon_cmd($vmid, "drive-mirror", %$opts); };
6837     if (my $err = $@) {
6838         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6839         warn "$@\n" if $@;
6840         die "mirroring error: $err\n";
6841     }
6842
6843     qemu_drive_mirror_monitor ($vmid, $vmiddst, $jobs, $skipcomplete, $qga);
6844 }
6845
6846 sub qemu_drive_mirror_monitor {
6847     my ($vmid, $vmiddst, $jobs, $skipcomplete, $qga) = @_;
6848
6849     eval {
6850         my $err_complete = 0;
6851
6852         while (1) {
6853             die "storage migration timed out\n" if $err_complete > 300;
6854
6855             my $stats = mon_cmd($vmid, "query-block-jobs");
6856
6857             my $running_mirror_jobs = {};
6858             foreach my $stat (@$stats) {
6859                 next if $stat->{type} ne 'mirror';
6860                 $running_mirror_jobs->{$stat->{device}} = $stat;
6861             }
6862
6863             my $readycounter = 0;
6864
6865             foreach my $job (keys %$jobs) {
6866
6867                 if(defined($jobs->{$job}->{complete}) && !defined($running_mirror_jobs->{$job})) {
6868                     print "$job : finished\n";
6869                     delete $jobs->{$job};
6870                     next;
6871                 }
6872
6873                 die "$job: mirroring has been cancelled\n" if !defined($running_mirror_jobs->{$job});
6874
6875                 my $busy = $running_mirror_jobs->{$job}->{busy};
6876                 my $ready = $running_mirror_jobs->{$job}->{ready};
6877                 if (my $total = $running_mirror_jobs->{$job}->{len}) {
6878                     my $transferred = $running_mirror_jobs->{$job}->{offset} || 0;
6879                     my $remaining = $total - $transferred;
6880                     my $percent = sprintf "%.2f", ($transferred * 100 / $total);
6881
6882                     print "$job: transferred: $transferred bytes remaining: $remaining bytes total: $total bytes progression: $percent % busy: $busy ready: $ready \n";
6883                 }
6884
6885                 $readycounter++ if $running_mirror_jobs->{$job}->{ready};
6886             }
6887
6888             last if scalar(keys %$jobs) == 0;
6889
6890             if ($readycounter == scalar(keys %$jobs)) {
6891                 print "all mirroring jobs are ready \n";
6892                 last if $skipcomplete; #do the complete later
6893
6894                 if ($vmiddst && $vmiddst != $vmid) {
6895                     my $agent_running = $qga && qga_check_running($vmid);
6896                     if ($agent_running) {
6897                         print "freeze filesystem\n";
6898                         eval { mon_cmd($vmid, "guest-fsfreeze-freeze"); };
6899                     } else {
6900                         print "suspend vm\n";
6901                         eval { PVE::QemuServer::vm_suspend($vmid, 1); };
6902                     }
6903
6904                     # if we clone a disk for a new target vm, we don't switch the disk
6905                     PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs);
6906
6907                     if ($agent_running) {
6908                         print "unfreeze filesystem\n";
6909                         eval { mon_cmd($vmid, "guest-fsfreeze-thaw"); };
6910                     } else {
6911                         print "resume vm\n";
6912                         eval {  PVE::QemuServer::vm_resume($vmid, 1, 1); };
6913                     }
6914
6915                     last;
6916                 } else {
6917
6918                     foreach my $job (keys %$jobs) {
6919                         # try to switch the disk if source and destination are on the same guest
6920                         print "$job: Completing block job...\n";
6921
6922                         eval { mon_cmd($vmid, "block-job-complete", device => $job) };
6923                         if ($@ =~ m/cannot be completed/) {
6924                             print "$job: Block job cannot be completed, try again.\n";
6925                             $err_complete++;
6926                         }else {
6927                             print "$job: Completed successfully.\n";
6928                             $jobs->{$job}->{complete} = 1;
6929                         }
6930                     }
6931                 }
6932             }
6933             sleep 1;
6934         }
6935     };
6936     my $err = $@;
6937
6938     if ($err) {
6939         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6940         die "mirroring error: $err";
6941     }
6942
6943 }
6944
6945 sub qemu_blockjobs_cancel {
6946     my ($vmid, $jobs) = @_;
6947
6948     foreach my $job (keys %$jobs) {
6949         print "$job: Cancelling block job\n";
6950         eval { mon_cmd($vmid, "block-job-cancel", device => $job); };
6951         $jobs->{$job}->{cancel} = 1;
6952     }
6953
6954     while (1) {
6955         my $stats = mon_cmd($vmid, "query-block-jobs");
6956
6957         my $running_jobs = {};
6958         foreach my $stat (@$stats) {
6959             $running_jobs->{$stat->{device}} = $stat;
6960         }
6961
6962         foreach my $job (keys %$jobs) {
6963
6964             if (defined($jobs->{$job}->{cancel}) && !defined($running_jobs->{$job})) {
6965                 print "$job: Done.\n";
6966                 delete $jobs->{$job};
6967             }
6968         }
6969
6970         last if scalar(keys %$jobs) == 0;
6971
6972         sleep 1;
6973     }
6974 }
6975
6976 sub clone_disk {
6977     my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
6978         $newvmid, $storage, $format, $full, $newvollist, $jobs, $skipcomplete, $qga, $bwlimit) = @_;
6979
6980     my $newvolid;
6981
6982     if (!$full) {
6983         print "create linked clone of drive $drivename ($drive->{file})\n";
6984         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
6985         push @$newvollist, $newvolid;
6986     } else {
6987
6988         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6989         $storeid = $storage if $storage;
6990
6991         my $dst_format = resolve_dst_disk_format($storecfg, $storeid, $volname, $format);
6992         my ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 3);
6993
6994         print "create full clone of drive $drivename ($drive->{file})\n";
6995         my $name = undef;
6996         if (drive_is_cloudinit($drive)) {
6997             $name = "vm-$newvmid-cloudinit";
6998             $name .= ".$dst_format" if $dst_format ne 'raw';
6999             $snapname = undef;
7000             $size = PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE;
7001         }
7002         $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $newvmid, $dst_format, $name, ($size/1024));
7003         push @$newvollist, $newvolid;
7004
7005         PVE::Storage::activate_volumes($storecfg, [$newvolid]);
7006
7007         if (drive_is_cloudinit($drive)) {
7008             goto no_data_clone;
7009         }
7010
7011         my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
7012         if (!$running || $snapname) {
7013             # TODO: handle bwlimits
7014             qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit);
7015         } else {
7016
7017             my $kvmver = get_running_qemu_version ($vmid);
7018             if (!min_version($kvmver, 2, 7)) {
7019                 die "drive-mirror with iothread requires qemu version 2.7 or higher\n"
7020                     if $drive->{iothread};
7021             }
7022
7023             qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid, $sparseinit, $jobs, $skipcomplete, $qga, $bwlimit);
7024         }
7025     }
7026
7027 no_data_clone:
7028     my ($size) = PVE::Storage::volume_size_info($storecfg, $newvolid, 3);
7029
7030     my $disk = $drive;
7031     $disk->{format} = undef;
7032     $disk->{file} = $newvolid;
7033     $disk->{size} = $size;
7034
7035     return $disk;
7036 }
7037
7038 sub get_running_qemu_version {
7039     my ($vmid) = @_;
7040     my $res = mon_cmd($vmid, "query-version");
7041     return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
7042 }
7043
7044 sub qemu_use_old_bios_files {
7045     my ($machine_type) = @_;
7046
7047     return if !$machine_type;
7048
7049     my $use_old_bios_files = undef;
7050
7051     if ($machine_type =~ m/^(\S+)\.pxe$/) {
7052         $machine_type = $1;
7053         $use_old_bios_files = 1;
7054     } else {
7055         my $version = PVE::QemuServer::Machine::extract_version($machine_type, kvm_user_version());
7056         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
7057         # load new efi bios files on migration. So this hack is required to allow
7058         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
7059         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
7060         $use_old_bios_files = !min_version($version, 2, 4);
7061     }
7062
7063     return ($use_old_bios_files, $machine_type);
7064 }
7065
7066 sub create_efidisk($$$$$) {
7067     my ($storecfg, $storeid, $vmid, $fmt, $arch) = @_;
7068
7069     my (undef, $ovmf_vars) = get_ovmf_files($arch);
7070     die "EFI vars default image not found\n" if ! -f $ovmf_vars;
7071
7072     my $vars_size_b = -s $ovmf_vars;
7073     my $vars_size = PVE::Tools::convert_size($vars_size_b, 'b' => 'kb');
7074     my $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $fmt, undef, $vars_size);
7075     PVE::Storage::activate_volumes($storecfg, [$volid]);
7076
7077     qemu_img_convert($ovmf_vars, $volid, $vars_size_b, undef, 0);
7078
7079     return ($volid, $vars_size);
7080 }
7081
7082 sub vm_iothreads_list {
7083     my ($vmid) = @_;
7084
7085     my $res = mon_cmd($vmid, 'query-iothreads');
7086
7087     my $iothreads = {};
7088     foreach my $iothread (@$res) {
7089         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
7090     }
7091
7092     return $iothreads;
7093 }
7094
7095 sub scsihw_infos {
7096     my ($conf, $drive) = @_;
7097
7098     my $maxdev = 0;
7099
7100     if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)) {
7101         $maxdev = 7;
7102     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
7103         $maxdev = 1;
7104     } else {
7105         $maxdev = 256;
7106     }
7107
7108     my $controller = int($drive->{index} / $maxdev);
7109     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single') ? "virtioscsi" : "scsihw";
7110
7111     return ($maxdev, $controller, $controller_prefix);
7112 }
7113
7114 sub add_hyperv_enlightenments {
7115     my ($cpuFlags, $winversion, $machine_version, $bios, $gpu_passthrough, $hv_vendor_id) = @_;
7116
7117     return if $winversion < 6;
7118     return if $bios && $bios eq 'ovmf' && $winversion < 8;
7119
7120     if ($gpu_passthrough || defined($hv_vendor_id)) {
7121         $hv_vendor_id //= 'proxmox';
7122         push @$cpuFlags , "hv_vendor_id=$hv_vendor_id";
7123     }
7124
7125     if (min_version($machine_version, 2, 3)) {
7126         push @$cpuFlags , 'hv_spinlocks=0x1fff';
7127         push @$cpuFlags , 'hv_vapic';
7128         push @$cpuFlags , 'hv_time';
7129     } else {
7130         push @$cpuFlags , 'hv_spinlocks=0xffff';
7131     }
7132
7133     if (min_version($machine_version, 2, 6)) {
7134         push @$cpuFlags , 'hv_reset';
7135         push @$cpuFlags , 'hv_vpindex';
7136         push @$cpuFlags , 'hv_runtime';
7137     }
7138
7139     if ($winversion >= 7) {
7140         push @$cpuFlags , 'hv_relaxed';
7141
7142         if (min_version($machine_version, 2, 12)) {
7143             push @$cpuFlags , 'hv_synic';
7144             push @$cpuFlags , 'hv_stimer';
7145         }
7146
7147         if (min_version($machine_version, 3, 1)) {
7148             push @$cpuFlags , 'hv_ipi';
7149         }
7150     }
7151 }
7152
7153 sub windows_version {
7154     my ($ostype) = @_;
7155
7156     return 0 if !$ostype;
7157
7158     my $winversion = 0;
7159
7160     if($ostype eq 'wxp' || $ostype eq 'w2k3' || $ostype eq 'w2k') {
7161         $winversion = 5;
7162     } elsif($ostype eq 'w2k8' || $ostype eq 'wvista') {
7163         $winversion = 6;
7164     } elsif ($ostype =~ m/^win(\d+)$/) {
7165         $winversion = $1;
7166     }
7167
7168     return $winversion;
7169 }
7170
7171 sub resolve_dst_disk_format {
7172         my ($storecfg, $storeid, $src_volname, $format) = @_;
7173         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
7174
7175         if (!$format) {
7176             # if no target format is specified, use the source disk format as hint
7177             if ($src_volname) {
7178                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
7179                 $format = qemu_img_format($scfg, $src_volname);
7180             } else {
7181                 return $defFormat;
7182             }
7183         }
7184
7185         # test if requested format is supported - else use default
7186         my $supported = grep { $_ eq $format } @$validFormats;
7187         $format = $defFormat if !$supported;
7188         return $format;
7189 }
7190
7191 sub resolve_first_disk {
7192     my $conf = shift;
7193     my @disks = PVE::QemuServer::valid_drive_names();
7194     my $firstdisk;
7195     foreach my $ds (reverse @disks) {
7196         next if !$conf->{$ds};
7197         my $disk = PVE::QemuServer::parse_drive($ds, $conf->{$ds});
7198         next if PVE::QemuServer::drive_is_cdrom($disk);
7199         $firstdisk = $ds;
7200     }
7201     return $firstdisk;
7202 }
7203
7204 sub generate_uuid {
7205     my ($uuid, $uuid_str);
7206     UUID::generate($uuid);
7207     UUID::unparse($uuid, $uuid_str);
7208     return $uuid_str;
7209 }
7210
7211 sub generate_smbios1_uuid {
7212     return "uuid=".generate_uuid();
7213 }
7214
7215 sub nbd_stop {
7216     my ($vmid) = @_;
7217
7218     mon_cmd($vmid, 'nbd-server-stop');
7219 }
7220
7221 sub create_reboot_request {
7222     my ($vmid) = @_;
7223     open(my $fh, '>', "/run/qemu-server/$vmid.reboot")
7224         or die "failed to create reboot trigger file: $!\n";
7225     close($fh);
7226 }
7227
7228 sub clear_reboot_request {
7229     my ($vmid) = @_;
7230     my $path = "/run/qemu-server/$vmid.reboot";
7231     my $res = 0;
7232
7233     $res = unlink($path);
7234     die "could not remove reboot request for $vmid: $!"
7235         if !$res && $! != POSIX::ENOENT;
7236
7237     return $res;
7238 }
7239
7240 # bash completion helper
7241
7242 sub complete_backup_archives {
7243     my ($cmdname, $pname, $cvalue) = @_;
7244
7245     my $cfg = PVE::Storage::config();
7246
7247     my $storeid;
7248
7249     if ($cvalue =~ m/^([^:]+):/) {
7250         $storeid = $1;
7251     }
7252
7253     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
7254
7255     my $res = [];
7256     foreach my $id (keys %$data) {
7257         foreach my $item (@{$data->{$id}}) {
7258             next if $item->{format} !~ m/^vma\.(gz|lzo)$/;
7259             push @$res, $item->{volid} if defined($item->{volid});
7260         }
7261     }
7262
7263     return $res;
7264 }
7265
7266 my $complete_vmid_full = sub {
7267     my ($running) = @_;
7268
7269     my $idlist = vmstatus();
7270
7271     my $res = [];
7272
7273     foreach my $id (keys %$idlist) {
7274         my $d = $idlist->{$id};
7275         if (defined($running)) {
7276             next if $d->{template};
7277             next if $running && $d->{status} ne 'running';
7278             next if !$running && $d->{status} eq 'running';
7279         }
7280         push @$res, $id;
7281
7282     }
7283     return $res;
7284 };
7285
7286 sub complete_vmid {
7287     return &$complete_vmid_full();
7288 }
7289
7290 sub complete_vmid_stopped {
7291     return &$complete_vmid_full(0);
7292 }
7293
7294 sub complete_vmid_running {
7295     return &$complete_vmid_full(1);
7296 }
7297
7298 sub complete_storage {
7299
7300     my $cfg = PVE::Storage::config();
7301     my $ids = $cfg->{ids};
7302
7303     my $res = [];
7304     foreach my $sid (keys %$ids) {
7305         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
7306         next if !$ids->{$sid}->{content}->{images};
7307         push @$res, $sid;
7308     }
7309
7310     return $res;
7311 }
7312
7313 sub complete_migration_storage {
7314     my ($cmd, $param, $current_value, $all_args) = @_;
7315
7316     my $targetnode = @$all_args[1];
7317
7318     my $cfg = PVE::Storage::config();
7319     my $ids = $cfg->{ids};
7320
7321     my $res = [];
7322     foreach my $sid (keys %$ids) {
7323         next if !PVE::Storage::storage_check_enabled($cfg, $sid, $targetnode, 1);
7324         next if !$ids->{$sid}->{content}->{images};
7325         push @$res, $sid;
7326     }
7327
7328     return $res;
7329 }
7330
7331 1;