PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5
   6 use Cwd 'abs_path';
   7 use Digest::SHA;
   8 use Fcntl ':flock';
   9 use Fcntl;
  10 use File::Basename;
  11 use File::Copy qw(copy);
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use IO::Dir;
  16 use IO::File;
  17 use IO::Handle;
  18 use IO::Select;
  19 use IO::Socket::UNIX;
  20 use IPC::Open3;
  21 use JSON;
  22 use MIME::Base64;
  23 use POSIX;
  24 use Storable qw(dclone);
  25 use Time::HiRes qw(gettimeofday);
  26 use URI::Escape;
  27 use UUID;
  28
  29 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file cfs_lock_file);
  30 use PVE::DataCenterConfig;
  31 use PVE::Exception qw(raise raise_param_exc);
  32 use PVE::GuestHelpers;
  33 use PVE::INotify;
  34 use PVE::JSONSchema qw(get_standard_option);
  35 use PVE::ProcFSTools;
  36 use PVE::RPCEnvironment;
  37 use PVE::Storage;
  38 use PVE::SysFSTools;
  39 use PVE::Systemd;
  40 use PVE::Tools qw(run_command lock_file lock_file_full file_read_firstline dir_glob_foreach get_host_arch $IPV6RE);
  41
  42 use PVE::QMPClient;
  43 use PVE::QemuConfig;
  44 use PVE::QemuServer::Helpers qw(min_version);
  45 use PVE::QemuServer::Cloudinit;
  46 use PVE::QemuServer::Machine;
  47 use PVE::QemuServer::Memory;
  48 use PVE::QemuServer::Monitor qw(mon_cmd);
  49 use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr print_pcie_root_port);
  50 use PVE::QemuServer::USB qw(parse_usb_device);
  51
  52 my $EDK2_FW_BASE = '/usr/share/pve-edk2-firmware/';
  53 my $OVMF = {
  54     x86_64 => [
  55         "$EDK2_FW_BASE/OVMF_CODE.fd",
  56         "$EDK2_FW_BASE/OVMF_VARS.fd"
  57     ],
  58     aarch64 => [
  59         "$EDK2_FW_BASE/AAVMF_CODE.fd",
  60         "$EDK2_FW_BASE/AAVMF_VARS.fd"
  61     ],
  62 };
  63
  64 my $qemu_snap_storage = { rbd => 1 };
  65
  66 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  67
  68 my $QEMU_FORMAT_RE = qr/raw|cow|qcow|qcow2|qed|vmdk|cloop/;
  69
  70 # Note about locking: we use flock on the config file protect
  71 # against concurent actions.
  72 # Aditionaly, we have a 'lock' setting in the config file. This
  73 # can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
  74 # allowed when such lock is set. But you can ignore this kind of
  75 # lock with the --skiplock flag.
  76
  77 cfs_register_file('/qemu-server/',
  78                   \&parse_vm_config,
  79                   \&write_vm_config);
  80
  81 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
  82     description => "Some command save/restore state from this location.",
  83     type => 'string',
  84     maxLength => 128,
  85     optional => 1,
  86 });
  87
  88 PVE::JSONSchema::register_standard_option('pve-qm-image-format', {
  89     type => 'string',
  90     enum => [qw(raw cow qcow qed qcow2 vmdk cloop)],
  91     description => "The drive's backing file's data format.",
  92     optional => 1,
  93 });
  94
  95 PVE::JSONSchema::register_standard_option('pve-qemu-machine', {
  96         description => "Specifies the Qemu machine type.",
  97         type => 'string',
  98         pattern => '(pc|pc(-i440fx)?-\d+(\.\d+)+(\.pxe)?|q35|pc-q35-\d+(\.\d+)+(\.pxe)?|virt(?:-\d+(\.\d+)+)?)',
  99         maxLength => 40,
 100         optional => 1,
 101 });
 102
 103 #no warnings 'redefine';
 104
 105 sub cgroups_write {
 106    my ($controller, $vmid, $option, $value) = @_;
 107
 108    my $path = "/sys/fs/cgroup/$controller/qemu.slice/$vmid.scope/$option";
 109    PVE::ProcFSTools::write_proc_entry($path, $value);
 110
 111 }
 112
 113 my $nodename = PVE::INotify::nodename();
 114
 115 my $cpu_vendor_list = {
 116     # Intel CPUs
 117     486 => 'GenuineIntel',
 118     pentium => 'GenuineIntel',
 119     pentium2  => 'GenuineIntel',
 120     pentium3  => 'GenuineIntel',
 121     coreduo => 'GenuineIntel',
 122     core2duo => 'GenuineIntel',
 123     Conroe  => 'GenuineIntel',
 124     Penryn  => 'GenuineIntel',
 125     Nehalem  => 'GenuineIntel',
 126     'Nehalem-IBRS'  => 'GenuineIntel',
 127     Westmere => 'GenuineIntel',
 128     'Westmere-IBRS' => 'GenuineIntel',
 129     SandyBridge => 'GenuineIntel',
 130     'SandyBridge-IBRS' => 'GenuineIntel',
 131     IvyBridge => 'GenuineIntel',
 132     'IvyBridge-IBRS' => 'GenuineIntel',
 133     Haswell => 'GenuineIntel',
 134     'Haswell-IBRS' => 'GenuineIntel',
 135     'Haswell-noTSX' => 'GenuineIntel',
 136     'Haswell-noTSX-IBRS' => 'GenuineIntel',
 137     Broadwell => 'GenuineIntel',
 138     'Broadwell-IBRS' => 'GenuineIntel',
 139     'Broadwell-noTSX' => 'GenuineIntel',
 140     'Broadwell-noTSX-IBRS' => 'GenuineIntel',
 141     'Skylake-Client' => 'GenuineIntel',
 142     'Skylake-Client-IBRS' => 'GenuineIntel',
 143     'Skylake-Server' => 'GenuineIntel',
 144     'Skylake-Server-IBRS' => 'GenuineIntel',
 145     'Cascadelake-Server' => 'GenuineIntel',
 146     KnightsMill => 'GenuineIntel',
 147
 148
 149     # AMD CPUs
 150     athlon => 'AuthenticAMD',
 151     phenom  => 'AuthenticAMD',
 152     Opteron_G1  => 'AuthenticAMD',
 153     Opteron_G2  => 'AuthenticAMD',
 154     Opteron_G3  => 'AuthenticAMD',
 155     Opteron_G4  => 'AuthenticAMD',
 156     Opteron_G5  => 'AuthenticAMD',
 157     EPYC => 'AuthenticAMD',
 158     'EPYC-IBPB' => 'AuthenticAMD',
 159
 160     # generic types, use vendor from host node
 161     host => 'default',
 162     kvm32 => 'default',
 163     kvm64 => 'default',
 164     qemu32 => 'default',
 165     qemu64 => 'default',
 166     max => 'default',
 167 };
 168
 169 my @supported_cpu_flags = (
 170     'pcid',
 171     'spec-ctrl',
 172     'ibpb',
 173     'ssbd',
 174     'virt-ssbd',
 175     'amd-ssbd',
 176     'amd-no-ssb',
 177     'pdpe1gb',
 178     'md-clear',
 179     'hv-tlbflush',
 180     'hv-evmcs',
 181     'aes'
 182 );
 183 my $cpu_flag = qr/[+-](@{[join('|', @supported_cpu_flags)]})/;
 184
 185 my $cpu_fmt = {
 186     cputype => {
 187         description => "Emulated CPU type.",
 188         type => 'string',
 189         enum => [ sort { "\L$a" cmp "\L$b" } keys %$cpu_vendor_list ],
 190         default => 'kvm64',
 191         default_key => 1,
 192     },
 193     hidden => {
 194         description => "Do not identify as a KVM virtual machine.",
 195         type => 'boolean',
 196         optional => 1,
 197         default => 0
 198     },
 199     'hv-vendor-id' => {
 200         type => 'string',
 201         pattern => qr/[a-zA-Z0-9]{1,12}/,
 202         format_description => 'vendor-id',
 203         description => 'The Hyper-V vendor ID. Some drivers or programs inside Windows guests need a specific ID.',
 204         optional => 1,
 205     },
 206     flags => {
 207         description => "List of additional CPU flags separated by ';'."
 208                      . " Use '+FLAG' to enable, '-FLAG' to disable a flag."
 209                      . " Currently supported flags: @{[join(', ', @supported_cpu_flags)]}.",
 210         format_description => '+FLAG[;-FLAG...]',
 211         type => 'string',
 212         pattern => qr/$cpu_flag(;$cpu_flag)*/,
 213         optional => 1,
 214     },
 215 };
 216
 217 my $watchdog_fmt = {
 218     model => {
 219         default_key => 1,
 220         type => 'string',
 221         enum => [qw(i6300esb ib700)],
 222         description => "Watchdog type to emulate.",
 223         default => 'i6300esb',
 224         optional => 1,
 225     },
 226     action => {
 227         type => 'string',
 228         enum => [qw(reset shutdown poweroff pause debug none)],
 229         description => "The action to perform if after activation the guest fails to poll the watchdog in time.",
 230         optional => 1,
 231     },
 232 };
 233 PVE::JSONSchema::register_format('pve-qm-watchdog', $watchdog_fmt);
 234
 235 my $agent_fmt = {
 236     enabled => {
 237         description => "Enable/disable Qemu GuestAgent.",
 238         type => 'boolean',
 239         default => 0,
 240         default_key => 1,
 241     },
 242     fstrim_cloned_disks => {
 243         description => "Run fstrim after cloning/moving a disk.",
 244         type => 'boolean',
 245         optional => 1,
 246         default => 0
 247     },
 248     type => {
 249         description => "Select the agent type",
 250         type => 'string',
 251         default => 'virtio',
 252         optional => 1,
 253         enum => [qw(virtio isa)],
 254     },
 255 };
 256
 257 my $vga_fmt = {
 258     type => {
 259         description => "Select the VGA type.",
 260         type => 'string',
 261         default => 'std',
 262         optional => 1,
 263         default_key => 1,
 264         enum => [qw(cirrus qxl qxl2 qxl3 qxl4 none serial0 serial1 serial2 serial3 std virtio vmware)],
 265     },
 266     memory => {
 267         description => "Sets the VGA memory (in MiB). Has no effect with serial display.",
 268         type => 'integer',
 269         optional => 1,
 270         minimum => 4,
 271         maximum => 512,
 272     },
 273 };
 274
 275 my $ivshmem_fmt = {
 276     size => {
 277         type => 'integer',
 278         minimum => 1,
 279         description => "The size of the file in MB.",
 280     },
 281     name => {
 282         type => 'string',
 283         pattern => '[a-zA-Z0-9\-]+',
 284         optional => 1,
 285         format_description => 'string',
 286         description => "The name of the file. Will be prefixed with 'pve-shm-'. Default is the VMID. Will be deleted when the VM is stopped.",
 287     },
 288 };
 289
 290 my $audio_fmt = {
 291     device => {
 292         type => 'string',
 293         enum => [qw(ich9-intel-hda intel-hda AC97)],
 294         description =>  "Configure an audio device."
 295     },
 296     driver =>  {
 297         type => 'string',
 298         enum => ['spice'],
 299         default => 'spice',
 300         optional => 1,
 301         description => "Driver backend for the audio device."
 302     },
 303 };
 304
 305 my $spice_enhancements_fmt = {
 306     foldersharing => {
 307         type => 'boolean',
 308         optional => 1,
 309         default => '0',
 310         description =>  "Enable folder sharing via SPICE. Needs Spice-WebDAV daemon installed in the VM."
 311     },
 312     videostreaming =>  {
 313         type => 'string',
 314         enum => ['off', 'all', 'filter'],
 315         default => 'off',
 316         optional => 1,
 317         description => "Enable video streaming. Uses compression for detected video streams."
 318     },
 319 };
 320
 321 my $confdesc = {
 322     onboot => {
 323         optional => 1,
 324         type => 'boolean',
 325         description => "Specifies whether a VM will be started during system bootup.",
 326         default => 0,
 327     },
 328     autostart => {
 329         optional => 1,
 330         type => 'boolean',
 331         description => "Automatic restart after crash (currently ignored).",
 332         default => 0,
 333     },
 334     hotplug => {
 335         optional => 1,
 336         type => 'string', format => 'pve-hotplug-features',
 337         description => "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
 338         default => 'network,disk,usb',
 339     },
 340     reboot => {
 341         optional => 1,
 342         type => 'boolean',
 343         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 344         default => 1,
 345     },
 346     lock => {
 347         optional => 1,
 348         type => 'string',
 349         description => "Lock/unlock the VM.",
 350         enum => [qw(backup clone create migrate rollback snapshot snapshot-delete suspending suspended)],
 351     },
 352     cpulimit => {
 353         optional => 1,
 354         type => 'number',
 355         description => "Limit of CPU usage.",
 356         verbose_description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
 357         minimum => 0,
 358         maximum => 128,
 359         default => 0,
 360     },
 361     cpuunits => {
 362         optional => 1,
 363         type => 'integer',
 364         description => "CPU weight for a VM.",
 365         verbose_description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.",
 366         minimum => 2,
 367         maximum => 262144,
 368         default => 1024,
 369     },
 370     memory => {
 371         optional => 1,
 372         type => 'integer',
 373         description => "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
 374         minimum => 16,
 375         default => 512,
 376     },
 377     balloon => {
 378         optional => 1,
 379         type => 'integer',
 380         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 381         minimum => 0,
 382     },
 383     shares => {
 384         optional => 1,
 385         type => 'integer',
 386         description => "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning. Auto-ballooning is done by pvestatd.",
 387         minimum => 0,
 388         maximum => 50000,
 389         default => 1000,
 390     },
 391     keyboard => {
 392         optional => 1,
 393         type => 'string',
 394         description => "Keybord layout for vnc server. Default is read from the '/etc/pve/datacenter.cfg' configuration file.".
 395                        "It should not be necessary to set it.",
 396         enum => PVE::Tools::kvmkeymaplist(),
 397         default => undef,
 398     },
 399     name => {
 400         optional => 1,
 401         type => 'string', format => 'dns-name',
 402         description => "Set a name for the VM. Only used on the configuration web interface.",
 403     },
 404     scsihw => {
 405         optional => 1,
 406         type => 'string',
 407         description => "SCSI controller model",
 408         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 409         default => 'lsi',
 410     },
 411     description => {
 412         optional => 1,
 413         type => 'string',
 414         description => "Description for the VM. Only used on the configuration web interface. This is saved as comment inside the configuration file.",
 415     },
 416     ostype => {
 417         optional => 1,
 418         type => 'string',
 419         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 l24 l26 solaris)],
 420         description => "Specify guest operating system.",
 421         verbose_description => <<EODESC,
 422 Specify guest operating system. This is used to enable special
 423 optimization/features for specific operating systems:
 424
 425 [horizontal]
 426 other;; unspecified OS
 427 wxp;; Microsoft Windows XP
 428 w2k;; Microsoft Windows 2000
 429 w2k3;; Microsoft Windows 2003
 430 w2k8;; Microsoft Windows 2008
 431 wvista;; Microsoft Windows Vista
 432 win7;; Microsoft Windows 7
 433 win8;; Microsoft Windows 8/2012/2012r2
 434 win10;; Microsoft Windows 10/2016
 435 l24;; Linux 2.4 Kernel
 436 l26;; Linux 2.6 - 5.X Kernel
 437 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
 438 EODESC
 439     },
 440     boot => {
 441         optional => 1,
 442         type => 'string',
 443         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n).",
 444         pattern => '[acdn]{1,4}',
 445         default => 'cdn',
 446     },
 447     bootdisk => {
 448         optional => 1,
 449         type => 'string', format => 'pve-qm-bootdisk',
 450         description => "Enable booting from specified disk.",
 451         pattern => '(ide|sata|scsi|virtio)\d+',
 452     },
 453     smp => {
 454         optional => 1,
 455         type => 'integer',
 456         description => "The number of CPUs. Please use option -sockets instead.",
 457         minimum => 1,
 458         default => 1,
 459     },
 460     sockets => {
 461         optional => 1,
 462         type => 'integer',
 463         description => "The number of CPU sockets.",
 464         minimum => 1,
 465         default => 1,
 466     },
 467     cores => {
 468         optional => 1,
 469         type => 'integer',
 470         description => "The number of cores per socket.",
 471         minimum => 1,
 472         default => 1,
 473     },
 474     numa => {
 475         optional => 1,
 476         type => 'boolean',
 477         description => "Enable/disable NUMA.",
 478         default => 0,
 479     },
 480     hugepages => {
 481         optional => 1,
 482         type => 'string',
 483         description => "Enable/disable hugepages memory.",
 484         enum => [qw(any 2 1024)],
 485     },
 486     vcpus => {
 487         optional => 1,
 488         type => 'integer',
 489         description => "Number of hotplugged vcpus.",
 490         minimum => 1,
 491         default => 0,
 492     },
 493     acpi => {
 494         optional => 1,
 495         type => 'boolean',
 496         description => "Enable/disable ACPI.",
 497         default => 1,
 498     },
 499     agent => {
 500         optional => 1,
 501         description => "Enable/disable Qemu GuestAgent and its properties.",
 502         type => 'string',
 503         format => $agent_fmt,
 504     },
 505     kvm => {
 506         optional => 1,
 507         type => 'boolean',
 508         description => "Enable/disable KVM hardware virtualization.",
 509         default => 1,
 510     },
 511     tdf => {
 512         optional => 1,
 513         type => 'boolean',
 514         description => "Enable/disable time drift fix.",
 515         default => 0,
 516     },
 517     localtime => {
 518         optional => 1,
 519         type => 'boolean',
 520         description => "Set the real time clock to local time. This is enabled by default if ostype indicates a Microsoft OS.",
 521     },
 522     freeze => {
 523         optional => 1,
 524         type => 'boolean',
 525         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 526     },
 527     vga => {
 528         optional => 1,
 529         type => 'string', format => $vga_fmt,
 530         description => "Configure the VGA hardware.",
 531         verbose_description => "Configure the VGA Hardware. If you want to use ".
 532             "high resolution modes (>= 1280x1024x16) you may need to increase " .
 533             "the vga memory option. Since QEMU 2.9 the default VGA display type " .
 534             "is 'std' for all OS types besides some Windows versions (XP and " .
 535             "older) which use 'cirrus'. The 'qxl' option enables the SPICE " .
 536             "display server. For win* OS you can select how many independent " .
 537             "displays you want, Linux guests can add displays them self.\n".
 538             "You can also run without any graphic card, using a serial device as terminal.",
 539     },
 540     watchdog => {
 541         optional => 1,
 542         type => 'string', format => 'pve-qm-watchdog',
 543         description => "Create a virtual hardware watchdog device.",
 544         verbose_description => "Create a virtual hardware watchdog device. Once enabled" .
 545             " (by a guest action), the watchdog must be periodically polled " .
 546             "by an agent inside the guest or else the watchdog will reset " .
 547             "the guest (or execute the respective action specified)",
 548     },
 549     startdate => {
 550         optional => 1,
 551         type => 'string',
 552         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 553         description => "Set the initial date of the real time clock. Valid format for date are: 'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 554         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 555         default => 'now',
 556     },
 557     startup =>  get_standard_option('pve-startup-order'),
 558     template => {
 559         optional => 1,
 560         type => 'boolean',
 561         description => "Enable/disable Template.",
 562         default => 0,
 563     },
 564     args => {
 565         optional => 1,
 566         type => 'string',
 567         description => "Arbitrary arguments passed to kvm.",
 568         verbose_description => <<EODESCR,
 569 Arbitrary arguments passed to kvm, for example:
 570
 571 args: -no-reboot -no-hpet
 572
 573 NOTE: this option is for experts only.
 574 EODESCR
 575     },
 576     tablet => {
 577         optional => 1,
 578         type => 'boolean',
 579         default => 1,
 580         description => "Enable/disable the USB tablet device.",
 581         verbose_description => "Enable/disable the USB tablet device. This device is " .
 582             "usually needed to allow absolute mouse positioning with VNC. " .
 583             "Else the mouse runs out of sync with normal VNC clients. " .
 584             "If you're running lots of console-only guests on one host, " .
 585             "you may consider disabling this to save some context switches. " .
 586             "This is turned off by default if you use spice (-vga=qxl).",
 587     },
 588     migrate_speed => {
 589         optional => 1,
 590         type => 'integer',
 591         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 592         minimum => 0,
 593         default => 0,
 594     },
 595     migrate_downtime => {
 596         optional => 1,
 597         type => 'number',
 598         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 599         minimum => 0,
 600         default => 0.1,
 601     },
 602     cdrom => {
 603         optional => 1,
 604         type => 'string', format => 'pve-qm-ide',
 605         typetext => '<volume>',
 606         description => "This is an alias for option -ide2",
 607     },
 608     cpu => {
 609         optional => 1,
 610         description => "Emulated CPU type.",
 611         type => 'string',
 612         format => $cpu_fmt,
 613     },
 614     parent => get_standard_option('pve-snapshot-name', {
 615         optional => 1,
 616         description => "Parent snapshot name. This is used internally, and should not be modified.",
 617     }),
 618     snaptime => {
 619         optional => 1,
 620         description => "Timestamp for snapshots.",
 621         type => 'integer',
 622         minimum => 0,
 623     },
 624     vmstate => {
 625         optional => 1,
 626         type => 'string', format => 'pve-volume-id',
 627         description => "Reference to a volume which stores the VM state. This is used internally for snapshots.",
 628     },
 629     vmstatestorage => get_standard_option('pve-storage-id', {
 630         description => "Default storage for VM state volumes/files.",
 631         optional => 1,
 632     }),
 633     runningmachine => get_standard_option('pve-qemu-machine', {
 634         description => "Specifies the Qemu machine type of the running vm. This is used internally for snapshots.",
 635     }),
 636     machine => get_standard_option('pve-qemu-machine'),
 637     arch => {
 638         description => "Virtual processor architecture. Defaults to the host.",
 639         optional => 1,
 640         type => 'string',
 641         enum => [qw(x86_64 aarch64)],
 642     },
 643     smbios1 => {
 644         description => "Specify SMBIOS type 1 fields.",
 645         type => 'string', format => 'pve-qm-smbios1',
 646         maxLength => 512,
 647         optional => 1,
 648     },
 649     protection => {
 650         optional => 1,
 651         type => 'boolean',
 652         description => "Sets the protection flag of the VM. This will disable the remove VM and remove disk operations.",
 653         default => 0,
 654     },
 655     bios => {
 656         optional => 1,
 657         type => 'string',
 658         enum => [ qw(seabios ovmf) ],
 659         description => "Select BIOS implementation.",
 660         default => 'seabios',
 661     },
 662     vmgenid => {
 663         type => 'string',
 664         pattern => '(?:[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}|[01])',
 665         format_description => 'UUID',
 666         description => "Set VM Generation ID. Use '1' to autogenerate on create or update, pass '0' to disable explicitly.",
 667         verbose_description => "The VM generation ID (vmgenid) device exposes a".
 668             " 128-bit integer value identifier to the guest OS. This allows to".
 669             " notify the guest operating system when the virtual machine is".
 670             " executed with a different configuration (e.g. snapshot execution".
 671             " or creation from a template). The guest operating system notices".
 672             " the change, and is then able to react as appropriate by marking".
 673             " its copies of distributed databases as dirty, re-initializing its".
 674             " random number generator, etc.\n".
 675             "Note that auto-creation only works when done throug API/CLI create".
 676             " or update methods, but not when manually editing the config file.",
 677         default => "1 (autogenerated)",
 678         optional => 1,
 679     },
 680     hookscript => {
 681         type => 'string',
 682         format => 'pve-volume-id',
 683         optional => 1,
 684         description => "Script that will be executed during various steps in the vms lifetime.",
 685     },
 686     ivshmem => {
 687         type => 'string',
 688         format => $ivshmem_fmt,
 689         description => "Inter-VM shared memory. Useful for direct communication between VMs, or to the host.",
 690         optional => 1,
 691     },
 692     audio0 => {
 693         type => 'string',
 694         format => $audio_fmt,
 695         description => "Configure a audio device, useful in combination with QXL/Spice.",
 696         optional => 1
 697     },
 698     spice_enhancements => {
 699         type => 'string',
 700         format => $spice_enhancements_fmt,
 701         description => "Configure additional enhancements for SPICE.",
 702         optional => 1
 703     },
 704 };
 705
 706 my $cicustom_fmt = {
 707     meta => {
 708         type => 'string',
 709         optional => 1,
 710         description => 'Specify a custom file containing all meta data passed to the VM via cloud-init. This is provider specific meaning configdrive2 and nocloud differ.',
 711         format => 'pve-volume-id',
 712         format_description => 'volume',
 713     },
 714     network => {
 715         type => 'string',
 716         optional => 1,
 717         description => 'Specify a custom file containing all network data passed to the VM via cloud-init.',
 718         format => 'pve-volume-id',
 719         format_description => 'volume',
 720     },
 721     user => {
 722         type => 'string',
 723         optional => 1,
 724         description => 'Specify a custom file containing all user data passed to the VM via cloud-init.',
 725         format => 'pve-volume-id',
 726         format_description => 'volume',
 727     },
 728 };
 729 PVE::JSONSchema::register_format('pve-qm-cicustom', $cicustom_fmt);
 730
 731 my $confdesc_cloudinit = {
 732     citype => {
 733         optional => 1,
 734         type => 'string',
 735         description => 'Specifies the cloud-init configuration format. The default depends on the configured operating system type (`ostype`. We use the `nocloud` format for Linux, and `configdrive2` for windows.',
 736         enum => ['configdrive2', 'nocloud'],
 737     },
 738     ciuser => {
 739         optional => 1,
 740         type => 'string',
 741         description => "cloud-init: User name to change ssh keys and password for instead of the image's configured default user.",
 742     },
 743     cipassword => {
 744         optional => 1,
 745         type => 'string',
 746         description => 'cloud-init: Password to assign the user. Using this is generally not recommended. Use ssh keys instead. Also note that older cloud-init versions do not support hashed passwords.',
 747     },
 748     cicustom => {
 749         optional => 1,
 750         type => 'string',
 751         description => 'cloud-init: Specify custom files to replace the automatically generated ones at start.',
 752         format => 'pve-qm-cicustom',
 753     },
 754     searchdomain => {
 755         optional => 1,
 756         type => 'string',
 757         description => "cloud-init: Sets DNS search domains for a container. Create will automatically use the setting from the host if neither searchdomain nor nameserver are set.",
 758     },
 759     nameserver => {
 760         optional => 1,
 761         type => 'string', format => 'address-list',
 762         description => "cloud-init: Sets DNS server IP address for a container. Create will automatically use the setting from the host if neither searchdomain nor nameserver are set.",
 763     },
 764     sshkeys => {
 765         optional => 1,
 766         type => 'string',
 767         format => 'urlencoded',
 768         description => "cloud-init: Setup public SSH keys (one key per line, OpenSSH format).",
 769     },
 770 };
 771
 772 # what about other qemu settings ?
 773 #cpu => 'string',
 774 #machine => 'string',
 775 #fda => 'file',
 776 #fdb => 'file',
 777 #mtdblock => 'file',
 778 #sd => 'file',
 779 #pflash => 'file',
 780 #snapshot => 'bool',
 781 #bootp => 'file',
 782 ##tftp => 'dir',
 783 ##smb => 'dir',
 784 #kernel => 'file',
 785 #append => 'string',
 786 #initrd => 'file',
 787 ##soundhw => 'string',
 788
 789 while (my ($k, $v) = each %$confdesc) {
 790     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 791 }
 792
 793 my $MAX_IDE_DISKS = 4;
 794 my $MAX_SCSI_DISKS = 14;
 795 my $MAX_VIRTIO_DISKS = 16;
 796 my $MAX_SATA_DISKS = 6;
 797 my $MAX_USB_DEVICES = 5;
 798 my $MAX_NETS = 32;
 799 my $MAX_UNUSED_DISKS = 256;
 800 my $MAX_HOSTPCI_DEVICES = 16;
 801 my $MAX_SERIAL_PORTS = 4;
 802 my $MAX_PARALLEL_PORTS = 3;
 803 my $MAX_NUMA = 8;
 804
 805 my $numa_fmt = {
 806     cpus => {
 807         type => "string",
 808         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 809         description => "CPUs accessing this NUMA node.",
 810         format_description => "id[-id];...",
 811     },
 812     memory => {
 813         type => "number",
 814         description => "Amount of memory this NUMA node provides.",
 815         optional => 1,
 816     },
 817     hostnodes => {
 818         type => "string",
 819         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 820         description => "Host NUMA nodes to use.",
 821         format_description => "id[-id];...",
 822         optional => 1,
 823     },
 824     policy => {
 825         type => 'string',
 826         enum => [qw(preferred bind interleave)],
 827         description => "NUMA allocation policy.",
 828         optional => 1,
 829     },
 830 };
 831 PVE::JSONSchema::register_format('pve-qm-numanode', $numa_fmt);
 832 my $numadesc = {
 833     optional => 1,
 834     type => 'string', format => $numa_fmt,
 835     description => "NUMA topology.",
 836 };
 837 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 838
 839 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 840     $confdesc->{"numa$i"} = $numadesc;
 841 }
 842
 843 my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000',  'pcnet',  'virtio',
 844                       'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3',
 845                       'e1000-82540em', 'e1000-82544gc', 'e1000-82545em'];
 846 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 847
 848 my $net_fmt_bridge_descr = <<__EOD__;
 849 Bridge to attach the network device to. The Proxmox VE standard bridge
 850 is called 'vmbr0'.
 851
 852 If you do not specify a bridge, we create a kvm user (NATed) network
 853 device, which provides DHCP and DNS services. The following addresses
 854 are used:
 855
 856  10.0.2.2   Gateway
 857  10.0.2.3   DNS Server
 858  10.0.2.4   SMB Server
 859
 860 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 861 __EOD__
 862
 863 my $net_fmt = {
 864     macaddr  => get_standard_option('mac-addr', {
 865         description => "MAC address. That address must be unique withing your network. This is automatically generated if not specified.",
 866     }),
 867     model => {
 868         type => 'string',
 869         description => "Network Card Model. The 'virtio' model provides the best performance with very low CPU overhead. If your guest does not support this driver, it is usually best to use 'e1000'.",
 870         enum => $nic_model_list,
 871         default_key => 1,
 872     },
 873     (map { $_ => { keyAlias => 'model', alias => 'macaddr' }} @$nic_model_list),
 874     bridge => {
 875         type => 'string',
 876         description => $net_fmt_bridge_descr,
 877         format_description => 'bridge',
 878         optional => 1,
 879     },
 880     queues => {
 881         type => 'integer',
 882         minimum => 0, maximum => 16,
 883         description => 'Number of packet queues to be used on the device.',
 884         optional => 1,
 885     },
 886     rate => {
 887         type => 'number',
 888         minimum => 0,
 889         description => "Rate limit in mbps (megabytes per second) as floating point number.",
 890         optional => 1,
 891     },
 892     tag => {
 893         type => 'integer',
 894         minimum => 1, maximum => 4094,
 895         description => 'VLAN tag to apply to packets on this interface.',
 896         optional => 1,
 897     },
 898     trunks => {
 899         type => 'string',
 900         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 901         description => 'VLAN trunks to pass through this interface.',
 902         format_description => 'vlanid[;vlanid...]',
 903         optional => 1,
 904     },
 905     firewall => {
 906         type => 'boolean',
 907         description => 'Whether this interface should be protected by the firewall.',
 908         optional => 1,
 909     },
 910     link_down => {
 911         type => 'boolean',
 912         description => 'Whether this interface should be disconnected (like pulling the plug).',
 913         optional => 1,
 914     },
 915 };
 916
 917 my $netdesc = {
 918     optional => 1,
 919     type => 'string', format => $net_fmt,
 920     description => "Specify network devices.",
 921 };
 922
 923 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 924
 925 my $ipconfig_fmt = {
 926     ip => {
 927         type => 'string',
 928         format => 'pve-ipv4-config',
 929         format_description => 'IPv4Format/CIDR',
 930         description => 'IPv4 address in CIDR format.',
 931         optional => 1,
 932         default => 'dhcp',
 933     },
 934     gw => {
 935         type => 'string',
 936         format => 'ipv4',
 937         format_description => 'GatewayIPv4',
 938         description => 'Default gateway for IPv4 traffic.',
 939         optional => 1,
 940         requires => 'ip',
 941     },
 942     ip6 => {
 943         type => 'string',
 944         format => 'pve-ipv6-config',
 945         format_description => 'IPv6Format/CIDR',
 946         description => 'IPv6 address in CIDR format.',
 947         optional => 1,
 948         default => 'dhcp',
 949     },
 950     gw6 => {
 951         type => 'string',
 952         format => 'ipv6',
 953         format_description => 'GatewayIPv6',
 954         description => 'Default gateway for IPv6 traffic.',
 955         optional => 1,
 956         requires => 'ip6',
 957     },
 958 };
 959 PVE::JSONSchema::register_format('pve-qm-ipconfig', $ipconfig_fmt);
 960 my $ipconfigdesc = {
 961     optional => 1,
 962     type => 'string', format => 'pve-qm-ipconfig',
 963     description => <<'EODESCR',
 964 cloud-init: Specify IP addresses and gateways for the corresponding interface.
 965
 966 IP addresses use CIDR notation, gateways are optional but need an IP of the same type specified.
 967
 968 The special string 'dhcp' can be used for IP addresses to use DHCP, in which case no explicit gateway should be provided.
 969 For IPv6 the special string 'auto' can be used to use stateless autoconfiguration.
 970
 971 If cloud-init is enabled and neither an IPv4 nor an IPv6 address is specified, it defaults to using dhcp on IPv4.
 972 EODESCR
 973 };
 974 PVE::JSONSchema::register_standard_option("pve-qm-ipconfig", $netdesc);
 975
 976 for (my $i = 0; $i < $MAX_NETS; $i++)  {
 977     $confdesc->{"net$i"} = $netdesc;
 978     $confdesc_cloudinit->{"ipconfig$i"} = $ipconfigdesc;
 979 }
 980
 981 foreach my $key (keys %$confdesc_cloudinit) {
 982     $confdesc->{$key} = $confdesc_cloudinit->{$key};
 983 }
 984
 985 PVE::JSONSchema::register_format('pve-volume-id-or-qm-path', \&verify_volume_id_or_qm_path);
 986 sub verify_volume_id_or_qm_path {
 987     my ($volid, $noerr) = @_;
 988
 989     if ($volid eq 'none' || $volid eq 'cdrom' || $volid =~ m|^/|) {
 990         return $volid;
 991     }
 992
 993     # if its neither 'none' nor 'cdrom' nor a path, check if its a volume-id
 994     $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') };
 995     if ($@) {
 996         return undef if $noerr;
 997         die $@;
 998     }
 999     return $volid;
1000 }
1001
1002 my $drivename_hash;
1003
1004 my %drivedesc_base = (
1005     volume => { alias => 'file' },
1006     file => {
1007         type => 'string',
1008         format => 'pve-volume-id-or-qm-path',
1009         default_key => 1,
1010         format_description => 'volume',
1011         description => "The drive's backing volume.",
1012     },
1013     media => {
1014         type => 'string',
1015         enum => [qw(cdrom disk)],
1016         description => "The drive's media type.",
1017         default => 'disk',
1018         optional => 1
1019     },
1020     cyls => {
1021         type => 'integer',
1022         description => "Force the drive's physical geometry to have a specific cylinder count.",
1023         optional => 1
1024     },
1025     heads => {
1026         type => 'integer',
1027         description => "Force the drive's physical geometry to have a specific head count.",
1028         optional => 1
1029     },
1030     secs => {
1031         type => 'integer',
1032         description => "Force the drive's physical geometry to have a specific sector count.",
1033         optional => 1
1034     },
1035     trans => {
1036         type => 'string',
1037         enum => [qw(none lba auto)],
1038         description => "Force disk geometry bios translation mode.",
1039         optional => 1,
1040     },
1041     snapshot => {
1042         type => 'boolean',
1043         description => "Controls qemu's snapshot mode feature."
1044             . " If activated, changes made to the disk are temporary and will"
1045             . " be discarded when the VM is shutdown.",
1046         optional => 1,
1047     },
1048     cache => {
1049         type => 'string',
1050         enum => [qw(none writethrough writeback unsafe directsync)],
1051         description => "The drive's cache mode",
1052         optional => 1,
1053     },
1054     format => get_standard_option('pve-qm-image-format'),
1055     size => {
1056         type => 'string',
1057         format => 'disk-size',
1058         format_description => 'DiskSize',
1059         description => "Disk size. This is purely informational and has no effect.",
1060         optional => 1,
1061     },
1062     backup => {
1063         type => 'boolean',
1064         description => "Whether the drive should be included when making backups.",
1065         optional => 1,
1066     },
1067     replicate => {
1068         type => 'boolean',
1069         description => 'Whether the drive should considered for replication jobs.',
1070         optional => 1,
1071         default => 1,
1072     },
1073     rerror => {
1074         type => 'string',
1075         enum => [qw(ignore report stop)],
1076         description => 'Read error action.',
1077         optional => 1,
1078     },
1079     werror => {
1080         type => 'string',
1081         enum => [qw(enospc ignore report stop)],
1082         description => 'Write error action.',
1083         optional => 1,
1084     },
1085     aio => {
1086         type => 'string',
1087         enum => [qw(native threads)],
1088         description => 'AIO type to use.',
1089         optional => 1,
1090     },
1091     discard => {
1092         type => 'string',
1093         enum => [qw(ignore on)],
1094         description => 'Controls whether to pass discard/trim requests to the underlying storage.',
1095         optional => 1,
1096     },
1097     detect_zeroes => {
1098         type => 'boolean',
1099         description => 'Controls whether to detect and try to optimize writes of zeroes.',
1100         optional => 1,
1101     },
1102     serial => {
1103         type => 'string',
1104         format => 'urlencoded',
1105         format_description => 'serial',
1106         maxLength => 20*3, # *3 since it's %xx url enoded
1107         description => "The drive's reported serial number, url-encoded, up to 20 bytes long.",
1108         optional => 1,
1109     },
1110     shared => {
1111         type => 'boolean',
1112         description => 'Mark this locally-managed volume as available on all nodes',
1113         verbose_description => "Mark this locally-managed volume as available on all nodes.\n\nWARNING: This option does not share the volume automatically, it assumes it is shared already!",
1114         optional => 1,
1115         default => 0,
1116     }
1117 );
1118
1119 my %iothread_fmt = ( iothread => {
1120         type => 'boolean',
1121         description => "Whether to use iothreads for this drive",
1122         optional => 1,
1123 });
1124
1125 my %model_fmt = (
1126     model => {
1127         type => 'string',
1128         format => 'urlencoded',
1129         format_description => 'model',
1130         maxLength => 40*3, # *3 since it's %xx url enoded
1131         description => "The drive's reported model name, url-encoded, up to 40 bytes long.",
1132         optional => 1,
1133     },
1134 );
1135
1136 my %queues_fmt = (
1137     queues => {
1138         type => 'integer',
1139         description => "Number of queues.",
1140         minimum => 2,
1141         optional => 1
1142     }
1143 );
1144
1145 my %scsiblock_fmt = (
1146     scsiblock => {
1147         type => 'boolean',
1148         description => "whether to use scsi-block for full passthrough of host block device\n\nWARNING: can lead to I/O errors in combination with low memory or high memory fragmentation on host",
1149         optional => 1,
1150         default => 0,
1151     },
1152 );
1153
1154 my %ssd_fmt = (
1155     ssd => {
1156         type => 'boolean',
1157         description => "Whether to expose this drive as an SSD, rather than a rotational hard disk.",
1158         optional => 1,
1159     },
1160 );
1161
1162 my %wwn_fmt = (
1163     wwn => {
1164         type => 'string',
1165         pattern => qr/^(0x)[0-9a-fA-F]{16}/,
1166         format_description => 'wwn',
1167         description => "The drive's worldwide name, encoded as 16 bytes hex string, prefixed by '0x'.",
1168         optional => 1,
1169     },
1170 );
1171
1172 my $add_throttle_desc = sub {
1173     my ($key, $type, $what, $unit, $longunit, $minimum) = @_;
1174     my $d = {
1175         type => $type,
1176         format_description => $unit,
1177         description => "Maximum $what in $longunit.",
1178         optional => 1,
1179     };
1180     $d->{minimum} = $minimum if defined($minimum);
1181     $drivedesc_base{$key} = $d;
1182 };
1183 # throughput: (leaky bucket)
1184 $add_throttle_desc->('bps',     'integer', 'r/w speed',   'bps',  'bytes per second');
1185 $add_throttle_desc->('bps_rd',  'integer', 'read speed',  'bps',  'bytes per second');
1186 $add_throttle_desc->('bps_wr',  'integer', 'write speed', 'bps',  'bytes per second');
1187 $add_throttle_desc->('mbps',    'number',  'r/w speed',   'mbps', 'megabytes per second');
1188 $add_throttle_desc->('mbps_rd', 'number',  'read speed',  'mbps', 'megabytes per second');
1189 $add_throttle_desc->('mbps_wr', 'number',  'write speed', 'mbps', 'megabytes per second');
1190 $add_throttle_desc->('iops',    'integer', 'r/w I/O',     'iops', 'operations per second');
1191 $add_throttle_desc->('iops_rd', 'integer', 'read I/O',    'iops', 'operations per second');
1192 $add_throttle_desc->('iops_wr', 'integer', 'write I/O',   'iops', 'operations per second');
1193
1194 # pools: (pool of IO before throttling starts taking effect)
1195 $add_throttle_desc->('mbps_max',    'number',  'unthrottled r/w pool',       'mbps', 'megabytes per second');
1196 $add_throttle_desc->('mbps_rd_max', 'number',  'unthrottled read pool',      'mbps', 'megabytes per second');
1197 $add_throttle_desc->('mbps_wr_max', 'number',  'unthrottled write pool',     'mbps', 'megabytes per second');
1198 $add_throttle_desc->('iops_max',    'integer', 'unthrottled r/w I/O pool',   'iops', 'operations per second');
1199 $add_throttle_desc->('iops_rd_max', 'integer', 'unthrottled read I/O pool',  'iops', 'operations per second');
1200 $add_throttle_desc->('iops_wr_max', 'integer', 'unthrottled write I/O pool', 'iops', 'operations per second');
1201
1202 # burst lengths
1203 $add_throttle_desc->('bps_max_length',     'integer', 'length of I/O bursts',       'seconds', 'seconds', 1);
1204 $add_throttle_desc->('bps_rd_max_length',  'integer', 'length of read I/O bursts',  'seconds', 'seconds', 1);
1205 $add_throttle_desc->('bps_wr_max_length',  'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
1206 $add_throttle_desc->('iops_max_length',    'integer', 'length of I/O bursts',       'seconds', 'seconds', 1);
1207 $add_throttle_desc->('iops_rd_max_length', 'integer', 'length of read I/O bursts',  'seconds', 'seconds', 1);
1208 $add_throttle_desc->('iops_wr_max_length', 'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
1209
1210 # legacy support
1211 $drivedesc_base{'bps_rd_length'} = { alias => 'bps_rd_max_length' };
1212 $drivedesc_base{'bps_wr_length'} = { alias => 'bps_wr_max_length' };
1213 $drivedesc_base{'iops_rd_length'} = { alias => 'iops_rd_max_length' };
1214 $drivedesc_base{'iops_wr_length'} = { alias => 'iops_wr_max_length' };
1215
1216 my $ide_fmt = {
1217     %drivedesc_base,
1218     %model_fmt,
1219     %ssd_fmt,
1220     %wwn_fmt,
1221 };
1222 PVE::JSONSchema::register_format("pve-qm-ide", $ide_fmt);
1223
1224 my $idedesc = {
1225     optional => 1,
1226     type => 'string', format => $ide_fmt,
1227     description => "Use volume as IDE hard disk or CD-ROM (n is 0 to " .($MAX_IDE_DISKS -1) . ").",
1228 };
1229 PVE::JSONSchema::register_standard_option("pve-qm-ide", $idedesc);
1230
1231 my $scsi_fmt = {
1232     %drivedesc_base,
1233     %iothread_fmt,
1234     %queues_fmt,
1235     %scsiblock_fmt,
1236     %ssd_fmt,
1237     %wwn_fmt,
1238 };
1239 my $scsidesc = {
1240     optional => 1,
1241     type => 'string', format => $scsi_fmt,
1242     description => "Use volume as SCSI hard disk or CD-ROM (n is 0 to " . ($MAX_SCSI_DISKS - 1) . ").",
1243 };
1244 PVE::JSONSchema::register_standard_option("pve-qm-scsi", $scsidesc);
1245
1246 my $sata_fmt = {
1247     %drivedesc_base,
1248     %ssd_fmt,
1249     %wwn_fmt,
1250 };
1251 my $satadesc = {
1252     optional => 1,
1253     type => 'string', format => $sata_fmt,
1254     description => "Use volume as SATA hard disk or CD-ROM (n is 0 to " . ($MAX_SATA_DISKS - 1). ").",
1255 };
1256 PVE::JSONSchema::register_standard_option("pve-qm-sata", $satadesc);
1257
1258 my $virtio_fmt = {
1259     %drivedesc_base,
1260     %iothread_fmt,
1261 };
1262 my $virtiodesc = {
1263     optional => 1,
1264     type => 'string', format => $virtio_fmt,
1265     description => "Use volume as VIRTIO hard disk (n is 0 to " . ($MAX_VIRTIO_DISKS - 1) . ").",
1266 };
1267 PVE::JSONSchema::register_standard_option("pve-qm-virtio", $virtiodesc);
1268
1269 my $alldrive_fmt = {
1270     %drivedesc_base,
1271     %iothread_fmt,
1272     %model_fmt,
1273     %queues_fmt,
1274     %scsiblock_fmt,
1275     %ssd_fmt,
1276     %wwn_fmt,
1277 };
1278
1279 my $efidisk_fmt = {
1280     volume => { alias => 'file' },
1281     file => {
1282         type => 'string',
1283         format => 'pve-volume-id-or-qm-path',
1284         default_key => 1,
1285         format_description => 'volume',
1286         description => "The drive's backing volume.",
1287     },
1288     format => get_standard_option('pve-qm-image-format'),
1289     size => {
1290         type => 'string',
1291         format => 'disk-size',
1292         format_description => 'DiskSize',
1293         description => "Disk size. This is purely informational and has no effect.",
1294         optional => 1,
1295     },
1296 };
1297
1298 my $efidisk_desc = {
1299     optional => 1,
1300     type => 'string', format => $efidisk_fmt,
1301     description => "Configure a Disk for storing EFI vars",
1302 };
1303
1304 PVE::JSONSchema::register_standard_option("pve-qm-efidisk", $efidisk_desc);
1305
1306 my $usb_fmt = {
1307     host => {
1308         default_key => 1,
1309         type => 'string', format => 'pve-qm-usb-device',
1310         format_description => 'HOSTUSBDEVICE|spice',
1311         description => <<EODESCR,
1312 The Host USB device or port or the value 'spice'. HOSTUSBDEVICE syntax is:
1313
1314  'bus-port(.port)*' (decimal numbers) or
1315  'vendor_id:product_id' (hexadeciaml numbers) or
1316  'spice'
1317
1318 You can use the 'lsusb -t' command to list existing usb devices.
1319
1320 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1321
1322 The value 'spice' can be used to add a usb redirection devices for spice.
1323 EODESCR
1324     },
1325     usb3 => {
1326         optional => 1,
1327         type => 'boolean',
1328         description => "Specifies whether if given host option is a USB3 device or port.",
1329         default => 0,
1330     },
1331 };
1332
1333 my $usbdesc = {
1334     optional => 1,
1335     type => 'string', format => $usb_fmt,
1336     description => "Configure an USB device (n is 0 to 4).",
1337 };
1338 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
1339
1340 my $PCIRE = qr/[a-f0-9]{2}:[a-f0-9]{2}(?:\.[a-f0-9])?/;
1341 my $hostpci_fmt = {
1342     host => {
1343         default_key => 1,
1344         type => 'string',
1345         pattern => qr/$PCIRE(;$PCIRE)*/,
1346         format_description => 'HOSTPCIID[;HOSTPCIID2...]',
1347         description => <<EODESCR,
1348 Host PCI device pass through. The PCI ID of a host's PCI device or a list
1349 of PCI virtual functions of the host. HOSTPCIID syntax is:
1350
1351 'bus:dev.func' (hexadecimal numbers)
1352
1353 You can us the 'lspci' command to list existing PCI devices.
1354 EODESCR
1355     },
1356     rombar => {
1357         type => 'boolean',
1358         description =>  "Specify whether or not the device's ROM will be visible in the guest's memory map.",
1359         optional => 1,
1360         default => 1,
1361     },
1362     romfile => {
1363         type => 'string',
1364         pattern => '[^,;]+',
1365         format_description => 'string',
1366         description => "Custom pci device rom filename (must be located in /usr/share/kvm/).",
1367         optional => 1,
1368     },
1369     pcie => {
1370         type => 'boolean',
1371         description =>  "Choose the PCI-express bus (needs the 'q35' machine model).",
1372         optional => 1,
1373         default => 0,
1374     },
1375     'x-vga' => {
1376         type => 'boolean',
1377         description =>  "Enable vfio-vga device support.",
1378         optional => 1,
1379         default => 0,
1380     },
1381     'mdev' => {
1382         type => 'string',
1383         format_description => 'string',
1384         pattern => '[^/\.:]+',
1385         optional => 1,
1386         description => <<EODESCR
1387 The type of mediated device to use.
1388 An instance of this type will be created on startup of the VM and
1389 will be cleaned up when the VM stops.
1390 EODESCR
1391     }
1392 };
1393 PVE::JSONSchema::register_format('pve-qm-hostpci', $hostpci_fmt);
1394
1395 my $hostpcidesc = {
1396         optional => 1,
1397         type => 'string', format => 'pve-qm-hostpci',
1398         description => "Map host PCI devices into guest.",
1399         verbose_description =>  <<EODESCR,
1400 Map host PCI devices into guest.
1401
1402 NOTE: This option allows direct access to host hardware. So it is no longer
1403 possible to migrate such machines - use with special care.
1404
1405 CAUTION: Experimental! User reported problems with this option.
1406 EODESCR
1407 };
1408 PVE::JSONSchema::register_standard_option("pve-qm-hostpci", $hostpcidesc);
1409
1410 my $serialdesc = {
1411         optional => 1,
1412         type => 'string',
1413         pattern => '(/dev/.+|socket)',
1414         description =>  "Create a serial device inside the VM (n is 0 to 3)",
1415         verbose_description =>  <<EODESCR,
1416 Create a serial device inside the VM (n is 0 to 3), and pass through a
1417 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1418 host side (use 'qm terminal' to open a terminal connection).
1419
1420 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines - use with special care.
1421
1422 CAUTION: Experimental! User reported problems with this option.
1423 EODESCR
1424 };
1425
1426 my $paralleldesc= {
1427         optional => 1,
1428         type => 'string',
1429         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
1430         description =>  "Map host parallel devices (n is 0 to 2).",
1431         verbose_description =>  <<EODESCR,
1432 Map host parallel devices (n is 0 to 2).
1433
1434 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1435
1436 CAUTION: Experimental! User reported problems with this option.
1437 EODESCR
1438 };
1439
1440 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
1441     $confdesc->{"parallel$i"} = $paralleldesc;
1442 }
1443
1444 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
1445     $confdesc->{"serial$i"} = $serialdesc;
1446 }
1447
1448 for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
1449     $confdesc->{"hostpci$i"} = $hostpcidesc;
1450 }
1451
1452 for (my $i = 0; $i < $MAX_IDE_DISKS; $i++)  {
1453     $drivename_hash->{"ide$i"} = 1;
1454     $confdesc->{"ide$i"} = $idedesc;
1455 }
1456
1457 for (my $i = 0; $i < $MAX_SATA_DISKS; $i++)  {
1458     $drivename_hash->{"sata$i"} = 1;
1459     $confdesc->{"sata$i"} = $satadesc;
1460 }
1461
1462 for (my $i = 0; $i < $MAX_SCSI_DISKS; $i++)  {
1463     $drivename_hash->{"scsi$i"} = 1;
1464     $confdesc->{"scsi$i"} = $scsidesc ;
1465 }
1466
1467 for (my $i = 0; $i < $MAX_VIRTIO_DISKS; $i++)  {
1468     $drivename_hash->{"virtio$i"} = 1;
1469     $confdesc->{"virtio$i"} = $virtiodesc;
1470 }
1471
1472 $drivename_hash->{efidisk0} = 1;
1473 $confdesc->{efidisk0} = $efidisk_desc;
1474
1475 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
1476     $confdesc->{"usb$i"} = $usbdesc;
1477 }
1478
1479 my $unuseddesc = {
1480     optional => 1,
1481     type => 'string', format => 'pve-volume-id',
1482     description => "Reference to unused volumes. This is used internally, and should not be modified manually.",
1483 };
1484
1485 for (my $i = 0; $i < $MAX_UNUSED_DISKS; $i++)  {
1486     $confdesc->{"unused$i"} = $unuseddesc;
1487 }
1488
1489 my $kvm_api_version = 0;
1490
1491 sub kvm_version {
1492     return $kvm_api_version if $kvm_api_version;
1493
1494     open my $fh, '<', '/dev/kvm'
1495         or return undef;
1496
1497     # 0xae00 => KVM_GET_API_VERSION
1498     $kvm_api_version = ioctl($fh, 0xae00, 0);
1499
1500     return $kvm_api_version;
1501 }
1502
1503 my $kvm_user_version = {};
1504 my $kvm_mtime = {};
1505
1506 sub kvm_user_version {
1507     my ($binary) = @_;
1508
1509     $binary //= get_command_for_arch(get_host_arch()); # get the native arch by default
1510     my $st = stat($binary);
1511
1512     my $cachedmtime = $kvm_mtime->{$binary} // -1;
1513     return $kvm_user_version->{$binary} if $kvm_user_version->{$binary} &&
1514         $cachedmtime == $st->mtime;
1515
1516     $kvm_user_version->{$binary} = 'unknown';
1517     $kvm_mtime->{$binary} = $st->mtime;
1518
1519     my $code = sub {
1520         my $line = shift;
1521         if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1522             $kvm_user_version->{$binary} = $2;
1523         }
1524     };
1525
1526     eval { run_command([$binary, '--version'], outfunc => $code); };
1527     warn $@ if $@;
1528
1529     return $kvm_user_version->{$binary};
1530
1531 }
1532
1533 sub kernel_has_vhost_net {
1534     return -c '/dev/vhost-net';
1535 }
1536
1537 sub valid_drive_names {
1538     # order is important - used to autoselect boot disk
1539     return ((map { "ide$_" } (0 .. ($MAX_IDE_DISKS - 1))),
1540             (map { "scsi$_" } (0 .. ($MAX_SCSI_DISKS - 1))),
1541             (map { "virtio$_" } (0 .. ($MAX_VIRTIO_DISKS - 1))),
1542             (map { "sata$_" } (0 .. ($MAX_SATA_DISKS - 1))),
1543             'efidisk0');
1544 }
1545
1546 sub is_valid_drivename {
1547     my $dev = shift;
1548
1549     return defined($drivename_hash->{$dev});
1550 }
1551
1552 sub option_exists {
1553     my $key = shift;
1554     return defined($confdesc->{$key});
1555 }
1556
1557 my $cdrom_path;
1558 sub get_cdrom_path {
1559
1560     return  $cdrom_path if $cdrom_path;
1561
1562     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
1563     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
1564     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
1565 }
1566
1567 sub get_iso_path {
1568     my ($storecfg, $vmid, $cdrom) = @_;
1569
1570     if ($cdrom eq 'cdrom') {
1571         return get_cdrom_path();
1572     } elsif ($cdrom eq 'none') {
1573         return '';
1574     } elsif ($cdrom =~ m|^/|) {
1575         return $cdrom;
1576     } else {
1577         return PVE::Storage::path($storecfg, $cdrom);
1578     }
1579 }
1580
1581 # try to convert old style file names to volume IDs
1582 sub filename_to_volume_id {
1583     my ($vmid, $file, $media) = @_;
1584
1585      if (!($file eq 'none' || $file eq 'cdrom' ||
1586           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1587
1588         return undef if $file =~ m|/|;
1589
1590         if ($media && $media eq 'cdrom') {
1591             $file = "local:iso/$file";
1592         } else {
1593             $file = "local:$vmid/$file";
1594         }
1595     }
1596
1597     return $file;
1598 }
1599
1600 sub verify_media_type {
1601     my ($opt, $vtype, $media) = @_;
1602
1603     return if !$media;
1604
1605     my $etype;
1606     if ($media eq 'disk') {
1607         $etype = 'images';
1608     } elsif ($media eq 'cdrom') {
1609         $etype = 'iso';
1610     } else {
1611         die "internal error";
1612     }
1613
1614     return if ($vtype eq $etype);
1615
1616     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1617 }
1618
1619 sub cleanup_drive_path {
1620     my ($opt, $storecfg, $drive) = @_;
1621
1622     # try to convert filesystem paths to volume IDs
1623
1624     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1625         ($drive->{file} !~ m|^/dev/.+|) &&
1626         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1627         ($drive->{file} !~ m/^\d+$/)) {
1628         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1629         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"}) if !$vtype;
1630         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1631         verify_media_type($opt, $vtype, $drive->{media});
1632         $drive->{file} = $volid;
1633     }
1634
1635     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1636 }
1637
1638 sub parse_hotplug_features {
1639     my ($data) = @_;
1640
1641     my $res = {};
1642
1643     return $res if $data eq '0';
1644
1645     $data = $confdesc->{hotplug}->{default} if $data eq '1';
1646
1647     foreach my $feature (PVE::Tools::split_list($data)) {
1648         if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
1649             $res->{$1} = 1;
1650         } else {
1651             die "invalid hotplug feature '$feature'\n";
1652         }
1653     }
1654     return $res;
1655 }
1656
1657 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1658 sub pve_verify_hotplug_features {
1659     my ($value, $noerr) = @_;
1660
1661     return $value if parse_hotplug_features($value);
1662
1663     return undef if $noerr;
1664
1665     die "unable to parse hotplug option\n";
1666 }
1667
1668 # ideX = [volume=]volume-id[,media=d][,cyls=c,heads=h,secs=s[,trans=t]]
1669 #        [,snapshot=on|off][,cache=on|off][,format=f][,backup=yes|no]
1670 #        [,rerror=ignore|report|stop][,werror=enospc|ignore|report|stop]
1671 #        [,aio=native|threads][,discard=ignore|on][,detect_zeroes=on|off]
1672 #        [,iothread=on][,serial=serial][,model=model]
1673
1674 sub parse_drive {
1675     my ($key, $data) = @_;
1676
1677     my ($interface, $index);
1678
1679     if ($key =~ m/^([^\d]+)(\d+)$/) {
1680         $interface = $1;
1681         $index = $2;
1682     } else {
1683         return undef;
1684     }
1685
1686     my $desc = $key =~ /^unused\d+$/ ? $alldrive_fmt
1687                                      : $confdesc->{$key}->{format};
1688     if (!$desc) {
1689         warn "invalid drive key: $key\n";
1690         return undef;
1691     }
1692     my $res = eval { PVE::JSONSchema::parse_property_string($desc, $data) };
1693     return undef if !$res;
1694     $res->{interface} = $interface;
1695     $res->{index} = $index;
1696
1697     my $error = 0;
1698     foreach my $opt (qw(bps bps_rd bps_wr)) {
1699         if (my $bps = defined(delete $res->{$opt})) {
1700             if (defined($res->{"m$opt"})) {
1701                 warn "both $opt and m$opt specified\n";
1702                 ++$error;
1703                 next;
1704             }
1705             $res->{"m$opt"} = sprintf("%.3f", $bps / (1024*1024.0));
1706         }
1707     }
1708
1709     # can't use the schema's 'requires' because of the mbps* => bps* "transforming aliases"
1710     for my $requirement (
1711         [mbps_max => 'mbps'],
1712         [mbps_rd_max => 'mbps_rd'],
1713         [mbps_wr_max => 'mbps_wr'],
1714         [miops_max => 'miops'],
1715         [miops_rd_max => 'miops_rd'],
1716         [miops_wr_max => 'miops_wr'],
1717         [bps_max_length => 'mbps_max'],
1718         [bps_rd_max_length => 'mbps_rd_max'],
1719         [bps_wr_max_length => 'mbps_wr_max'],
1720         [iops_max_length => 'iops_max'],
1721         [iops_rd_max_length => 'iops_rd_max'],
1722         [iops_wr_max_length => 'iops_wr_max']) {
1723         my ($option, $requires) = @$requirement;
1724         if ($res->{$option} && !$res->{$requires}) {
1725             warn "$option requires $requires\n";
1726             ++$error;
1727         }
1728     }
1729
1730     return undef if $error;
1731
1732     return undef if $res->{mbps_rd} && $res->{mbps};
1733     return undef if $res->{mbps_wr} && $res->{mbps};
1734     return undef if $res->{iops_rd} && $res->{iops};
1735     return undef if $res->{iops_wr} && $res->{iops};
1736
1737     if ($res->{media} && ($res->{media} eq 'cdrom')) {
1738         return undef if $res->{snapshot} || $res->{trans} || $res->{format};
1739         return undef if $res->{heads} || $res->{secs} || $res->{cyls};
1740         return undef if $res->{interface} eq 'virtio';
1741     }
1742
1743     if (my $size = $res->{size}) {
1744         return undef if !defined($res->{size} = PVE::JSONSchema::parse_size($size));
1745     }
1746
1747     return $res;
1748 }
1749
1750 sub print_drive {
1751     my ($vmid, $drive) = @_;
1752     my $data = { %$drive };
1753     delete $data->{$_} for qw(index interface);
1754     return PVE::JSONSchema::print_property_string($data, $alldrive_fmt);
1755 }
1756
1757 sub scsi_inquiry {
1758     my($fh, $noerr) = @_;
1759
1760     my $SG_IO = 0x2285;
1761     my $SG_GET_VERSION_NUM = 0x2282;
1762
1763     my $versionbuf = "\x00" x 8;
1764     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1765     if (!$ret) {
1766         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1767         return undef;
1768     }
1769     my $version = unpack("I", $versionbuf);
1770     if ($version < 30000) {
1771         die "scsi generic interface too old\n"  if !$noerr;
1772         return undef;
1773     }
1774
1775     my $buf = "\x00" x 36;
1776     my $sensebuf = "\x00" x 8;
1777     my $cmd = pack("C x3 C x1", 0x12, 36);
1778
1779     # see /usr/include/scsi/sg.h
1780     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1781
1782     my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1783                       length($sensebuf), 0, length($buf), $buf,
1784                       $cmd, $sensebuf, 6000);
1785
1786     $ret = ioctl($fh, $SG_IO, $packet);
1787     if (!$ret) {
1788         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1789         return undef;
1790     }
1791
1792     my @res = unpack($sg_io_hdr_t, $packet);
1793     if ($res[17] || $res[18]) {
1794         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1795         return undef;
1796     }
1797
1798     my $res = {};
1799     (my $byte0, my $byte1, $res->{vendor},
1800      $res->{product}, $res->{revision}) = unpack("C C x6 A8 A16 A4", $buf);
1801
1802     $res->{removable} = $byte1 & 128 ? 1 : 0;
1803     $res->{type} = $byte0 & 31;
1804
1805     return $res;
1806 }
1807
1808 sub path_is_scsi {
1809     my ($path) = @_;
1810
1811     my $fh = IO::File->new("+<$path") || return undef;
1812     my $res = scsi_inquiry($fh, 1);
1813     close($fh);
1814
1815     return $res;
1816 }
1817
1818 sub print_tabletdevice_full {
1819     my ($conf, $arch) = @_;
1820
1821     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
1822
1823     # we use uhci for old VMs because tablet driver was buggy in older qemu
1824     my $usbbus;
1825     if (PVE::QemuServer::Machine::machine_type_is_q35($conf) || $arch eq 'aarch64') {
1826         $usbbus = 'ehci';
1827     } else {
1828         $usbbus = 'uhci';
1829     }
1830
1831     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1832 }
1833
1834 sub print_keyboarddevice_full {
1835     my ($conf, $arch, $machine) = @_;
1836
1837     return undef if $arch ne 'aarch64';
1838
1839     return "usb-kbd,id=keyboard,bus=ehci.0,port=2";
1840 }
1841
1842 sub print_drivedevice_full {
1843     my ($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type) = @_;
1844
1845     my $device = '';
1846     my $maxdev = 0;
1847
1848     if ($drive->{interface} eq 'virtio') {
1849         my $pciaddr = print_pci_addr("$drive->{interface}$drive->{index}", $bridges, $arch, $machine_type);
1850         $device = "virtio-blk-pci,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}$pciaddr";
1851         $device .= ",iothread=iothread-$drive->{interface}$drive->{index}" if $drive->{iothread};
1852     } elsif ($drive->{interface} eq 'scsi') {
1853
1854         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1855         my $unit = $drive->{index} % $maxdev;
1856         my $devicetype = 'hd';
1857         my $path = '';
1858         if (drive_is_cdrom($drive)) {
1859             $devicetype = 'cd';
1860         } else {
1861             if ($drive->{file} =~ m|^/|) {
1862                 $path = $drive->{file};
1863                 if (my $info = path_is_scsi($path)) {
1864                     if ($info->{type} == 0 && $drive->{scsiblock}) {
1865                         $devicetype = 'block';
1866                     } elsif ($info->{type} == 1) { # tape
1867                         $devicetype = 'generic';
1868                     }
1869                 }
1870             } else {
1871                  $path = PVE::Storage::path($storecfg, $drive->{file});
1872             }
1873
1874             # for compatibility only, we prefer scsi-hd (#2408, #2355, #2380)
1875             my $version = PVE::QemuServer::Machine::extract_version($machine_type) // kvm_user_version();
1876             if ($path =~ m/^iscsi\:\/\// &&
1877                !min_version($version, 4, 1)) {
1878                 $devicetype = 'generic';
1879             }
1880         }
1881
1882         if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)){
1883             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1884         } else {
1885             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0,lun=$drive->{index},drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1886         }
1887
1888         if ($drive->{ssd} && ($devicetype eq 'block' || $devicetype eq 'hd')) {
1889             $device .= ",rotation_rate=1";
1890         }
1891         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1892
1893     } elsif ($drive->{interface} eq 'ide' || $drive->{interface} eq 'sata') {
1894         my $maxdev = ($drive->{interface} eq 'sata') ? $MAX_SATA_DISKS : 2;
1895         my $controller = int($drive->{index} / $maxdev);
1896         my $unit = $drive->{index} % $maxdev;
1897         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1898
1899         $device = "ide-$devicetype";
1900         if ($drive->{interface} eq 'ide') {
1901             $device .= ",bus=ide.$controller,unit=$unit";
1902         } else {
1903             $device .= ",bus=ahci$controller.$unit";
1904         }
1905         $device .= ",drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1906
1907         if ($devicetype eq 'hd') {
1908             if (my $model = $drive->{model}) {
1909                 $model = URI::Escape::uri_unescape($model);
1910                 $device .= ",model=$model";
1911             }
1912             if ($drive->{ssd}) {
1913                 $device .= ",rotation_rate=1";
1914             }
1915         }
1916         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1917     } elsif ($drive->{interface} eq 'usb') {
1918         die "implement me";
1919         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1920     } else {
1921         die "unsupported interface type";
1922     }
1923
1924     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1925
1926     if (my $serial = $drive->{serial}) {
1927         $serial = URI::Escape::uri_unescape($serial);
1928         $device .= ",serial=$serial";
1929     }
1930
1931
1932     return $device;
1933 }
1934
1935 sub get_initiator_name {
1936     my $initiator;
1937
1938     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return undef;
1939     while (defined(my $line = <$fh>)) {
1940         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1941         $initiator = $1;
1942         last;
1943     }
1944     $fh->close();
1945
1946     return $initiator;
1947 }
1948
1949 sub print_drive_full {
1950     my ($storecfg, $vmid, $drive) = @_;
1951
1952     my $path;
1953     my $volid = $drive->{file};
1954     my $format;
1955
1956     if (drive_is_cdrom($drive)) {
1957         $path = get_iso_path($storecfg, $vmid, $volid);
1958     } else {
1959         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1960         if ($storeid) {
1961             $path = PVE::Storage::path($storecfg, $volid);
1962             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
1963             $format = qemu_img_format($scfg, $volname);
1964         } else {
1965             $path = $volid;
1966             $format = "raw";
1967         }
1968    }
1969
1970     my $opts = '';
1971     my @qemu_drive_options = qw(heads secs cyls trans media format cache rerror werror aio discard);
1972     foreach my $o (@qemu_drive_options) {
1973         $opts .= ",$o=$drive->{$o}" if defined($drive->{$o});
1974     }
1975
1976     # snapshot only accepts on|off
1977     if (defined($drive->{snapshot})) {
1978         my $v = $drive->{snapshot} ? 'on' : 'off';
1979         $opts .= ",snapshot=$v";
1980     }
1981
1982     foreach my $type (['', '-total'], [_rd => '-read'], [_wr => '-write']) {
1983         my ($dir, $qmpname) = @$type;
1984         if (my $v = $drive->{"mbps$dir"}) {
1985             $opts .= ",throttling.bps$qmpname=".int($v*1024*1024);
1986         }
1987         if (my $v = $drive->{"mbps${dir}_max"}) {
1988             $opts .= ",throttling.bps$qmpname-max=".int($v*1024*1024);
1989         }
1990         if (my $v = $drive->{"bps${dir}_max_length"}) {
1991             $opts .= ",throttling.bps$qmpname-max-length=$v";
1992         }
1993         if (my $v = $drive->{"iops${dir}"}) {
1994             $opts .= ",throttling.iops$qmpname=$v";
1995         }
1996         if (my $v = $drive->{"iops${dir}_max"}) {
1997             $opts .= ",throttling.iops$qmpname-max=$v";
1998         }
1999         if (my $v = $drive->{"iops${dir}_max_length"}) {
2000             $opts .= ",throttling.iops$qmpname-max-length=$v";
2001         }
2002     }
2003
2004     $opts .= ",format=$format" if $format && !$drive->{format};
2005
2006     my $cache_direct = 0;
2007
2008     if (my $cache = $drive->{cache}) {
2009         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
2010     } elsif (!drive_is_cdrom($drive)) {
2011         $opts .= ",cache=none";
2012         $cache_direct = 1;
2013     }
2014
2015     # aio native works only with O_DIRECT
2016     if (!$drive->{aio}) {
2017         if($cache_direct) {
2018             $opts .= ",aio=native";
2019         } else {
2020             $opts .= ",aio=threads";
2021         }
2022     }
2023
2024     if (!drive_is_cdrom($drive)) {
2025         my $detectzeroes;
2026         if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
2027             $detectzeroes = 'off';
2028         } elsif ($drive->{discard}) {
2029             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
2030         } else {
2031             # This used to be our default with discard not being specified:
2032             $detectzeroes = 'on';
2033         }
2034         $opts .= ",detect-zeroes=$detectzeroes" if $detectzeroes;
2035     }
2036
2037     my $pathinfo = $path ? "file=$path," : '';
2038
2039     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
2040 }
2041
2042 sub print_netdevice_full {
2043     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files, $arch, $machine_type) = @_;
2044
2045     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
2046
2047     my $device = $net->{model};
2048     if ($net->{model} eq 'virtio') {
2049          $device = 'virtio-net-pci';
2050      };
2051
2052     my $pciaddr = print_pci_addr("$netid", $bridges, $arch, $machine_type);
2053     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
2054     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
2055         #Consider we have N queues, the number of vectors needed is 2*N + 2 (plus one config interrupt and control vq)
2056         my $vectors = $net->{queues} * 2 + 2;
2057         $tmpstr .= ",vectors=$vectors,mq=on";
2058     }
2059     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
2060
2061     if ($use_old_bios_files) {
2062         my $romfile;
2063         if ($device eq 'virtio-net-pci') {
2064             $romfile = 'pxe-virtio.rom';
2065         } elsif ($device eq 'e1000') {
2066             $romfile = 'pxe-e1000.rom';
2067         } elsif ($device eq 'ne2k') {
2068             $romfile = 'pxe-ne2k_pci.rom';
2069         } elsif ($device eq 'pcnet') {
2070             $romfile = 'pxe-pcnet.rom';
2071         } elsif ($device eq 'rtl8139') {
2072             $romfile = 'pxe-rtl8139.rom';
2073         }
2074         $tmpstr .= ",romfile=$romfile" if $romfile;
2075     }
2076
2077     return $tmpstr;
2078 }
2079
2080 sub print_netdev_full {
2081     my ($vmid, $conf, $arch, $net, $netid, $hotplug) = @_;
2082
2083     my $i = '';
2084     if ($netid =~ m/^net(\d+)$/) {
2085         $i = int($1);
2086     }
2087
2088     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
2089
2090     my $ifname = "tap${vmid}i$i";
2091
2092     # kvm uses TUNSETIFF ioctl, and that limits ifname length
2093     die "interface name '$ifname' is too long (max 15 character)\n"
2094         if length($ifname) >= 16;
2095
2096     my $vhostparam = '';
2097     if (is_native($arch)) {
2098         $vhostparam = ',vhost=on' if kernel_has_vhost_net() && $net->{model} eq 'virtio';
2099     }
2100
2101     my $vmname = $conf->{name} || "vm$vmid";
2102
2103     my $netdev = "";
2104     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
2105
2106     if ($net->{bridge}) {
2107         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script,downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
2108     } else {
2109         $netdev = "type=user,id=$netid,hostname=$vmname";
2110     }
2111
2112     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
2113
2114     return $netdev;
2115 }
2116
2117
2118 sub print_cpu_device {
2119     my ($conf, $id) = @_;
2120
2121     my $kvm = $conf->{kvm} // 1;
2122     my $cpu = $kvm ? "kvm64" : "qemu64";
2123     if (my $cputype = $conf->{cpu}) {
2124         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
2125             or die "Cannot parse cpu description: $cputype\n";
2126         $cpu = $cpuconf->{cputype};
2127     }
2128
2129     my $cores = $conf->{cores} || 1;
2130
2131     my $current_core = ($id - 1) % $cores;
2132     my $current_socket = int(($id - 1 - $current_core)/$cores);
2133
2134     return "$cpu-x86_64-cpu,id=cpu$id,socket-id=$current_socket,core-id=$current_core,thread-id=0";
2135 }
2136
2137 my $vga_map = {
2138     'cirrus' => 'cirrus-vga',
2139     'std' => 'VGA',
2140     'vmware' => 'vmware-svga',
2141     'virtio' => 'virtio-vga',
2142 };
2143
2144 sub print_vga_device {
2145     my ($conf, $vga, $arch, $machine_version, $machine, $id, $qxlnum, $bridges) = @_;
2146
2147     my $type = $vga_map->{$vga->{type}};
2148     if ($arch eq 'aarch64' && defined($type) && $type eq 'virtio-vga') {
2149         $type = 'virtio-gpu';
2150     }
2151     my $vgamem_mb = $vga->{memory};
2152
2153     my $max_outputs = '';
2154     if ($qxlnum) {
2155         $type = $id ? 'qxl' : 'qxl-vga';
2156
2157         if (!$conf->{ostype} || $conf->{ostype} =~ m/^(?:l\d\d)|(?:other)$/) {
2158             # set max outputs so linux can have up to 4 qxl displays with one device
2159             if (min_version($machine_version, 4, 1)) {
2160                 $max_outputs = ",max_outputs=4";
2161             }
2162         }
2163     }
2164
2165     die "no devicetype for $vga->{type}\n" if !$type;
2166
2167     my $memory = "";
2168     if ($vgamem_mb) {
2169         if ($vga->{type} eq 'virtio') {
2170             my $bytes = PVE::Tools::convert_size($vgamem_mb, "mb" => "b");
2171             $memory = ",max_hostmem=$bytes";
2172         } elsif ($qxlnum) {
2173             # from https://www.spice-space.org/multiple-monitors.html
2174             $memory = ",vgamem_mb=$vga->{memory}";
2175             my $ram = $vgamem_mb * 4;
2176             my $vram = $vgamem_mb * 2;
2177             $memory .= ",ram_size_mb=$ram,vram_size_mb=$vram";
2178         } else {
2179             $memory = ",vgamem_mb=$vga->{memory}";
2180         }
2181     } elsif ($qxlnum && $id) {
2182         $memory = ",ram_size=67108864,vram_size=33554432";
2183     }
2184
2185     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
2186     my $vgaid = "vga" . ($id // '');
2187     my $pciaddr;
2188
2189     if ($q35 && $vgaid eq 'vga') {
2190         # the first display uses pcie.0 bus on q35 machines
2191         $pciaddr = print_pcie_addr($vgaid, $bridges, $arch, $machine);
2192     } else {
2193         $pciaddr = print_pci_addr($vgaid, $bridges, $arch, $machine);
2194     }
2195
2196     return "$type,id=${vgaid}${memory}${max_outputs}${pciaddr}";
2197 }
2198
2199 sub drive_is_cloudinit {
2200     my ($drive) = @_;
2201     return $drive->{file} =~ m@[:/]vm-\d+-cloudinit(?:\.$QEMU_FORMAT_RE)?$@;
2202 }
2203
2204 sub drive_is_cdrom {
2205     my ($drive, $exclude_cloudinit) = @_;
2206
2207     return 0 if $exclude_cloudinit && drive_is_cloudinit($drive);
2208
2209     return $drive && $drive->{media} && ($drive->{media} eq 'cdrom');
2210
2211 }
2212
2213 sub parse_number_sets {
2214     my ($set) = @_;
2215     my $res = [];
2216     foreach my $part (split(/;/, $set)) {
2217         if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
2218             die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
2219             push @$res, [ $1, $2 ];
2220         } else {
2221             die "invalid range: $part\n";
2222         }
2223     }
2224     return $res;
2225 }
2226
2227 sub parse_numa {
2228     my ($data) = @_;
2229
2230     my $res = PVE::JSONSchema::parse_property_string($numa_fmt, $data);
2231     $res->{cpus} = parse_number_sets($res->{cpus}) if defined($res->{cpus});
2232     $res->{hostnodes} = parse_number_sets($res->{hostnodes}) if defined($res->{hostnodes});
2233     return $res;
2234 }
2235
2236 sub parse_hostpci {
2237     my ($value) = @_;
2238
2239     return undef if !$value;
2240
2241     my $res = PVE::JSONSchema::parse_property_string($hostpci_fmt, $value);
2242
2243     my @idlist = split(/;/, $res->{host});
2244     delete $res->{host};
2245     foreach my $id (@idlist) {
2246         if ($id =~ m/\./) { # full id 00:00.1
2247             push @{$res->{pciid}}, {
2248                 id => $id,
2249             };
2250         } else { # partial id 00:00
2251             $res->{pciid} = PVE::SysFSTools::lspci($id);
2252         }
2253     }
2254     return $res;
2255 }
2256
2257 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
2258 sub parse_net {
2259     my ($data) = @_;
2260
2261     my $res = eval { PVE::JSONSchema::parse_property_string($net_fmt, $data) };
2262     if ($@) {
2263         warn $@;
2264         return undef;
2265     }
2266     if (!defined($res->{macaddr})) {
2267         my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
2268         $res->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix});
2269     }
2270     return $res;
2271 }
2272
2273 # ipconfigX ip=cidr,gw=ip,ip6=cidr,gw6=ip
2274 sub parse_ipconfig {
2275     my ($data) = @_;
2276
2277     my $res = eval { PVE::JSONSchema::parse_property_string($ipconfig_fmt, $data) };
2278     if ($@) {
2279         warn $@;
2280         return undef;
2281     }
2282
2283     if ($res->{gw} && !$res->{ip}) {
2284         warn 'gateway specified without specifying an IP address';
2285         return undef;
2286     }
2287     if ($res->{gw6} && !$res->{ip6}) {
2288         warn 'IPv6 gateway specified without specifying an IPv6 address';
2289         return undef;
2290     }
2291     if ($res->{gw} && $res->{ip} eq 'dhcp') {
2292         warn 'gateway specified together with DHCP';
2293         return undef;
2294     }
2295     if ($res->{gw6} && $res->{ip6} !~ /^$IPV6RE/) {
2296         # gw6 + auto/dhcp
2297         warn "IPv6 gateway specified together with $res->{ip6} address";
2298         return undef;
2299     }
2300
2301     if (!$res->{ip} && !$res->{ip6}) {
2302         return { ip => 'dhcp', ip6 => 'dhcp' };
2303     }
2304
2305     return $res;
2306 }
2307
2308 sub print_net {
2309     my $net = shift;
2310
2311     return PVE::JSONSchema::print_property_string($net, $net_fmt);
2312 }
2313
2314 sub add_random_macs {
2315     my ($settings) = @_;
2316
2317     foreach my $opt (keys %$settings) {
2318         next if $opt !~ m/^net(\d+)$/;
2319         my $net = parse_net($settings->{$opt});
2320         next if !$net;
2321         $settings->{$opt} = print_net($net);
2322     }
2323 }
2324
2325 sub vm_is_volid_owner {
2326     my ($storecfg, $vmid, $volid) = @_;
2327
2328     if ($volid !~  m|^/|) {
2329         my ($path, $owner);
2330         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
2331         if ($owner && ($owner == $vmid)) {
2332             return 1;
2333         }
2334     }
2335
2336     return undef;
2337 }
2338
2339 sub vmconfig_register_unused_drive {
2340     my ($storecfg, $vmid, $conf, $drive) = @_;
2341
2342     if (drive_is_cloudinit($drive)) {
2343         eval { PVE::Storage::vdisk_free($storecfg, $drive->{file}) };
2344         warn $@ if $@;
2345     } elsif (!drive_is_cdrom($drive)) {
2346         my $volid = $drive->{file};
2347         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
2348             PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
2349         }
2350     }
2351 }
2352
2353 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str][,base64=bool]
2354 my $smbios1_fmt = {
2355     uuid => {
2356         type => 'string',
2357         pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
2358         format_description => 'UUID',
2359         description => "Set SMBIOS1 UUID.",
2360         optional => 1,
2361     },
2362     version => {
2363         type => 'string',
2364         pattern => '[A-Za-z0-9+\/]+={0,2}',
2365         format_description => 'Base64 encoded string',
2366         description => "Set SMBIOS1 version.",
2367         optional => 1,
2368     },
2369     serial => {
2370         type => 'string',
2371         pattern => '[A-Za-z0-9+\/]+={0,2}',
2372         format_description => 'Base64 encoded string',
2373         description => "Set SMBIOS1 serial number.",
2374         optional => 1,
2375     },
2376     manufacturer => {
2377         type => 'string',
2378         pattern => '[A-Za-z0-9+\/]+={0,2}',
2379         format_description => 'Base64 encoded string',
2380         description => "Set SMBIOS1 manufacturer.",
2381         optional => 1,
2382     },
2383     product => {
2384         type => 'string',
2385         pattern => '[A-Za-z0-9+\/]+={0,2}',
2386         format_description => 'Base64 encoded string',
2387         description => "Set SMBIOS1 product ID.",
2388         optional => 1,
2389     },
2390     sku => {
2391         type => 'string',
2392         pattern => '[A-Za-z0-9+\/]+={0,2}',
2393         format_description => 'Base64 encoded string',
2394         description => "Set SMBIOS1 SKU string.",
2395         optional => 1,
2396     },
2397     family => {
2398         type => 'string',
2399         pattern => '[A-Za-z0-9+\/]+={0,2}',
2400         format_description => 'Base64 encoded string',
2401         description => "Set SMBIOS1 family string.",
2402         optional => 1,
2403     },
2404     base64 => {
2405         type => 'boolean',
2406         description => 'Flag to indicate that the SMBIOS values are base64 encoded',
2407         optional => 1,
2408     },
2409 };
2410
2411 sub parse_smbios1 {
2412     my ($data) = @_;
2413
2414     my $res = eval { PVE::JSONSchema::parse_property_string($smbios1_fmt, $data) };
2415     warn $@ if $@;
2416     return $res;
2417 }
2418
2419 sub print_smbios1 {
2420     my ($smbios1) = @_;
2421     return PVE::JSONSchema::print_property_string($smbios1, $smbios1_fmt);
2422 }
2423
2424 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_fmt);
2425
2426 PVE::JSONSchema::register_format('pve-qm-bootdisk', \&verify_bootdisk);
2427 sub verify_bootdisk {
2428     my ($value, $noerr) = @_;
2429
2430     return $value if is_valid_drivename($value);
2431
2432     return undef if $noerr;
2433
2434     die "invalid boot disk '$value'\n";
2435 }
2436
2437 sub parse_watchdog {
2438     my ($value) = @_;
2439
2440     return undef if !$value;
2441
2442     my $res = eval { PVE::JSONSchema::parse_property_string($watchdog_fmt, $value) };
2443     warn $@ if $@;
2444     return $res;
2445 }
2446
2447 sub parse_guest_agent {
2448     my ($value) = @_;
2449
2450     return {} if !defined($value->{agent});
2451
2452     my $res = eval { PVE::JSONSchema::parse_property_string($agent_fmt, $value->{agent}) };
2453     warn $@ if $@;
2454
2455     # if the agent is disabled ignore the other potentially set properties
2456     return {} if !$res->{enabled};
2457     return $res;
2458 }
2459
2460 sub parse_vga {
2461     my ($value) = @_;
2462
2463     return {} if !$value;
2464     my $res = eval { PVE::JSONSchema::parse_property_string($vga_fmt, $value) };
2465     warn $@ if $@;
2466     return $res;
2467 }
2468
2469 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
2470 sub verify_usb_device {
2471     my ($value, $noerr) = @_;
2472
2473     return $value if parse_usb_device($value);
2474
2475     return undef if $noerr;
2476
2477     die "unable to parse usb device\n";
2478 }
2479
2480 # add JSON properties for create and set function
2481 sub json_config_properties {
2482     my $prop = shift;
2483
2484     foreach my $opt (keys %$confdesc) {
2485         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate' || $opt eq 'runningmachine';
2486         $prop->{$opt} = $confdesc->{$opt};
2487     }
2488
2489     return $prop;
2490 }
2491
2492 # return copy of $confdesc_cloudinit to generate documentation
2493 sub cloudinit_config_properties {
2494
2495     return dclone($confdesc_cloudinit);
2496 }
2497
2498 sub check_type {
2499     my ($key, $value) = @_;
2500
2501     die "unknown setting '$key'\n" if !$confdesc->{$key};
2502
2503     my $type = $confdesc->{$key}->{type};
2504
2505     if (!defined($value)) {
2506         die "got undefined value\n";
2507     }
2508
2509     if ($value =~ m/[\n\r]/) {
2510         die "property contains a line feed\n";
2511     }
2512
2513     if ($type eq 'boolean') {
2514         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
2515         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
2516         die "type check ('boolean') failed - got '$value'\n";
2517     } elsif ($type eq 'integer') {
2518         return int($1) if $value =~ m/^(\d+)$/;
2519         die "type check ('integer') failed - got '$value'\n";
2520     } elsif ($type eq 'number') {
2521         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
2522         die "type check ('number') failed - got '$value'\n";
2523     } elsif ($type eq 'string') {
2524         if (my $fmt = $confdesc->{$key}->{format}) {
2525             PVE::JSONSchema::check_format($fmt, $value);
2526             return $value;
2527         }
2528         $value =~ s/^\"(.*)\"$/$1/;
2529         return $value;
2530     } else {
2531         die "internal error"
2532     }
2533 }
2534
2535 sub destroy_vm {
2536     my ($storecfg, $vmid, $skiplock, $replacement_conf) = @_;
2537
2538     my $conf = PVE::QemuConfig->load_config($vmid);
2539
2540     PVE::QemuConfig->check_lock($conf) if !$skiplock;
2541
2542     if ($conf->{template}) {
2543         # check if any base image is still used by a linked clone
2544         foreach_drive($conf, sub {
2545                 my ($ds, $drive) = @_;
2546                 return if drive_is_cdrom($drive);
2547
2548                 my $volid = $drive->{file};
2549                 return if !$volid || $volid =~ m|^/|;
2550
2551                 die "base volume '$volid' is still in use by linked cloned\n"
2552                     if PVE::Storage::volume_is_base_and_used($storecfg, $volid);
2553
2554         });
2555     }
2556
2557     # only remove disks owned by this VM
2558     foreach_drive($conf, sub {
2559         my ($ds, $drive) = @_;
2560         return if drive_is_cdrom($drive, 1);
2561
2562         my $volid = $drive->{file};
2563         return if !$volid || $volid =~ m|^/|;
2564
2565         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
2566         return if !$path || !$owner || ($owner != $vmid);
2567
2568         eval { PVE::Storage::vdisk_free($storecfg, $volid) };
2569         warn "Could not remove disk '$volid', check manually: $@" if $@;
2570     });
2571
2572     # also remove unused disk
2573     my $vmdisks = PVE::Storage::vdisk_list($storecfg, undef, $vmid);
2574     PVE::Storage::foreach_volid($vmdisks, sub {
2575         my ($volid, $sid, $volname, $d) = @_;
2576         eval { PVE::Storage::vdisk_free($storecfg, $volid) };
2577         warn $@ if $@;
2578     });
2579
2580     if (defined $replacement_conf) {
2581         PVE::QemuConfig->write_config($vmid, $replacement_conf);
2582     } else {
2583         PVE::QemuConfig->destroy_config($vmid);
2584     }
2585 }
2586
2587 sub parse_vm_config {
2588     my ($filename, $raw) = @_;
2589
2590     return undef if !defined($raw);
2591
2592     my $res = {
2593         digest => Digest::SHA::sha1_hex($raw),
2594         snapshots => {},
2595         pending => {},
2596     };
2597
2598     $filename =~ m|/qemu-server/(\d+)\.conf$|
2599         || die "got strange filename '$filename'";
2600
2601     my $vmid = $1;
2602
2603     my $conf = $res;
2604     my $descr;
2605     my $section = '';
2606
2607     my @lines = split(/\n/, $raw);
2608     foreach my $line (@lines) {
2609         next if $line =~ m/^\s*$/;
2610
2611         if ($line =~ m/^\[PENDING\]\s*$/i) {
2612             $section = 'pending';
2613             if (defined($descr)) {
2614                 $descr =~ s/\s+$//;
2615                 $conf->{description} = $descr;
2616             }
2617             $descr = undef;
2618             $conf = $res->{$section} = {};
2619             next;
2620
2621         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2622             $section = $1;
2623             if (defined($descr)) {
2624                 $descr =~ s/\s+$//;
2625                 $conf->{description} = $descr;
2626             }
2627             $descr = undef;
2628             $conf = $res->{snapshots}->{$section} = {};
2629             next;
2630         }
2631
2632         if ($line =~ m/^\#(.*)\s*$/) {
2633             $descr = '' if !defined($descr);
2634             $descr .= PVE::Tools::decode_text($1) . "\n";
2635             next;
2636         }
2637
2638         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2639             $descr = '' if !defined($descr);
2640             $descr .= PVE::Tools::decode_text($2);
2641         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2642             $conf->{snapstate} = $1;
2643         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2644             my $key = $1;
2645             my $value = $2;
2646             $conf->{$key} = $value;
2647         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2648             my $value = $1;
2649             if ($section eq 'pending') {
2650                 $conf->{delete} = $value; # we parse this later
2651             } else {
2652                 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
2653             }
2654         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(.+?)\s*$/) {
2655             my $key = $1;
2656             my $value = $2;
2657             eval { $value = check_type($key, $value); };
2658             if ($@) {
2659                 warn "vm $vmid - unable to parse value of '$key' - $@";
2660             } else {
2661                 $key = 'ide2' if $key eq 'cdrom';
2662                 my $fmt = $confdesc->{$key}->{format};
2663                 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2664                     my $v = parse_drive($key, $value);
2665                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2666                         $v->{file} = $volid;
2667                         $value = print_drive($vmid, $v);
2668                     } else {
2669                         warn "vm $vmid - unable to parse value of '$key'\n";
2670                         next;
2671                     }
2672                 }
2673
2674                 $conf->{$key} = $value;
2675             }
2676         }
2677     }
2678
2679     if (defined($descr)) {
2680         $descr =~ s/\s+$//;
2681         $conf->{description} = $descr;
2682     }
2683     delete $res->{snapstate}; # just to be sure
2684
2685     return $res;
2686 }
2687
2688 sub write_vm_config {
2689     my ($filename, $conf) = @_;
2690
2691     delete $conf->{snapstate}; # just to be sure
2692
2693     if ($conf->{cdrom}) {
2694         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2695         $conf->{ide2} = $conf->{cdrom};
2696         delete $conf->{cdrom};
2697     }
2698
2699     # we do not use 'smp' any longer
2700     if ($conf->{sockets}) {
2701         delete $conf->{smp};
2702     } elsif ($conf->{smp}) {
2703         $conf->{sockets} = $conf->{smp};
2704         delete $conf->{cores};
2705         delete $conf->{smp};
2706     }
2707
2708     my $used_volids = {};
2709
2710     my $cleanup_config = sub {
2711         my ($cref, $pending, $snapname) = @_;
2712
2713         foreach my $key (keys %$cref) {
2714             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2715                 $key eq 'snapstate' || $key eq 'pending';
2716             my $value = $cref->{$key};
2717             if ($key eq 'delete') {
2718                 die "propertry 'delete' is only allowed in [PENDING]\n"
2719                     if !$pending;
2720                 # fixme: check syntax?
2721                 next;
2722             }
2723             eval { $value = check_type($key, $value); };
2724             die "unable to parse value of '$key' - $@" if $@;
2725
2726             $cref->{$key} = $value;
2727
2728             if (!$snapname && is_valid_drivename($key)) {
2729                 my $drive = parse_drive($key, $value);
2730                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2731             }
2732         }
2733     };
2734
2735     &$cleanup_config($conf);
2736
2737     &$cleanup_config($conf->{pending}, 1);
2738
2739     foreach my $snapname (keys %{$conf->{snapshots}}) {
2740         die "internal error" if $snapname eq 'pending';
2741         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2742     }
2743
2744     # remove 'unusedX' settings if we re-add a volume
2745     foreach my $key (keys %$conf) {
2746         my $value = $conf->{$key};
2747         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2748             delete $conf->{$key};
2749         }
2750     }
2751
2752     my $generate_raw_config = sub {
2753         my ($conf, $pending) = @_;
2754
2755         my $raw = '';
2756
2757         # add description as comment to top of file
2758         if (defined(my $descr = $conf->{description})) {
2759             if ($descr) {
2760                 foreach my $cl (split(/\n/, $descr)) {
2761                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2762                 }
2763             } else {
2764                 $raw .= "#\n" if $pending;
2765             }
2766         }
2767
2768         foreach my $key (sort keys %$conf) {
2769             next if $key eq 'digest' || $key eq 'description' || $key eq 'pending' || $key eq 'snapshots';
2770             $raw .= "$key: $conf->{$key}\n";
2771         }
2772         return $raw;
2773     };
2774
2775     my $raw = &$generate_raw_config($conf);
2776
2777     if (scalar(keys %{$conf->{pending}})){
2778         $raw .= "\n[PENDING]\n";
2779         $raw .= &$generate_raw_config($conf->{pending}, 1);
2780     }
2781
2782     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2783         $raw .= "\n[$snapname]\n";
2784         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2785     }
2786
2787     return $raw;
2788 }
2789
2790 sub load_defaults {
2791
2792     my $res = {};
2793
2794     # we use static defaults from our JSON schema configuration
2795     foreach my $key (keys %$confdesc) {
2796         if (defined(my $default = $confdesc->{$key}->{default})) {
2797             $res->{$key} = $default;
2798         }
2799     }
2800
2801     return $res;
2802 }
2803
2804 sub config_list {
2805     my $vmlist = PVE::Cluster::get_vmlist();
2806     my $res = {};
2807     return $res if !$vmlist || !$vmlist->{ids};
2808     my $ids = $vmlist->{ids};
2809
2810     foreach my $vmid (keys %$ids) {
2811         my $d = $ids->{$vmid};
2812         next if !$d->{node} || $d->{node} ne $nodename;
2813         next if !$d->{type} || $d->{type} ne 'qemu';
2814         $res->{$vmid}->{exists} = 1;
2815     }
2816     return $res;
2817 }
2818
2819 # test if VM uses local resources (to prevent migration)
2820 sub check_local_resources {
2821     my ($conf, $noerr) = @_;
2822
2823     my @loc_res = ();
2824
2825     push @loc_res, "hostusb" if $conf->{hostusb}; # old syntax
2826     push @loc_res, "hostpci" if $conf->{hostpci}; # old syntax
2827
2828     push @loc_res, "ivshmem" if $conf->{ivshmem};
2829
2830     foreach my $k (keys %$conf) {
2831         next if $k =~ m/^usb/ && ($conf->{$k} =~ m/^spice(?![^,])/);
2832         # sockets are safe: they will recreated be on the target side post-migrate
2833         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2834         push @loc_res, $k if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2835     }
2836
2837     die "VM uses local resources\n" if scalar @loc_res && !$noerr;
2838
2839     return \@loc_res;
2840 }
2841
2842 # check if used storages are available on all nodes (use by migrate)
2843 sub check_storage_availability {
2844     my ($storecfg, $conf, $node) = @_;
2845
2846     foreach_drive($conf, sub {
2847         my ($ds, $drive) = @_;
2848
2849         my $volid = $drive->{file};
2850         return if !$volid;
2851
2852         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2853         return if !$sid;
2854
2855         # check if storage is available on both nodes
2856         my $scfg = PVE::Storage::storage_check_node($storecfg, $sid);
2857         PVE::Storage::storage_check_node($storecfg, $sid, $node);
2858    });
2859 }
2860
2861 # list nodes where all VM images are available (used by has_feature API)
2862 sub shared_nodes {
2863     my ($conf, $storecfg) = @_;
2864
2865     my $nodelist = PVE::Cluster::get_nodelist();
2866     my $nodehash = { map { $_ => 1 } @$nodelist };
2867     my $nodename = PVE::INotify::nodename();
2868
2869     foreach_drive($conf, sub {
2870         my ($ds, $drive) = @_;
2871
2872         my $volid = $drive->{file};
2873         return if !$volid;
2874
2875         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2876         if ($storeid) {
2877             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2878             if ($scfg->{disable}) {
2879                 $nodehash = {};
2880             } elsif (my $avail = $scfg->{nodes}) {
2881                 foreach my $node (keys %$nodehash) {
2882                     delete $nodehash->{$node} if !$avail->{$node};
2883                 }
2884             } elsif (!$scfg->{shared}) {
2885                 foreach my $node (keys %$nodehash) {
2886                     delete $nodehash->{$node} if $node ne $nodename
2887                 }
2888             }
2889         }
2890     });
2891
2892     return $nodehash
2893 }
2894
2895 sub check_local_storage_availability {
2896     my ($conf, $storecfg) = @_;
2897
2898     my $nodelist = PVE::Cluster::get_nodelist();
2899     my $nodehash = { map { $_ => {} } @$nodelist };
2900
2901     foreach_drive($conf, sub {
2902         my ($ds, $drive) = @_;
2903
2904         my $volid = $drive->{file};
2905         return if !$volid;
2906
2907         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2908         if ($storeid) {
2909             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2910
2911             if ($scfg->{disable}) {
2912                 foreach my $node (keys %$nodehash) {
2913                     $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2914                 }
2915             } elsif (my $avail = $scfg->{nodes}) {
2916                 foreach my $node (keys %$nodehash) {
2917                     if (!$avail->{$node}) {
2918                         $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2919                     }
2920                 }
2921             }
2922         }
2923     });
2924
2925     foreach my $node (values %$nodehash) {
2926         if (my $unavail = $node->{unavailable_storages}) {
2927             $node->{unavailable_storages} = [ sort keys %$unavail ];
2928         }
2929     }
2930
2931     return $nodehash
2932 }
2933
2934 # Compat only, use assert_config_exists_on_node and vm_running_locally where possible
2935 sub check_running {
2936     my ($vmid, $nocheck, $node) = @_;
2937
2938     PVE::QemuConfig::assert_config_exists_on_node($vmid, $node) if !$nocheck;
2939     return PVE::QemuServer::Helpers::vm_running_locally($vmid);
2940 }
2941
2942 sub vzlist {
2943
2944     my $vzlist = config_list();
2945
2946     my $fd = IO::Dir->new($PVE::QemuServer::Helpers::var_run_tmpdir) || return $vzlist;
2947
2948     while (defined(my $de = $fd->read)) {
2949         next if $de !~ m/^(\d+)\.pid$/;
2950         my $vmid = $1;
2951         next if !defined($vzlist->{$vmid});
2952         if (my $pid = check_running($vmid)) {
2953             $vzlist->{$vmid}->{pid} = $pid;
2954         }
2955     }
2956
2957     return $vzlist;
2958 }
2959
2960 sub disksize {
2961     my ($storecfg, $conf) = @_;
2962
2963     my $bootdisk = $conf->{bootdisk};
2964     return undef if !$bootdisk;
2965     return undef if !is_valid_drivename($bootdisk);
2966
2967     return undef if !$conf->{$bootdisk};
2968
2969     my $drive = parse_drive($bootdisk, $conf->{$bootdisk});
2970     return undef if !defined($drive);
2971
2972     return undef if drive_is_cdrom($drive);
2973
2974     my $volid = $drive->{file};
2975     return undef if !$volid;
2976
2977     return $drive->{size};
2978 }
2979
2980 our $vmstatus_return_properties = {
2981     vmid => get_standard_option('pve-vmid'),
2982     status => {
2983         description => "Qemu process status.",
2984         type => 'string',
2985         enum => ['stopped', 'running'],
2986     },
2987     maxmem => {
2988         description => "Maximum memory in bytes.",
2989         type => 'integer',
2990         optional => 1,
2991         renderer => 'bytes',
2992     },
2993     maxdisk => {
2994         description => "Root disk size in bytes.",
2995         type => 'integer',
2996         optional => 1,
2997         renderer => 'bytes',
2998     },
2999     name => {
3000         description => "VM name.",
3001         type => 'string',
3002         optional => 1,
3003     },
3004     qmpstatus => {
3005         description => "Qemu QMP agent status.",
3006         type => 'string',
3007         optional => 1,
3008     },
3009     pid => {
3010         description => "PID of running qemu process.",
3011         type => 'integer',
3012         optional => 1,
3013     },
3014     uptime => {
3015         description => "Uptime.",
3016         type => 'integer',
3017         optional => 1,
3018         renderer => 'duration',
3019     },
3020     cpus => {
3021         description => "Maximum usable CPUs.",
3022         type => 'number',
3023         optional => 1,
3024     },
3025     lock => {
3026         description => "The current config lock, if any.",
3027         type => 'string',
3028         optional => 1,
3029     }
3030 };
3031
3032 my $last_proc_pid_stat;
3033
3034 # get VM status information
3035 # This must be fast and should not block ($full == false)
3036 # We only query KVM using QMP if $full == true (this can be slow)
3037 sub vmstatus {
3038     my ($opt_vmid, $full) = @_;
3039
3040     my $res = {};
3041
3042     my $storecfg = PVE::Storage::config();
3043
3044     my $list = vzlist();
3045     my $defaults = load_defaults();
3046
3047     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
3048
3049     my $cpucount = $cpuinfo->{cpus} || 1;
3050
3051     foreach my $vmid (keys %$list) {
3052         next if $opt_vmid && ($vmid ne $opt_vmid);
3053
3054         my $conf = PVE::QemuConfig->load_config($vmid);
3055
3056         my $d = { vmid => $vmid };
3057         $d->{pid} = $list->{$vmid}->{pid};
3058
3059         # fixme: better status?
3060         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
3061
3062         my $size = disksize($storecfg, $conf);
3063         if (defined($size)) {
3064             $d->{disk} = 0; # no info available
3065             $d->{maxdisk} = $size;
3066         } else {
3067             $d->{disk} = 0;
3068             $d->{maxdisk} = 0;
3069         }
3070
3071         $d->{cpus} = ($conf->{sockets} || $defaults->{sockets})
3072             * ($conf->{cores} || $defaults->{cores});
3073         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
3074         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
3075
3076         $d->{name} = $conf->{name} || "VM $vmid";
3077         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024)
3078             : $defaults->{memory}*(1024*1024);
3079
3080         if ($conf->{balloon}) {
3081             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
3082             $d->{shares} = defined($conf->{shares}) ? $conf->{shares}
3083                 : $defaults->{shares};
3084         }
3085
3086         $d->{uptime} = 0;
3087         $d->{cpu} = 0;
3088         $d->{mem} = 0;
3089
3090         $d->{netout} = 0;
3091         $d->{netin} = 0;
3092
3093         $d->{diskread} = 0;
3094         $d->{diskwrite} = 0;
3095
3096         $d->{template} = PVE::QemuConfig->is_template($conf);
3097
3098         $d->{serial} = 1 if conf_has_serial($conf);
3099         $d->{lock} = $conf->{lock} if $conf->{lock};
3100
3101         $res->{$vmid} = $d;
3102     }
3103
3104     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
3105     foreach my $dev (keys %$netdev) {
3106         next if $dev !~ m/^tap([1-9]\d*)i/;
3107         my $vmid = $1;
3108         my $d = $res->{$vmid};
3109         next if !$d;
3110
3111         $d->{netout} += $netdev->{$dev}->{receive};
3112         $d->{netin} += $netdev->{$dev}->{transmit};
3113
3114         if ($full) {
3115             $d->{nics}->{$dev}->{netout} = $netdev->{$dev}->{receive};
3116             $d->{nics}->{$dev}->{netin} = $netdev->{$dev}->{transmit};
3117         }
3118
3119     }
3120
3121     my $ctime = gettimeofday;
3122
3123     foreach my $vmid (keys %$list) {
3124
3125         my $d = $res->{$vmid};
3126         my $pid = $d->{pid};
3127         next if !$pid;
3128
3129         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
3130         next if !$pstat; # not running
3131
3132         my $used = $pstat->{utime} + $pstat->{stime};
3133
3134         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
3135
3136         if ($pstat->{vsize}) {
3137             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
3138         }
3139
3140         my $old = $last_proc_pid_stat->{$pid};
3141         if (!$old) {
3142             $last_proc_pid_stat->{$pid} = {
3143                 time => $ctime,
3144                 used => $used,
3145                 cpu => 0,
3146             };
3147             next;
3148         }
3149
3150         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
3151
3152         if ($dtime > 1000) {
3153             my $dutime = $used -  $old->{used};
3154
3155             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
3156             $last_proc_pid_stat->{$pid} = {
3157                 time => $ctime,
3158                 used => $used,
3159                 cpu => $d->{cpu},
3160             };
3161         } else {
3162             $d->{cpu} = $old->{cpu};
3163         }
3164     }
3165
3166     return $res if !$full;
3167
3168     my $qmpclient = PVE::QMPClient->new();
3169
3170     my $ballooncb = sub {
3171         my ($vmid, $resp) = @_;
3172
3173         my $info = $resp->{'return'};
3174         return if !$info->{max_mem};
3175
3176         my $d = $res->{$vmid};
3177
3178         # use memory assigned to VM
3179         $d->{maxmem} = $info->{max_mem};
3180         $d->{balloon} = $info->{actual};
3181
3182         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
3183             $d->{mem} = $info->{total_mem} - $info->{free_mem};
3184             $d->{freemem} = $info->{free_mem};
3185         }
3186
3187         $d->{ballooninfo} = $info;
3188     };
3189
3190     my $blockstatscb = sub {
3191         my ($vmid, $resp) = @_;
3192         my $data = $resp->{'return'} || [];
3193         my $totalrdbytes = 0;
3194         my $totalwrbytes = 0;
3195
3196         for my $blockstat (@$data) {
3197             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
3198             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
3199
3200             $blockstat->{device} =~ s/drive-//;
3201             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
3202         }
3203         $res->{$vmid}->{diskread} = $totalrdbytes;
3204         $res->{$vmid}->{diskwrite} = $totalwrbytes;
3205     };
3206
3207     my $statuscb = sub {
3208         my ($vmid, $resp) = @_;
3209
3210         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
3211         # this fails if ballon driver is not loaded, so this must be
3212         # the last commnand (following command are aborted if this fails).
3213         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
3214
3215         my $status = 'unknown';
3216         if (!defined($status = $resp->{'return'}->{status})) {
3217             warn "unable to get VM status\n";
3218             return;
3219         }
3220
3221         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
3222     };
3223
3224     foreach my $vmid (keys %$list) {
3225         next if $opt_vmid && ($vmid ne $opt_vmid);
3226         next if !$res->{$vmid}->{pid}; # not running
3227         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
3228     }
3229
3230     $qmpclient->queue_execute(undef, 2);
3231
3232     foreach my $vmid (keys %$list) {
3233         next if $opt_vmid && ($vmid ne $opt_vmid);
3234         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
3235     }
3236
3237     return $res;
3238 }
3239
3240 sub foreach_drive {
3241     my ($conf, $func, @param) = @_;
3242
3243     foreach my $ds (valid_drive_names()) {
3244         next if !defined($conf->{$ds});
3245
3246         my $drive = parse_drive($ds, $conf->{$ds});
3247         next if !$drive;
3248
3249         &$func($ds, $drive, @param);
3250     }
3251 }
3252
3253 sub foreach_volid {
3254     my ($conf, $func, @param) = @_;
3255
3256     my $volhash = {};
3257
3258     my $test_volid = sub {
3259         my ($volid, $is_cdrom, $replicate, $shared, $snapname, $size) = @_;
3260
3261         return if !$volid;
3262
3263         $volhash->{$volid}->{cdrom} //= 1;
3264         $volhash->{$volid}->{cdrom} = 0 if !$is_cdrom;
3265
3266         $volhash->{$volid}->{replicate} //= 0;
3267         $volhash->{$volid}->{replicate} = 1 if $replicate;
3268
3269         $volhash->{$volid}->{shared} //= 0;
3270         $volhash->{$volid}->{shared} = 1 if $shared;
3271
3272         $volhash->{$volid}->{referenced_in_config} //= 0;
3273         $volhash->{$volid}->{referenced_in_config} = 1 if !defined($snapname);
3274
3275         $volhash->{$volid}->{referenced_in_snapshot}->{$snapname} = 1
3276             if defined($snapname);
3277         $volhash->{$volid}->{size} = $size if $size;
3278     };
3279
3280     foreach_drive($conf, sub {
3281         my ($ds, $drive) = @_;
3282         $test_volid->($drive->{file}, drive_is_cdrom($drive), $drive->{replicate} // 1, $drive->{shared}, undef, $drive->{size});
3283     });
3284
3285     foreach my $snapname (keys %{$conf->{snapshots}}) {
3286         my $snap = $conf->{snapshots}->{$snapname};
3287         $test_volid->($snap->{vmstate}, 0, 1, $snapname);
3288         foreach_drive($snap, sub {
3289             my ($ds, $drive) = @_;
3290             $test_volid->($drive->{file}, drive_is_cdrom($drive), $drive->{replicate} // 1, $drive->{shared}, $snapname);
3291         });
3292     }
3293
3294     foreach my $volid (keys %$volhash) {
3295         &$func($volid, $volhash->{$volid}, @param);
3296     }
3297 }
3298
3299 sub conf_has_serial {
3300     my ($conf) = @_;
3301
3302     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3303         if ($conf->{"serial$i"}) {
3304             return 1;
3305         }
3306     }
3307
3308     return 0;
3309 }
3310
3311 sub conf_has_audio {
3312     my ($conf, $id) = @_;
3313
3314     $id //= 0;
3315     my $audio = $conf->{"audio$id"};
3316     return undef if !defined($audio);
3317
3318     my $audioproperties = PVE::JSONSchema::parse_property_string($audio_fmt, $audio);
3319     my $audiodriver = $audioproperties->{driver} // 'spice';
3320
3321     return {
3322         dev => $audioproperties->{device},
3323         dev_id => "audiodev$id",
3324         backend => $audiodriver,
3325         backend_id => "$audiodriver-backend${id}",
3326     };
3327 }
3328
3329 sub vga_conf_has_spice {
3330     my ($vga) = @_;
3331
3332     my $vgaconf = parse_vga($vga);
3333     my $vgatype = $vgaconf->{type};
3334     return 0 if !$vgatype || $vgatype !~ m/^qxl([234])?$/;
3335
3336     return $1 || 1;
3337 }
3338
3339 sub is_native($) {
3340     my ($arch) = @_;
3341     return get_host_arch() eq $arch;
3342 }
3343
3344 my $default_machines = {
3345     x86_64 => 'pc',
3346     aarch64 => 'virt',
3347 };
3348
3349 sub get_basic_machine_info {
3350     my ($conf, $forcemachine) = @_;
3351
3352     my $arch = $conf->{arch} // get_host_arch();
3353     my $machine = $forcemachine || $conf->{machine} || $default_machines->{$arch};
3354     return ($arch, $machine);
3355 }
3356
3357 sub get_ovmf_files($) {
3358     my ($arch) = @_;
3359
3360     my $ovmf = $OVMF->{$arch}
3361         or die "no OVMF images known for architecture '$arch'\n";
3362
3363     return @$ovmf;
3364 }
3365
3366 my $Arch2Qemu = {
3367     aarch64 => '/usr/bin/qemu-system-aarch64',
3368     x86_64 => '/usr/bin/qemu-system-x86_64',
3369 };
3370 sub get_command_for_arch($) {
3371     my ($arch) = @_;
3372     return '/usr/bin/kvm' if is_native($arch);
3373
3374     my $cmd = $Arch2Qemu->{$arch}
3375         or die "don't know how to emulate architecture '$arch'\n";
3376     return $cmd;
3377 }
3378
3379 sub get_cpu_options {
3380     my ($conf, $arch, $kvm, $kvm_off, $machine_version, $winversion, $gpu_passthrough) = @_;
3381
3382     my $cpuFlags = [];
3383     my $ostype = $conf->{ostype};
3384
3385     my $cpu = $kvm ? "kvm64" : "qemu64";
3386     if ($arch eq 'aarch64') {
3387         $cpu = 'cortex-a57';
3388     }
3389     my $hv_vendor_id;
3390     if (my $cputype = $conf->{cpu}) {
3391         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
3392             or die "Cannot parse cpu description: $cputype\n";
3393         $cpu = $cpuconf->{cputype};
3394         $kvm_off = 1 if $cpuconf->{hidden};
3395         $hv_vendor_id = $cpuconf->{'hv-vendor-id'};
3396
3397         if (defined(my $flags = $cpuconf->{flags})) {
3398             push @$cpuFlags, split(";", $flags);
3399         }
3400     }
3401
3402     push @$cpuFlags , '+lahf_lm' if $cpu eq 'kvm64' && $arch eq 'x86_64';
3403
3404     push @$cpuFlags , '-x2apic' if $ostype && $ostype eq 'solaris';
3405
3406     push @$cpuFlags, '+sep' if $cpu eq 'kvm64' || $cpu eq 'kvm32';
3407
3408     push @$cpuFlags, '-rdtscp' if $cpu =~ m/^Opteron/;
3409
3410     if (min_version($machine_version, 2, 3) && $arch eq 'x86_64') {
3411
3412         push @$cpuFlags , '+kvm_pv_unhalt' if $kvm;
3413         push @$cpuFlags , '+kvm_pv_eoi' if $kvm;
3414     }
3415
3416     add_hyperv_enlightenments($cpuFlags, $winversion, $machine_version, $conf->{bios}, $gpu_passthrough, $hv_vendor_id) if $kvm;
3417
3418     push @$cpuFlags, 'enforce' if $cpu ne 'host' && $kvm && $arch eq 'x86_64';
3419
3420     push @$cpuFlags, 'kvm=off' if $kvm_off;
3421
3422     if (my $cpu_vendor = $cpu_vendor_list->{$cpu}) {
3423         push @$cpuFlags, "vendor=${cpu_vendor}"
3424             if $cpu_vendor ne 'default';
3425     } elsif ($arch ne 'aarch64') {
3426         die "internal error"; # should not happen
3427     }
3428
3429     $cpu .= "," . join(',', @$cpuFlags) if scalar(@$cpuFlags);
3430
3431     return ('-cpu', $cpu);
3432 }
3433
3434 sub config_to_command {
3435     my ($storecfg, $vmid, $conf, $defaults, $forcemachine) = @_;
3436
3437     my $cmd = [];
3438     my $globalFlags = [];
3439     my $machineFlags = [];
3440     my $rtcFlags = [];
3441     my $devices = [];
3442     my $pciaddr = '';
3443     my $bridges = {};
3444     my $vernum = 0; # unknown
3445     my $ostype = $conf->{ostype};
3446     my $winversion = windows_version($ostype);
3447     my $kvm = $conf->{kvm};
3448
3449     my ($arch, $machine_type) = get_basic_machine_info($conf, $forcemachine);
3450     my $kvm_binary = get_command_for_arch($arch);
3451     my $kvmver = kvm_user_version($kvm_binary);
3452     my $machine_version = PVE::QemuServer::Machine::extract_version($machine_type) // $kvmver;
3453     $kvm //= 1 if is_native($arch);
3454
3455     if ($kvm) {
3456         die "KVM virtualisation configured, but not available. Either disable in VM configuration or enable in BIOS.\n"
3457             if !defined kvm_version();
3458     }
3459
3460     if ($kvmver =~ m/^(\d+)\.(\d+)$/) {
3461         $vernum = $1*1000000+$2*1000;
3462     } elsif ($kvmver =~ m/^(\d+)\.(\d+)\.(\d+)$/) {
3463         $vernum = $1*1000000+$2*1000+$3;
3464     }
3465
3466     die "detected old qemu-kvm binary ($kvmver)\n" if $vernum < 15000;
3467
3468     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
3469     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
3470     my $use_old_bios_files = undef;
3471     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3472
3473     my $cpuunits = defined($conf->{cpuunits}) ?
3474             $conf->{cpuunits} : $defaults->{cpuunits};
3475
3476     push @$cmd, $kvm_binary;
3477
3478     push @$cmd, '-id', $vmid;
3479
3480     my $vmname = $conf->{name} || "vm$vmid";
3481
3482     push @$cmd, '-name', $vmname;
3483
3484     my $use_virtio = 0;
3485
3486     my $qmpsocket = PVE::QemuServer::Helpers::qmp_socket($vmid);
3487     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server,nowait";
3488     push @$cmd, '-mon', "chardev=qmp,mode=control";
3489
3490     if (min_version($machine_version, 2, 12)) {
3491         push @$cmd, '-chardev', "socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5";
3492         push @$cmd, '-mon', "chardev=qmp-event,mode=control";
3493     }
3494
3495     push @$cmd, '-pidfile' , PVE::QemuServer::Helpers::pidfile_name($vmid);
3496
3497     push @$cmd, '-daemonize';
3498
3499     if ($conf->{smbios1}) {
3500         my $smbios_conf = parse_smbios1($conf->{smbios1});
3501         if ($smbios_conf->{base64}) {
3502             # Do not pass base64 flag to qemu
3503             delete $smbios_conf->{base64};
3504             my $smbios_string = "";
3505             foreach my $key (keys %$smbios_conf) {
3506                 my $value;
3507                 if ($key eq "uuid") {
3508                     $value = $smbios_conf->{uuid}
3509                 } else {
3510                     $value = decode_base64($smbios_conf->{$key});
3511                 }
3512                 # qemu accepts any binary data, only commas need escaping by double comma
3513                 $value =~ s/,/,,/g;
3514                 $smbios_string .= "," . $key . "=" . $value if $value;
3515             }
3516             push @$cmd, '-smbios', "type=1" . $smbios_string;
3517         } else {
3518             push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
3519         }
3520     }
3521
3522     if ($conf->{vmgenid}) {
3523         push @$devices, '-device', 'vmgenid,guid='.$conf->{vmgenid};
3524     }
3525
3526     my ($ovmf_code, $ovmf_vars) = get_ovmf_files($arch);
3527     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3528         die "uefi base image not found\n" if ! -f $ovmf_code;
3529
3530         my $path;
3531         my $format;
3532         if (my $efidisk = $conf->{efidisk0}) {
3533             my $d = PVE::JSONSchema::parse_property_string($efidisk_fmt, $efidisk);
3534             my ($storeid, $volname) = PVE::Storage::parse_volume_id($d->{file}, 1);
3535             $format = $d->{format};
3536             if ($storeid) {
3537                 $path = PVE::Storage::path($storecfg, $d->{file});
3538                 if (!defined($format)) {
3539                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
3540                     $format = qemu_img_format($scfg, $volname);
3541                 }
3542             } else {
3543                 $path = $d->{file};
3544                 die "efidisk format must be specified\n"
3545                     if !defined($format);
3546             }
3547         } else {
3548             warn "no efidisk configured! Using temporary efivars disk.\n";
3549             $path = "/tmp/$vmid-ovmf.fd";
3550             PVE::Tools::file_copy($ovmf_vars, $path, -s $ovmf_vars);
3551             $format = 'raw';
3552         }
3553
3554         push @$cmd, '-drive', "if=pflash,unit=0,format=raw,readonly,file=$ovmf_code";
3555         push @$cmd, '-drive', "if=pflash,unit=1,format=$format,id=drive-efidisk0,file=$path";
3556     }
3557
3558     # load q35 config
3559     if ($q35) {
3560         # we use different pcie-port hardware for qemu >= 4.0 for passthrough
3561         if (min_version($machine_version, 4, 0)) {
3562             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35-4.0.cfg';
3563         } else {
3564             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35.cfg';
3565         }
3566     }
3567
3568     # add usb controllers
3569     my @usbcontrollers = PVE::QemuServer::USB::get_usb_controllers($conf, $bridges, $arch, $machine_type, $usbdesc->{format}, $MAX_USB_DEVICES);
3570     push @$devices, @usbcontrollers if @usbcontrollers;
3571     my $vga = parse_vga($conf->{vga});
3572
3573     my $qxlnum = vga_conf_has_spice($conf->{vga});
3574     $vga->{type} = 'qxl' if $qxlnum;
3575
3576     if (!$vga->{type}) {
3577         if ($arch eq 'aarch64') {
3578             $vga->{type} = 'virtio';
3579         } elsif (min_version($machine_version, 2, 9)) {
3580             $vga->{type} = (!$winversion || $winversion >= 6) ? 'std' : 'cirrus';
3581         } else {
3582             $vga->{type} = ($winversion >= 6) ? 'std' : 'cirrus';
3583         }
3584     }
3585
3586     # enable absolute mouse coordinates (needed by vnc)
3587     my $tablet;
3588     if (defined($conf->{tablet})) {
3589         $tablet = $conf->{tablet};
3590     } else {
3591         $tablet = $defaults->{tablet};
3592         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
3593         $tablet = 0 if $vga->{type} =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
3594     }
3595
3596     if ($tablet) {
3597         push @$devices, '-device', print_tabletdevice_full($conf, $arch) if $tablet;
3598         my $kbd = print_keyboarddevice_full($conf, $arch);
3599         push @$devices, '-device', $kbd if defined($kbd);
3600     }
3601
3602     my $kvm_off = 0;
3603     my $gpu_passthrough;
3604
3605     # host pci devices
3606     for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
3607         my $id = "hostpci$i";
3608         my $d = parse_hostpci($conf->{$id});
3609         next if !$d;
3610
3611         if (my $pcie = $d->{pcie}) {
3612             die "q35 machine model is not enabled" if !$q35;
3613             # win7 wants to have the pcie devices directly on the pcie bus
3614             # instead of in the root port
3615             if ($winversion == 7) {
3616                 $pciaddr = print_pcie_addr("${id}bus0");
3617             } else {
3618                 # add more root ports if needed, 4 are present by default
3619                 # by pve-q35 cfgs, rest added here on demand.
3620                 if ($i > 3) {
3621                     push @$devices, '-device', print_pcie_root_port($i);
3622                 }
3623                 $pciaddr = print_pcie_addr($id);
3624             }
3625         } else {
3626             $pciaddr = print_pci_addr($id, $bridges, $arch, $machine_type);
3627         }
3628
3629         my $xvga = '';
3630         if ($d->{'x-vga'}) {
3631             $xvga = ',x-vga=on' if !($conf->{bios} && $conf->{bios} eq 'ovmf');
3632             $kvm_off = 1;
3633             $vga->{type} = 'none' if !defined($conf->{vga});
3634             $gpu_passthrough = 1;
3635         }
3636
3637         my $pcidevices = $d->{pciid};
3638         my $multifunction = 1 if @$pcidevices > 1;
3639
3640         my $sysfspath;
3641         if ($d->{mdev} && scalar(@$pcidevices) == 1) {
3642             my $pci_id = $pcidevices->[0]->{id};
3643             my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $i);
3644             $sysfspath = "/sys/bus/pci/devices/0000:$pci_id/$uuid";
3645         } elsif ($d->{mdev}) {
3646             warn "ignoring mediated device '$id' with multifunction device\n";
3647         }
3648
3649         my $j=0;
3650         foreach my $pcidevice (@$pcidevices) {
3651             my $devicestr = "vfio-pci";
3652
3653             if ($sysfspath) {
3654                 $devicestr .= ",sysfsdev=$sysfspath";
3655             } else {
3656                 $devicestr .= ",host=$pcidevice->{id}";
3657             }
3658
3659             my $mf_addr = $multifunction ? ".$j" : '';
3660             $devicestr .= ",id=${id}${mf_addr}${pciaddr}${mf_addr}";
3661
3662             if ($j == 0) {
3663                 $devicestr .= ',rombar=0' if defined($d->{rombar}) && !$d->{rombar};
3664                 $devicestr .= "$xvga";
3665                 $devicestr .= ",multifunction=on" if $multifunction;
3666                 $devicestr .= ",romfile=/usr/share/kvm/$d->{romfile}" if $d->{romfile};
3667             }
3668
3669             push @$devices, '-device', $devicestr;
3670             $j++;
3671         }
3672     }
3673
3674     # usb devices
3675     my $usb_dev_features = {};
3676     $usb_dev_features->{spice_usb3} = 1 if min_version($machine_version, 4, 0);
3677
3678     my @usbdevices = PVE::QemuServer::USB::get_usb_devices($conf, $usbdesc->{format}, $MAX_USB_DEVICES, $usb_dev_features);
3679     push @$devices, @usbdevices if @usbdevices;
3680     # serial devices
3681     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3682         if (my $path = $conf->{"serial$i"}) {
3683             if ($path eq 'socket') {
3684                 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
3685                 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server,nowait";
3686                 # On aarch64, serial0 is the UART device. Qemu only allows
3687                 # connecting UART devices via the '-serial' command line, as
3688                 # the device has a fixed slot on the hardware...
3689                 if ($arch eq 'aarch64' && $i == 0) {
3690                     push @$devices, '-serial', "chardev:serial$i";
3691                 } else {
3692                     push @$devices, '-device', "isa-serial,chardev=serial$i";
3693                 }
3694             } else {
3695                 die "no such serial device\n" if ! -c $path;
3696                 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
3697                 push @$devices, '-device', "isa-serial,chardev=serial$i";
3698             }
3699         }
3700     }
3701
3702     # parallel devices
3703     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
3704         if (my $path = $conf->{"parallel$i"}) {
3705             die "no such parallel device\n" if ! -c $path;
3706             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
3707             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
3708             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
3709         }
3710     }
3711
3712     if (my $audio = conf_has_audio($conf)) {
3713
3714         my $audiopciaddr = print_pci_addr("audio0", $bridges, $arch, $machine_type);
3715
3716         my $id = $audio->{dev_id};
3717         if ($audio->{dev} eq 'AC97') {
3718             push @$devices, '-device', "AC97,id=${id}${audiopciaddr}";
3719         } elsif ($audio->{dev} =~ /intel\-hda$/) {
3720             push @$devices, '-device', "$audio->{dev},id=${id}${audiopciaddr}";
3721             push @$devices, '-device', "hda-micro,id=${id}-codec0,bus=${id}.0,cad=0";
3722             push @$devices, '-device', "hda-duplex,id=${id}-codec1,bus=${id}.0,cad=1";
3723         } else {
3724             die "unkown audio device '$audio->{dev}', implement me!";
3725         }
3726
3727         push @$devices, '-audiodev', "$audio->{backend},id=$audio->{backend_id}";
3728     }
3729
3730     my $sockets = 1;
3731     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3732     $sockets = $conf->{sockets} if  $conf->{sockets};
3733
3734     my $cores = $conf->{cores} || 1;
3735
3736     my $maxcpus = $sockets * $cores;
3737
3738     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
3739
3740     my $allowed_vcpus = $cpuinfo->{cpus};
3741
3742     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n"
3743         if ($allowed_vcpus < $maxcpus);
3744
3745     if($hotplug_features->{cpu} && min_version($machine_version, 2, 7)) {
3746
3747         push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3748         for (my $i = 2; $i <= $vcpus; $i++)  {
3749             my $cpustr = print_cpu_device($conf,$i);
3750             push @$cmd, '-device', $cpustr;
3751         }
3752
3753     } else {
3754
3755         push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3756     }
3757     push @$cmd, '-nodefaults';
3758
3759     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
3760
3761     my $bootindex_hash = {};
3762     my $i = 1;
3763     foreach my $o (split(//, $bootorder)) {
3764         $bootindex_hash->{$o} = $i*100;
3765         $i++;
3766     }
3767
3768     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
3769
3770     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
3771
3772     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
3773
3774     if ($vga->{type} && $vga->{type} !~ m/^serial\d+$/ && $vga->{type} ne 'none'){
3775         push @$devices, '-device', print_vga_device($conf, $vga, $arch, $machine_version, $machine_type, undef, $qxlnum, $bridges);
3776         my $socket = PVE::QemuServer::Helpers::vnc_socket($vmid);
3777         push @$cmd,  '-vnc', "unix:$socket,password";
3778     } else {
3779         push @$cmd, '-vga', 'none' if $vga->{type} eq 'none';
3780         push @$cmd, '-nographic';
3781     }
3782
3783     # time drift fix
3784     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
3785
3786     my $useLocaltime = $conf->{localtime};
3787
3788     if ($winversion >= 5) { # windows
3789         $useLocaltime = 1 if !defined($conf->{localtime});
3790
3791         # use time drift fix when acpi is enabled
3792         if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
3793             $tdf = 1 if !defined($conf->{tdf});
3794         }
3795     }
3796
3797     if ($winversion >= 6) {
3798         push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3799         push @$cmd, '-no-hpet';
3800     }
3801
3802     push @$rtcFlags, 'driftfix=slew' if $tdf;
3803
3804     if (!$kvm) {
3805         push @$machineFlags, 'accel=tcg';
3806     }
3807
3808     if ($machine_type) {
3809         push @$machineFlags, "type=${machine_type}";
3810     }
3811
3812     if (($conf->{startdate}) && ($conf->{startdate} ne 'now')) {
3813         push @$rtcFlags, "base=$conf->{startdate}";
3814     } elsif ($useLocaltime) {
3815         push @$rtcFlags, 'base=localtime';
3816     }
3817
3818     push @$cmd, get_cpu_options($conf, $arch, $kvm, $kvm_off, $machine_version, $winversion, $gpu_passthrough);
3819
3820     PVE::QemuServer::Memory::config($conf, $vmid, $sockets, $cores, $defaults, $hotplug_features, $cmd);
3821
3822     push @$cmd, '-S' if $conf->{freeze};
3823
3824     push @$cmd, '-k', $conf->{keyboard} if defined($conf->{keyboard});
3825
3826     my $guest_agent = parse_guest_agent($conf);
3827
3828     if ($guest_agent->{enabled}) {
3829         my $qgasocket = PVE::QemuServer::Helpers::qmp_socket($vmid, 1);
3830         push @$devices, '-chardev', "socket,path=$qgasocket,server,nowait,id=qga0";
3831
3832         if (!$guest_agent->{type} || $guest_agent->{type} eq 'virtio') {
3833             my $pciaddr = print_pci_addr("qga0", $bridges, $arch, $machine_type);
3834             push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3835             push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3836         } elsif ($guest_agent->{type} eq 'isa') {
3837             push @$devices, '-device', "isa-serial,chardev=qga0";
3838         }
3839     }
3840
3841     my $spice_port;
3842
3843     if ($qxlnum) {
3844         if ($qxlnum > 1) {
3845             if ($winversion){
3846                 for(my $i = 1; $i < $qxlnum; $i++){
3847                     push @$devices, '-device', print_vga_device($conf, $vga, $arch, $machine_version, $machine_type, $i, $qxlnum, $bridges);
3848                 }
3849             } else {
3850                 # assume other OS works like Linux
3851                 my ($ram, $vram) = ("134217728", "67108864");
3852                 if ($vga->{memory}) {
3853                     $ram = PVE::Tools::convert_size($qxlnum*4*$vga->{memory}, 'mb' => 'b');
3854                     $vram = PVE::Tools::convert_size($qxlnum*2*$vga->{memory}, 'mb' => 'b');
3855                 }
3856                 push @$cmd, '-global', "qxl-vga.ram_size=$ram";
3857                 push @$cmd, '-global', "qxl-vga.vram_size=$vram";
3858             }
3859         }
3860
3861         my $pciaddr = print_pci_addr("spice", $bridges, $arch, $machine_type);
3862
3863         my $nodename = PVE::INotify::nodename();
3864         my $pfamily = PVE::Tools::get_host_address_family($nodename);
3865         my @nodeaddrs = PVE::Tools::getaddrinfo_all('localhost', family => $pfamily);
3866         die "failed to get an ip address of type $pfamily for 'localhost'\n" if !@nodeaddrs;
3867
3868         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3869         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3870         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3871
3872         my $localhost = PVE::Network::addr_to_ip($nodeaddrs[0]->{addr});
3873         $spice_port = PVE::Tools::next_spice_port($pfamily, $localhost);
3874
3875         my $spice_enhancement = PVE::JSONSchema::parse_property_string($spice_enhancements_fmt, $conf->{spice_enhancements} // '');
3876         if ($spice_enhancement->{foldersharing}) {
3877             push @$devices, '-chardev', "spiceport,id=foldershare,name=org.spice-space.webdav.0";
3878             push @$devices, '-device', "virtserialport,chardev=foldershare,name=org.spice-space.webdav.0";
3879         }
3880
3881         my $spice_opts = "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";
3882         $spice_opts .= ",streaming-video=$spice_enhancement->{videostreaming}" if $spice_enhancement->{videostreaming};
3883         push @$devices, '-spice', "$spice_opts";
3884     }
3885
3886     # enable balloon by default, unless explicitly disabled
3887     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3888         $pciaddr = print_pci_addr("balloon0", $bridges, $arch, $machine_type);
3889         push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
3890     }
3891
3892     if ($conf->{watchdog}) {
3893         my $wdopts = parse_watchdog($conf->{watchdog});
3894         $pciaddr = print_pci_addr("watchdog", $bridges, $arch, $machine_type);
3895         my $watchdog = $wdopts->{model} || 'i6300esb';
3896         push @$devices, '-device', "$watchdog$pciaddr";
3897         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3898     }
3899
3900     my $vollist = [];
3901     my $scsicontroller = {};
3902     my $ahcicontroller = {};
3903     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3904
3905     # Add iscsi initiator name if available
3906     if (my $initiator = get_initiator_name()) {
3907         push @$devices, '-iscsi', "initiator-name=$initiator";
3908     }
3909
3910     foreach_drive($conf, sub {
3911         my ($ds, $drive) = @_;
3912
3913         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3914             push @$vollist, $drive->{file};
3915         }
3916
3917         # ignore efidisk here, already added in bios/fw handling code above
3918         return if $drive->{interface} eq 'efidisk';
3919
3920         $use_virtio = 1 if $ds =~ m/^virtio/;
3921
3922         if (drive_is_cdrom ($drive)) {
3923             if ($bootindex_hash->{d}) {
3924                 $drive->{bootindex} = $bootindex_hash->{d};
3925                 $bootindex_hash->{d} += 1;
3926             }
3927         } else {
3928             if ($bootindex_hash->{c}) {
3929                 $drive->{bootindex} = $bootindex_hash->{c} if $conf->{bootdisk} && ($conf->{bootdisk} eq $ds);
3930                 $bootindex_hash->{c} += 1;
3931             }
3932         }
3933
3934         if($drive->{interface} eq 'virtio'){
3935            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3936         }
3937
3938         if ($drive->{interface} eq 'scsi') {
3939
3940             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
3941
3942             $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges, $arch, $machine_type);
3943             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
3944
3945             my $iothread = '';
3946             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
3947                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3948                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3949             } elsif ($drive->{iothread}) {
3950                 warn "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n";
3951             }
3952
3953             my $queues = '';
3954             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
3955                 $queues = ",num_queues=$drive->{queues}";
3956             }
3957
3958             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues" if !$scsicontroller->{$controller};
3959             $scsicontroller->{$controller}=1;
3960         }
3961
3962         if ($drive->{interface} eq 'sata') {
3963            my $controller = int($drive->{index} / $MAX_SATA_DISKS);
3964            $pciaddr = print_pci_addr("ahci$controller", $bridges, $arch, $machine_type);
3965            push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr" if !$ahcicontroller->{$controller};
3966            $ahcicontroller->{$controller}=1;
3967         }
3968
3969         my $drive_cmd = print_drive_full($storecfg, $vmid, $drive);
3970         push @$devices, '-drive',$drive_cmd;
3971         push @$devices, '-device', print_drivedevice_full($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type);
3972     });
3973
3974     for (my $i = 0; $i < $MAX_NETS; $i++) {
3975          next if !$conf->{"net$i"};
3976          my $d = parse_net($conf->{"net$i"});
3977          next if !$d;
3978
3979          $use_virtio = 1 if $d->{model} eq 'virtio';
3980
3981          if ($bootindex_hash->{n}) {
3982             $d->{bootindex} = $bootindex_hash->{n};
3983             $bootindex_hash->{n} += 1;
3984          }
3985
3986          my $netdevfull = print_netdev_full($vmid, $conf, $arch, $d, "net$i");
3987          push @$devices, '-netdev', $netdevfull;
3988
3989          my $netdevicefull = print_netdevice_full($vmid, $conf, $d, "net$i", $bridges, $use_old_bios_files, $arch, $machine_type);
3990          push @$devices, '-device', $netdevicefull;
3991     }
3992
3993     if ($conf->{ivshmem}) {
3994         my $ivshmem = PVE::JSONSchema::parse_property_string($ivshmem_fmt, $conf->{ivshmem});
3995
3996         my $bus;
3997         if ($q35) {
3998             $bus = print_pcie_addr("ivshmem");
3999         } else {
4000             $bus = print_pci_addr("ivshmem", $bridges, $arch, $machine_type);
4001         }
4002
4003         my $ivshmem_name = $ivshmem->{name} // $vmid;
4004         my $path = '/dev/shm/pve-shm-' . $ivshmem_name;
4005
4006         push @$devices, '-device', "ivshmem-plain,memdev=ivshmem$bus,";
4007         push @$devices, '-object', "memory-backend-file,id=ivshmem,share=on,mem-path=$path,size=$ivshmem->{size}M";
4008     }
4009
4010     if (!$q35) {
4011         # add pci bridges
4012         if (min_version($machine_version, 2, 3)) {
4013            $bridges->{1} = 1;
4014            $bridges->{2} = 1;
4015         }
4016
4017         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
4018
4019         for my $k (sort {$b cmp $a} keys %$bridges) {
4020             $pciaddr = print_pci_addr("pci.$k", undef, $arch, $machine_type);
4021             unshift @$devices, '-device', "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr" if $k > 0;
4022         }
4023     }
4024
4025     push @$cmd, @$devices;
4026     push @$cmd, '-rtc', join(',', @$rtcFlags)
4027         if scalar(@$rtcFlags);
4028     push @$cmd, '-machine', join(',', @$machineFlags)
4029         if scalar(@$machineFlags);
4030     push @$cmd, '-global', join(',', @$globalFlags)
4031         if scalar(@$globalFlags);
4032
4033     if (my $vmstate = $conf->{vmstate}) {
4034         my $statepath = PVE::Storage::path($storecfg, $vmstate);
4035         push @$vollist, $vmstate;
4036         push @$cmd, '-loadstate', $statepath;
4037     }
4038
4039     # add custom args
4040     if ($conf->{args}) {
4041         my $aa = PVE::Tools::split_args($conf->{args});
4042         push @$cmd, @$aa;
4043     }
4044
4045     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
4046 }
4047
4048 sub spice_port {
4049     my ($vmid) = @_;
4050
4051     my $res = mon_cmd($vmid, 'query-spice');
4052
4053     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
4054 }
4055
4056 sub vm_devices_list {
4057     my ($vmid) = @_;
4058
4059     my $res = mon_cmd($vmid, 'query-pci');
4060     my $devices_to_check = [];
4061     my $devices = {};
4062     foreach my $pcibus (@$res) {
4063         push @$devices_to_check, @{$pcibus->{devices}},
4064     }
4065
4066     while (@$devices_to_check) {
4067         my $to_check = [];
4068         for my $d (@$devices_to_check) {
4069             $devices->{$d->{'qdev_id'}} = 1 if $d->{'qdev_id'};
4070             next if !$d->{'pci_bridge'};
4071
4072             $devices->{$d->{'qdev_id'}} += scalar(@{$d->{'pci_bridge'}->{devices}});
4073             push @$to_check, @{$d->{'pci_bridge'}->{devices}};
4074         }
4075         $devices_to_check = $to_check;
4076     }
4077
4078     my $resblock = mon_cmd($vmid, 'query-block');
4079     foreach my $block (@$resblock) {
4080         if($block->{device} =~ m/^drive-(\S+)/){
4081                 $devices->{$1} = 1;
4082         }
4083     }
4084
4085     my $resmice = mon_cmd($vmid, 'query-mice');
4086     foreach my $mice (@$resmice) {
4087         if ($mice->{name} eq 'QEMU HID Tablet') {
4088             $devices->{tablet} = 1;
4089             last;
4090         }
4091     }
4092
4093     # for usb devices there is no query-usb
4094     # but we can iterate over the entries in
4095     # qom-list path=/machine/peripheral
4096     my $resperipheral = mon_cmd($vmid, 'qom-list', path => '/machine/peripheral');
4097     foreach my $per (@$resperipheral) {
4098         if ($per->{name} =~ m/^usb\d+$/) {
4099             $devices->{$per->{name}} = 1;
4100         }
4101     }
4102
4103     return $devices;
4104 }
4105
4106 sub vm_deviceplug {
4107     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4108
4109     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
4110
4111     my $devices_list = vm_devices_list($vmid);
4112     return 1 if defined($devices_list->{$deviceid});
4113
4114     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid, $arch, $machine_type); # add PCI bridge if we need it for the device
4115
4116     if ($deviceid eq 'tablet') {
4117
4118         qemu_deviceadd($vmid, print_tabletdevice_full($conf, $arch));
4119
4120     } elsif ($deviceid eq 'keyboard') {
4121
4122         qemu_deviceadd($vmid, print_keyboarddevice_full($conf, $arch));
4123
4124     } elsif ($deviceid =~ m/^usb(\d+)$/) {
4125
4126         die "usb hotplug currently not reliable\n";
4127         # since we can't reliably hot unplug all added usb devices
4128         # and usb passthrough disables live migration
4129         # we disable usb hotplugging for now
4130         qemu_deviceadd($vmid, PVE::QemuServer::USB::print_usbdevice_full($conf, $deviceid, $device));
4131
4132     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4133
4134         qemu_iothread_add($vmid, $deviceid, $device);
4135
4136         qemu_driveadd($storecfg, $vmid, $device);
4137         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, $arch, $machine_type);
4138
4139         qemu_deviceadd($vmid, $devicefull);
4140         eval { qemu_deviceaddverify($vmid, $deviceid); };
4141         if (my $err = $@) {
4142             eval { qemu_drivedel($vmid, $deviceid); };
4143             warn $@ if $@;
4144             die $err;
4145         }
4146
4147     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4148
4149
4150         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
4151         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
4152         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
4153
4154         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
4155
4156         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
4157             qemu_iothread_add($vmid, $deviceid, $device);
4158             $devicefull .= ",iothread=iothread-$deviceid";
4159         }
4160
4161         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
4162             $devicefull .= ",num_queues=$device->{queues}";
4163         }
4164
4165         qemu_deviceadd($vmid, $devicefull);
4166         qemu_deviceaddverify($vmid, $deviceid);
4167
4168     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4169
4170         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device, $arch, $machine_type);
4171         qemu_driveadd($storecfg, $vmid, $device);
4172
4173         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, $arch, $machine_type);
4174         eval { qemu_deviceadd($vmid, $devicefull); };
4175         if (my $err = $@) {
4176             eval { qemu_drivedel($vmid, $deviceid); };
4177             warn $@ if $@;
4178             die $err;
4179         }
4180
4181     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4182
4183         return undef if !qemu_netdevadd($vmid, $conf, $arch, $device, $deviceid);
4184
4185         my $machine_type = PVE::QemuServer::Machine::qemu_machine_pxe($vmid, $conf);
4186         my $use_old_bios_files = undef;
4187         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
4188
4189         my $netdevicefull = print_netdevice_full($vmid, $conf, $device, $deviceid, undef, $use_old_bios_files, $arch, $machine_type);
4190         qemu_deviceadd($vmid, $netdevicefull);
4191         eval {
4192             qemu_deviceaddverify($vmid, $deviceid);
4193             qemu_set_link_status($vmid, $deviceid, !$device->{link_down});
4194         };
4195         if (my $err = $@) {
4196             eval { qemu_netdevdel($vmid, $deviceid); };
4197             warn $@ if $@;
4198             die $err;
4199         }
4200
4201     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
4202
4203         my $bridgeid = $2;
4204         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
4205         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
4206
4207         qemu_deviceadd($vmid, $devicefull);
4208         qemu_deviceaddverify($vmid, $deviceid);
4209
4210     } else {
4211         die "can't hotplug device '$deviceid'\n";
4212     }
4213
4214     return 1;
4215 }
4216
4217 # fixme: this should raise exceptions on error!
4218 sub vm_deviceunplug {
4219     my ($vmid, $conf, $deviceid) = @_;
4220
4221     my $devices_list = vm_devices_list($vmid);
4222     return 1 if !defined($devices_list->{$deviceid});
4223
4224     die "can't unplug bootdisk" if $conf->{bootdisk} && $conf->{bootdisk} eq $deviceid;
4225
4226     if ($deviceid eq 'tablet' || $deviceid eq 'keyboard') {
4227
4228         qemu_devicedel($vmid, $deviceid);
4229
4230     } elsif ($deviceid =~ m/^usb\d+$/) {
4231
4232         die "usb hotplug currently not reliable\n";
4233         # when unplugging usb devices this way,
4234         # there may be remaining usb controllers/hubs
4235         # so we disable it for now
4236         qemu_devicedel($vmid, $deviceid);
4237         qemu_devicedelverify($vmid, $deviceid);
4238
4239     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4240
4241         qemu_devicedel($vmid, $deviceid);
4242         qemu_devicedelverify($vmid, $deviceid);
4243         qemu_drivedel($vmid, $deviceid);
4244         qemu_iothread_del($conf, $vmid, $deviceid);
4245
4246     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4247
4248         qemu_devicedel($vmid, $deviceid);
4249         qemu_devicedelverify($vmid, $deviceid);
4250         qemu_iothread_del($conf, $vmid, $deviceid);
4251
4252     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4253
4254         qemu_devicedel($vmid, $deviceid);
4255         qemu_drivedel($vmid, $deviceid);
4256         qemu_deletescsihw($conf, $vmid, $deviceid);
4257
4258     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4259
4260         qemu_devicedel($vmid, $deviceid);
4261         qemu_devicedelverify($vmid, $deviceid);
4262         qemu_netdevdel($vmid, $deviceid);
4263
4264     } else {
4265         die "can't unplug device '$deviceid'\n";
4266     }
4267
4268     return 1;
4269 }
4270
4271 sub qemu_deviceadd {
4272     my ($vmid, $devicefull) = @_;
4273
4274     $devicefull = "driver=".$devicefull;
4275     my %options =  split(/[=,]/, $devicefull);
4276
4277     mon_cmd($vmid, "device_add" , %options);
4278 }
4279
4280 sub qemu_devicedel {
4281     my ($vmid, $deviceid) = @_;
4282
4283     my $ret = mon_cmd($vmid, "device_del", id => $deviceid);
4284 }
4285
4286 sub qemu_iothread_add {
4287     my($vmid, $deviceid, $device) = @_;
4288
4289     if ($device->{iothread}) {
4290         my $iothreads = vm_iothreads_list($vmid);
4291         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
4292     }
4293 }
4294
4295 sub qemu_iothread_del {
4296     my($conf, $vmid, $deviceid) = @_;
4297
4298     my $confid = $deviceid;
4299     if ($deviceid =~ m/^(?:virtioscsi|scsihw)(\d+)$/) {
4300         $confid = 'scsi' . $1;
4301     }
4302     my $device = parse_drive($confid, $conf->{$confid});
4303     if ($device->{iothread}) {
4304         my $iothreads = vm_iothreads_list($vmid);
4305         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
4306     }
4307 }
4308
4309 sub qemu_objectadd {
4310     my($vmid, $objectid, $qomtype) = @_;
4311
4312     mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
4313
4314     return 1;
4315 }
4316
4317 sub qemu_objectdel {
4318     my($vmid, $objectid) = @_;
4319
4320     mon_cmd($vmid, "object-del", id => $objectid);
4321
4322     return 1;
4323 }
4324
4325 sub qemu_driveadd {
4326     my ($storecfg, $vmid, $device) = @_;
4327
4328     my $drive = print_drive_full($storecfg, $vmid, $device);
4329     $drive =~ s/\\/\\\\/g;
4330     my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_add auto \"$drive\"");
4331
4332     # If the command succeeds qemu prints: "OK"
4333     return 1 if $ret =~ m/OK/s;
4334
4335     die "adding drive failed: $ret\n";
4336 }
4337
4338 sub qemu_drivedel {
4339     my($vmid, $deviceid) = @_;
4340
4341     my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_del drive-$deviceid");
4342     $ret =~ s/^\s+//;
4343
4344     return 1 if $ret eq "";
4345
4346     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
4347     return 1 if $ret =~ m/Device \'.*?\' not found/s;
4348
4349     die "deleting drive $deviceid failed : $ret\n";
4350 }
4351
4352 sub qemu_deviceaddverify {
4353     my ($vmid, $deviceid) = @_;
4354
4355     for (my $i = 0; $i <= 5; $i++) {
4356          my $devices_list = vm_devices_list($vmid);
4357          return 1 if defined($devices_list->{$deviceid});
4358          sleep 1;
4359     }
4360
4361     die "error on hotplug device '$deviceid'\n";
4362 }
4363
4364
4365 sub qemu_devicedelverify {
4366     my ($vmid, $deviceid) = @_;
4367
4368     # need to verify that the device is correctly removed as device_del
4369     # is async and empty return is not reliable
4370
4371     for (my $i = 0; $i <= 5; $i++) {
4372          my $devices_list = vm_devices_list($vmid);
4373          return 1 if !defined($devices_list->{$deviceid});
4374          sleep 1;
4375     }
4376
4377     die "error on hot-unplugging device '$deviceid'\n";
4378 }
4379
4380 sub qemu_findorcreatescsihw {
4381     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4382
4383     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4384
4385     my $scsihwid="$controller_prefix$controller";
4386     my $devices_list = vm_devices_list($vmid);
4387
4388     if(!defined($devices_list->{$scsihwid})) {
4389         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device, $arch, $machine_type);
4390     }
4391
4392     return 1;
4393 }
4394
4395 sub qemu_deletescsihw {
4396     my ($conf, $vmid, $opt) = @_;
4397
4398     my $device = parse_drive($opt, $conf->{$opt});
4399
4400     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
4401         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
4402         return 1;
4403     }
4404
4405     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4406
4407     my $devices_list = vm_devices_list($vmid);
4408     foreach my $opt (keys %{$devices_list}) {
4409         if (PVE::QemuServer::is_valid_drivename($opt)) {
4410             my $drive = PVE::QemuServer::parse_drive($opt, $conf->{$opt});
4411             if($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
4412                 return 1;
4413             }
4414         }
4415     }
4416
4417     my $scsihwid="scsihw$controller";
4418
4419     vm_deviceunplug($vmid, $conf, $scsihwid);
4420
4421     return 1;
4422 }
4423
4424 sub qemu_add_pci_bridge {
4425     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4426
4427     my $bridges = {};
4428
4429     my $bridgeid;
4430
4431     print_pci_addr($device, $bridges, $arch, $machine_type);
4432
4433     while (my ($k, $v) = each %$bridges) {
4434         $bridgeid = $k;
4435     }
4436     return 1 if !defined($bridgeid) || $bridgeid < 1;
4437
4438     my $bridge = "pci.$bridgeid";
4439     my $devices_list = vm_devices_list($vmid);
4440
4441     if (!defined($devices_list->{$bridge})) {
4442         vm_deviceplug($storecfg, $conf, $vmid, $bridge, $arch, $machine_type);
4443     }
4444
4445     return 1;
4446 }
4447
4448 sub qemu_set_link_status {
4449     my ($vmid, $device, $up) = @_;
4450
4451     mon_cmd($vmid, "set_link", name => $device,
4452                up => $up ? JSON::true : JSON::false);
4453 }
4454
4455 sub qemu_netdevadd {
4456     my ($vmid, $conf, $arch, $device, $deviceid) = @_;
4457
4458     my $netdev = print_netdev_full($vmid, $conf, $arch, $device, $deviceid, 1);
4459     my %options =  split(/[=,]/, $netdev);
4460
4461     mon_cmd($vmid, "netdev_add",  %options);
4462     return 1;
4463 }
4464
4465 sub qemu_netdevdel {
4466     my ($vmid, $deviceid) = @_;
4467
4468     mon_cmd($vmid, "netdev_del", id => $deviceid);
4469 }
4470
4471 sub qemu_usb_hotplug {
4472     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4473
4474     return if !$device;
4475
4476     # remove the old one first
4477     vm_deviceunplug($vmid, $conf, $deviceid);
4478
4479     # check if xhci controller is necessary and available
4480     if ($device->{usb3}) {
4481
4482         my $devicelist = vm_devices_list($vmid);
4483
4484         if (!$devicelist->{xhci}) {
4485             my $pciaddr = print_pci_addr("xhci", undef, $arch, $machine_type);
4486             qemu_deviceadd($vmid, "nec-usb-xhci,id=xhci$pciaddr");
4487         }
4488     }
4489     my $d = parse_usb_device($device->{host});
4490     $d->{usb3} = $device->{usb3};
4491
4492     # add the new one
4493     vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $d, $arch, $machine_type);
4494 }
4495
4496 sub qemu_cpu_hotplug {
4497     my ($vmid, $conf, $vcpus) = @_;
4498
4499     my $machine_type = PVE::QemuServer::Machine::get_current_qemu_machine($vmid);
4500
4501     my $sockets = 1;
4502     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
4503     $sockets = $conf->{sockets} if  $conf->{sockets};
4504     my $cores = $conf->{cores} || 1;
4505     my $maxcpus = $sockets * $cores;
4506
4507     $vcpus = $maxcpus if !$vcpus;
4508
4509     die "you can't add more vcpus than maxcpus\n"
4510         if $vcpus > $maxcpus;
4511
4512     my $currentvcpus = $conf->{vcpus} || $maxcpus;
4513
4514     if ($vcpus < $currentvcpus) {
4515
4516         if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4517
4518             for (my $i = $currentvcpus; $i > $vcpus; $i--) {
4519                 qemu_devicedel($vmid, "cpu$i");
4520                 my $retry = 0;
4521                 my $currentrunningvcpus = undef;
4522                 while (1) {
4523                     $currentrunningvcpus = mon_cmd($vmid, "query-cpus");
4524                     last if scalar(@{$currentrunningvcpus}) == $i-1;
4525                     raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
4526                     $retry++;
4527                     sleep 1;
4528                 }
4529                 #update conf after each succesfull cpu unplug
4530                 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4531                 PVE::QemuConfig->write_config($vmid, $conf);
4532             }
4533         } else {
4534             die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
4535         }
4536
4537         return;
4538     }
4539
4540     my $currentrunningvcpus = mon_cmd($vmid, "query-cpus");
4541     die "vcpus in running vm does not match its configuration\n"
4542         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
4543
4544     if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4545
4546         for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
4547             my $cpustr = print_cpu_device($conf, $i);
4548             qemu_deviceadd($vmid, $cpustr);
4549
4550             my $retry = 0;
4551             my $currentrunningvcpus = undef;
4552             while (1) {
4553                 $currentrunningvcpus = mon_cmd($vmid, "query-cpus");
4554                 last if scalar(@{$currentrunningvcpus}) == $i;
4555                 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
4556                 sleep 1;
4557                 $retry++;
4558             }
4559             #update conf after each succesfull cpu hotplug
4560             $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4561             PVE::QemuConfig->write_config($vmid, $conf);
4562         }
4563     } else {
4564
4565         for (my $i = $currentvcpus; $i < $vcpus; $i++) {
4566             mon_cmd($vmid, "cpu-add", id => int($i));
4567         }
4568     }
4569 }
4570
4571 sub qemu_block_set_io_throttle {
4572     my ($vmid, $deviceid,
4573         $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
4574         $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max,
4575         $bps_max_length, $bps_rd_max_length, $bps_wr_max_length,
4576         $iops_max_length, $iops_rd_max_length, $iops_wr_max_length) = @_;
4577
4578     return if !check_running($vmid) ;
4579
4580     mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
4581         bps => int($bps),
4582         bps_rd => int($bps_rd),
4583         bps_wr => int($bps_wr),
4584         iops => int($iops),
4585         iops_rd => int($iops_rd),
4586         iops_wr => int($iops_wr),
4587         bps_max => int($bps_max),
4588         bps_rd_max => int($bps_rd_max),
4589         bps_wr_max => int($bps_wr_max),
4590         iops_max => int($iops_max),
4591         iops_rd_max => int($iops_rd_max),
4592         iops_wr_max => int($iops_wr_max),
4593         bps_max_length => int($bps_max_length),
4594         bps_rd_max_length => int($bps_rd_max_length),
4595         bps_wr_max_length => int($bps_wr_max_length),
4596         iops_max_length => int($iops_max_length),
4597         iops_rd_max_length => int($iops_rd_max_length),
4598         iops_wr_max_length => int($iops_wr_max_length),
4599     );
4600
4601 }
4602
4603 # old code, only used to shutdown old VM after update
4604 sub __read_avail {
4605     my ($fh, $timeout) = @_;
4606
4607     my $sel = new IO::Select;
4608     $sel->add($fh);
4609
4610     my $res = '';
4611     my $buf;
4612
4613     my @ready;
4614     while (scalar (@ready = $sel->can_read($timeout))) {
4615         my $count;
4616         if ($count = $fh->sysread($buf, 8192)) {
4617             if ($buf =~ /^(.*)\(qemu\) $/s) {
4618                 $res .= $1;
4619                 last;
4620             } else {
4621                 $res .= $buf;
4622             }
4623         } else {
4624             if (!defined($count)) {
4625                 die "$!\n";
4626             }
4627             last;
4628         }
4629     }
4630
4631     die "monitor read timeout\n" if !scalar(@ready);
4632
4633     return $res;
4634 }
4635
4636 sub qemu_block_resize {
4637     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4638
4639     my $running = check_running($vmid);
4640
4641     $size = 0 if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4642
4643     return if !$running;
4644
4645     mon_cmd($vmid, "block_resize", device => $deviceid, size => int($size));
4646
4647 }
4648
4649 sub qemu_volume_snapshot {
4650     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4651
4652     my $running = check_running($vmid);
4653
4654     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4655         mon_cmd($vmid, 'blockdev-snapshot-internal-sync', device => $deviceid, name => $snap);
4656     } else {
4657         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4658     }
4659 }
4660
4661 sub qemu_volume_snapshot_delete {
4662     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4663
4664     my $running = check_running($vmid);
4665
4666     if($running) {
4667
4668         $running = undef;
4669         my $conf = PVE::QemuConfig->load_config($vmid);
4670         foreach_drive($conf, sub {
4671             my ($ds, $drive) = @_;
4672             $running = 1 if $drive->{file} eq $volid;
4673         });
4674     }
4675
4676     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4677         mon_cmd($vmid, 'blockdev-snapshot-delete-internal-sync', device => $deviceid, name => $snap);
4678     } else {
4679         PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4680     }
4681 }
4682
4683 sub set_migration_caps {
4684     my ($vmid) = @_;
4685
4686     my $cap_ref = [];
4687
4688     my $enabled_cap = {
4689         "auto-converge" => 1,
4690         "xbzrle" => 1,
4691         "x-rdma-pin-all" => 0,
4692         "zero-blocks" => 0,
4693         "compress" => 0
4694     };
4695
4696     my $supported_capabilities = mon_cmd($vmid, "query-migrate-capabilities");
4697
4698     for my $supported_capability (@$supported_capabilities) {
4699         push @$cap_ref, {
4700             capability => $supported_capability->{capability},
4701             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4702         };
4703     }
4704
4705     mon_cmd($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4706 }
4707
4708 my $fast_plug_option = {
4709     'lock' => 1,
4710     'name' => 1,
4711     'onboot' => 1,
4712     'shares' => 1,
4713     'startup' => 1,
4714     'description' => 1,
4715     'protection' => 1,
4716     'vmstatestorage' => 1,
4717     'hookscript' => 1,
4718 };
4719
4720 # hotplug changes in [PENDING]
4721 # $selection hash can be used to only apply specified options, for
4722 # example: { cores => 1 } (only apply changed 'cores')
4723 # $errors ref is used to return error messages
4724 sub vmconfig_hotplug_pending {
4725     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4726
4727     my $defaults = load_defaults();
4728     my ($arch, $machine_type) = get_basic_machine_info($conf, undef);
4729
4730     # commit values which do not have any impact on running VM first
4731     # Note: those option cannot raise errors, we we do not care about
4732     # $selection and always apply them.
4733
4734     my $add_error = sub {
4735         my ($opt, $msg) = @_;
4736         $errors->{$opt} = "hotplug problem - $msg";
4737     };
4738
4739     my $changes = 0;
4740     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4741         if ($fast_plug_option->{$opt}) {
4742             $conf->{$opt} = $conf->{pending}->{$opt};
4743             delete $conf->{pending}->{$opt};
4744             $changes = 1;
4745         }
4746     }
4747
4748     if ($changes) {
4749         PVE::QemuConfig->write_config($vmid, $conf);
4750         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4751     }
4752
4753     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4754
4755     my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4756     foreach my $opt (sort keys %$pending_delete_hash) {
4757         next if $selection && !$selection->{$opt};
4758         my $force = $pending_delete_hash->{$opt}->{force};
4759         eval {
4760             if ($opt eq 'hotplug') {
4761                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
4762             } elsif ($opt eq 'tablet') {
4763                 die "skip\n" if !$hotplug_features->{usb};
4764                 if ($defaults->{tablet}) {
4765                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4766                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4767                         if $arch eq 'aarch64';
4768                 } else {
4769                     vm_deviceunplug($vmid, $conf, 'tablet');
4770                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4771                 }
4772             } elsif ($opt =~ m/^usb\d+/) {
4773                 die "skip\n";
4774                 # since we cannot reliably hot unplug usb devices
4775                 # we are disabling it
4776                 die "skip\n" if !$hotplug_features->{usb} || $conf->{$opt} =~ m/spice/i;
4777                 vm_deviceunplug($vmid, $conf, $opt);
4778             } elsif ($opt eq 'vcpus') {
4779                 die "skip\n" if !$hotplug_features->{cpu};
4780                 qemu_cpu_hotplug($vmid, $conf, undef);
4781             } elsif ($opt eq 'balloon') {
4782                 # enable balloon device is not hotpluggable
4783                 die "skip\n" if defined($conf->{balloon}) && $conf->{balloon} == 0;
4784                 # here we reset the ballooning value to memory
4785                 my $balloon = $conf->{memory} || $defaults->{memory};
4786                 mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4787             } elsif ($fast_plug_option->{$opt}) {
4788                 # do nothing
4789             } elsif ($opt =~ m/^net(\d+)$/) {
4790                 die "skip\n" if !$hotplug_features->{network};
4791                 vm_deviceunplug($vmid, $conf, $opt);
4792             } elsif (is_valid_drivename($opt)) {
4793                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4794                 vm_deviceunplug($vmid, $conf, $opt);
4795                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4796             } elsif ($opt =~ m/^memory$/) {
4797                 die "skip\n" if !$hotplug_features->{memory};
4798                 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
4799             } elsif ($opt eq 'cpuunits') {
4800                 cgroups_write("cpu", $vmid, "cpu.shares", $defaults->{cpuunits});
4801             } elsif ($opt eq 'cpulimit') {
4802                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", -1);
4803             } else {
4804                 die "skip\n";
4805             }
4806         };
4807         if (my $err = $@) {
4808             &$add_error($opt, $err) if $err ne "skip\n";
4809         } else {
4810             # save new config if hotplug was successful
4811             delete $conf->{$opt};
4812             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4813             PVE::QemuConfig->write_config($vmid, $conf);
4814             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4815         }
4816     }
4817
4818     my ($apply_pending_cloudinit, $apply_pending_cloudinit_done);
4819     $apply_pending_cloudinit = sub {
4820         return if $apply_pending_cloudinit_done; # once is enough
4821         $apply_pending_cloudinit_done = 1; # once is enough
4822
4823         my ($key, $value) = @_;
4824
4825         my @cloudinit_opts = keys %$confdesc_cloudinit;
4826         foreach my $opt (keys %{$conf->{pending}}) {
4827             next if !grep { $_ eq $opt } @cloudinit_opts;
4828             $conf->{$opt} = delete $conf->{pending}->{$opt};
4829         }
4830
4831         my $new_conf = { %$conf };
4832         $new_conf->{$key} = $value;
4833         PVE::QemuServer::Cloudinit::generate_cloudinitconfig($new_conf, $vmid);
4834     };
4835
4836     foreach my $opt (keys %{$conf->{pending}}) {
4837         next if $selection && !$selection->{$opt};
4838         my $value = $conf->{pending}->{$opt};
4839         eval {
4840             if ($opt eq 'hotplug') {
4841                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
4842             } elsif ($opt eq 'tablet') {
4843                 die "skip\n" if !$hotplug_features->{usb};
4844                 if ($value == 1) {
4845                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4846                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4847                         if $arch eq 'aarch64';
4848                 } elsif ($value == 0) {
4849                     vm_deviceunplug($vmid, $conf, 'tablet');
4850                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4851                 }
4852             } elsif ($opt =~ m/^usb\d+$/) {
4853                 die "skip\n";
4854                 # since we cannot reliably hot unplug usb devices
4855                 # we are disabling it
4856                 die "skip\n" if !$hotplug_features->{usb} || $value =~ m/spice/i;
4857                 my $d = eval { PVE::JSONSchema::parse_property_string($usbdesc->{format}, $value) };
4858                 die "skip\n" if !$d;
4859                 qemu_usb_hotplug($storecfg, $conf, $vmid, $opt, $d, $arch, $machine_type);
4860             } elsif ($opt eq 'vcpus') {
4861                 die "skip\n" if !$hotplug_features->{cpu};
4862                 qemu_cpu_hotplug($vmid, $conf, $value);
4863             } elsif ($opt eq 'balloon') {
4864                 # enable/disable balloning device is not hotpluggable
4865                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
4866                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
4867                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
4868
4869                 # allow manual ballooning if shares is set to zero
4870                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
4871                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
4872                     mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4873                 }
4874             } elsif ($opt =~ m/^net(\d+)$/) {
4875                 # some changes can be done without hotplug
4876                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
4877                                     $vmid, $opt, $value, $arch, $machine_type);
4878             } elsif (is_valid_drivename($opt)) {
4879                 # some changes can be done without hotplug
4880                 my $drive = parse_drive($opt, $value);
4881                 if (drive_is_cloudinit($drive)) {
4882                     &$apply_pending_cloudinit($opt, $value);
4883                 }
4884                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
4885                                      $vmid, $opt, $value, 1, $arch, $machine_type);
4886             } elsif ($opt =~ m/^memory$/) { #dimms
4887                 die "skip\n" if !$hotplug_features->{memory};
4888                 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
4889             } elsif ($opt eq 'cpuunits') {
4890                 cgroups_write("cpu", $vmid, "cpu.shares", $conf->{pending}->{$opt});
4891             } elsif ($opt eq 'cpulimit') {
4892                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
4893                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", $cpulimit);
4894             } else {
4895                 die "skip\n";  # skip non-hot-pluggable options
4896             }
4897         };
4898         if (my $err = $@) {
4899             &$add_error($opt, $err) if $err ne "skip\n";
4900         } else {
4901             # save new config if hotplug was successful
4902             $conf->{$opt} = $value;
4903             delete $conf->{pending}->{$opt};
4904             PVE::QemuConfig->write_config($vmid, $conf);
4905             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4906         }
4907     }
4908 }
4909
4910 sub try_deallocate_drive {
4911     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
4912
4913     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
4914         my $volid = $drive->{file};
4915         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
4916             my $sid = PVE::Storage::parse_volume_id($volid);
4917             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
4918
4919             # check if the disk is really unused
4920             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
4921                 if is_volume_in_use($storecfg, $conf, $key, $volid);
4922             PVE::Storage::vdisk_free($storecfg, $volid);
4923             return 1;
4924         } else {
4925             # If vm is not owner of this disk remove from config
4926             return 1;
4927         }
4928     }
4929
4930     return undef;
4931 }
4932
4933 sub vmconfig_delete_or_detach_drive {
4934     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
4935
4936     my $drive = parse_drive($opt, $conf->{$opt});
4937
4938     my $rpcenv = PVE::RPCEnvironment::get();
4939     my $authuser = $rpcenv->get_user();
4940
4941     if ($force) {
4942         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
4943         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
4944     } else {
4945         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
4946     }
4947 }
4948
4949
4950
4951 sub vmconfig_apply_pending {
4952     my ($vmid, $conf, $storecfg) = @_;
4953
4954     # cold plug
4955
4956     my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4957     foreach my $opt (sort keys %$pending_delete_hash) {
4958         die "internal error" if $opt =~ m/^unused/;
4959         my $force = $pending_delete_hash->{$opt}->{force};
4960         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4961         if (!defined($conf->{$opt})) {
4962             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4963             PVE::QemuConfig->write_config($vmid, $conf);
4964         } elsif (is_valid_drivename($opt)) {
4965             vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4966             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4967             delete $conf->{$opt};
4968             PVE::QemuConfig->write_config($vmid, $conf);
4969         } else {
4970             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4971             delete $conf->{$opt};
4972             PVE::QemuConfig->write_config($vmid, $conf);
4973         }
4974     }
4975
4976     $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4977
4978     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4979         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4980
4981         if (defined($conf->{$opt}) && ($conf->{$opt} eq $conf->{pending}->{$opt})) {
4982             # skip if nothing changed
4983         } elsif (is_valid_drivename($opt)) {
4984             vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
4985                 if defined($conf->{$opt});
4986             $conf->{$opt} = $conf->{pending}->{$opt};
4987         } else {
4988             $conf->{$opt} = $conf->{pending}->{$opt};
4989         }
4990
4991         delete $conf->{pending}->{$opt};
4992         PVE::QemuConfig->write_config($vmid, $conf);
4993     }
4994 }
4995
4996 my $safe_num_ne = sub {
4997     my ($a, $b) = @_;
4998
4999     return 0 if !defined($a) && !defined($b);
5000     return 1 if !defined($a);
5001     return 1 if !defined($b);
5002
5003     return $a != $b;
5004 };
5005
5006 my $safe_string_ne = sub {
5007     my ($a, $b) = @_;
5008
5009     return 0 if !defined($a) && !defined($b);
5010     return 1 if !defined($a);
5011     return 1 if !defined($b);
5012
5013     return $a ne $b;
5014 };
5015
5016 sub vmconfig_update_net {
5017     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
5018
5019     my $newnet = parse_net($value);
5020
5021     if ($conf->{$opt}) {
5022         my $oldnet = parse_net($conf->{$opt});
5023
5024         if (&$safe_string_ne($oldnet->{model}, $newnet->{model}) ||
5025             &$safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
5026             &$safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
5027             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
5028
5029             # for non online change, we try to hot-unplug
5030             die "skip\n" if !$hotplug;
5031             vm_deviceunplug($vmid, $conf, $opt);
5032         } else {
5033
5034             die "internal error" if $opt !~ m/net(\d+)/;
5035             my $iface = "tap${vmid}i$1";
5036
5037             if (&$safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
5038                 &$safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
5039                 &$safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
5040                 &$safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
5041                 PVE::Network::tap_unplug($iface);
5042                 PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
5043             } elsif (&$safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
5044                 # Rate can be applied on its own but any change above needs to
5045                 # include the rate in tap_plug since OVS resets everything.
5046                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
5047             }
5048
5049             if (&$safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
5050                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
5051             }
5052
5053             return 1;
5054         }
5055     }
5056
5057     if ($hotplug) {
5058         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet, $arch, $machine_type);
5059     } else {
5060         die "skip\n";
5061     }
5062 }
5063
5064 sub vmconfig_update_disk {
5065     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $force, $arch, $machine_type) = @_;
5066
5067     # fixme: do we need force?
5068
5069     my $drive = parse_drive($opt, $value);
5070
5071     if ($conf->{$opt}) {
5072
5073         if (my $old_drive = parse_drive($opt, $conf->{$opt}))  {
5074
5075             my $media = $drive->{media} || 'disk';
5076             my $oldmedia = $old_drive->{media} || 'disk';
5077             die "unable to change media type\n" if $media ne $oldmedia;
5078
5079             if (!drive_is_cdrom($old_drive)) {
5080
5081                 if ($drive->{file} ne $old_drive->{file}) {
5082
5083                     die "skip\n" if !$hotplug;
5084
5085                     # unplug and register as unused
5086                     vm_deviceunplug($vmid, $conf, $opt);
5087                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
5088
5089                 } else {
5090                     # update existing disk
5091
5092                     # skip non hotpluggable value
5093                     if (&$safe_string_ne($drive->{discard}, $old_drive->{discard}) ||
5094                         &$safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
5095                         &$safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
5096                         &$safe_string_ne($drive->{cache}, $old_drive->{cache})) {
5097                         die "skip\n";
5098                     }
5099
5100                     # apply throttle
5101                     if (&$safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
5102                         &$safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
5103                         &$safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
5104                         &$safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
5105                         &$safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
5106                         &$safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
5107                         &$safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
5108                         &$safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
5109                         &$safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
5110                         &$safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
5111                         &$safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
5112                         &$safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max}) ||
5113                         &$safe_num_ne($drive->{bps_max_length}, $old_drive->{bps_max_length}) ||
5114                         &$safe_num_ne($drive->{bps_rd_max_length}, $old_drive->{bps_rd_max_length}) ||
5115                         &$safe_num_ne($drive->{bps_wr_max_length}, $old_drive->{bps_wr_max_length}) ||
5116                         &$safe_num_ne($drive->{iops_max_length}, $old_drive->{iops_max_length}) ||
5117                         &$safe_num_ne($drive->{iops_rd_max_length}, $old_drive->{iops_rd_max_length}) ||
5118                         &$safe_num_ne($drive->{iops_wr_max_length}, $old_drive->{iops_wr_max_length})) {
5119
5120                         qemu_block_set_io_throttle($vmid,"drive-$opt",
5121                                                    ($drive->{mbps} || 0)*1024*1024,
5122                                                    ($drive->{mbps_rd} || 0)*1024*1024,
5123                                                    ($drive->{mbps_wr} || 0)*1024*1024,
5124                                                    $drive->{iops} || 0,
5125                                                    $drive->{iops_rd} || 0,
5126                                                    $drive->{iops_wr} || 0,
5127                                                    ($drive->{mbps_max} || 0)*1024*1024,
5128                                                    ($drive->{mbps_rd_max} || 0)*1024*1024,
5129                                                    ($drive->{mbps_wr_max} || 0)*1024*1024,
5130                                                    $drive->{iops_max} || 0,
5131                                                    $drive->{iops_rd_max} || 0,
5132                                                    $drive->{iops_wr_max} || 0,
5133                                                    $drive->{bps_max_length} || 1,
5134                                                    $drive->{bps_rd_max_length} || 1,
5135                                                    $drive->{bps_wr_max_length} || 1,
5136                                                    $drive->{iops_max_length} || 1,
5137                                                    $drive->{iops_rd_max_length} || 1,
5138                                                    $drive->{iops_wr_max_length} || 1);
5139
5140                     }
5141
5142                     return 1;
5143                 }
5144
5145             } else { # cdrom
5146
5147                 if ($drive->{file} eq 'none') {
5148                     mon_cmd($vmid, "eject",force => JSON::true,device => "drive-$opt");
5149                     if (drive_is_cloudinit($old_drive)) {
5150                         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive);
5151                     }
5152                 } else {
5153                     my $path = get_iso_path($storecfg, $vmid, $drive->{file});
5154                     mon_cmd($vmid, "eject", force => JSON::true,device => "drive-$opt"); # force eject if locked
5155                     mon_cmd($vmid, "change", device => "drive-$opt",target => "$path") if $path;
5156                 }
5157
5158                 return 1;
5159             }
5160         }
5161     }
5162
5163     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
5164     # hotplug new disks
5165     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
5166     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive, $arch, $machine_type);
5167 }
5168
5169 sub vm_start {
5170     my ($storecfg, $vmid, $statefile, $skiplock, $migratedfrom, $paused,
5171         $forcemachine, $spice_ticket, $migration_network, $migration_type, $targetstorage) = @_;
5172
5173     PVE::QemuConfig->lock_config($vmid, sub {
5174         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
5175
5176         die "you can't start a vm if it's a template\n" if PVE::QemuConfig->is_template($conf);
5177
5178         my $is_suspended = PVE::QemuConfig->has_lock($conf, 'suspended');
5179
5180         PVE::QemuConfig->check_lock($conf)
5181             if !($skiplock || $is_suspended);
5182
5183         die "VM $vmid already running\n" if check_running($vmid, undef, $migratedfrom);
5184
5185         # clean up leftover reboot request files
5186         eval { clear_reboot_request($vmid); };
5187         warn $@ if $@;
5188
5189         if (!$statefile && scalar(keys %{$conf->{pending}})) {
5190             vmconfig_apply_pending($vmid, $conf, $storecfg);
5191             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5192         }
5193
5194         PVE::QemuServer::Cloudinit::generate_cloudinitconfig($conf, $vmid);
5195
5196         my $defaults = load_defaults();
5197
5198         # set environment variable useful inside network script
5199         $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
5200
5201         my $local_volumes = {};
5202
5203         if ($targetstorage) {
5204             foreach_drive($conf, sub {
5205                 my ($ds, $drive) = @_;
5206
5207                 return if drive_is_cdrom($drive);
5208
5209                 my $volid = $drive->{file};
5210
5211                 return if !$volid;
5212
5213                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
5214
5215                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5216                 return if $scfg->{shared};
5217                 $local_volumes->{$ds} = [$volid, $storeid, $volname];
5218             });
5219
5220             my $format = undef;
5221
5222             foreach my $opt (sort keys %$local_volumes) {
5223
5224                 my ($volid, $storeid, $volname) = @{$local_volumes->{$opt}};
5225                 my $drive = parse_drive($opt, $conf->{$opt});
5226
5227                 #if remote storage is specified, use default format
5228                 if ($targetstorage && $targetstorage ne "1") {
5229                     $storeid = $targetstorage;
5230                     my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
5231                     $format = $defFormat;
5232                 } else {
5233                     #else we use same format than original
5234                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5235                     $format = qemu_img_format($scfg, $volid);
5236                 }
5237
5238                 my $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $format, undef, ($drive->{size}/1024));
5239                 my $newdrive = $drive;
5240                 $newdrive->{format} = $format;
5241                 $newdrive->{file} = $newvolid;
5242                 my $drivestr = PVE::QemuServer::print_drive($vmid, $newdrive);
5243                 $local_volumes->{$opt} = $drivestr;
5244                 #pass drive to conf for command line
5245                 $conf->{$opt} = $drivestr;
5246             }
5247         }
5248
5249         PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-start', 1);
5250
5251         if ($is_suspended) {
5252             # enforce machine type on suspended vm to ensure HW compatibility
5253             $forcemachine = $conf->{runningmachine};
5254             print "Resuming suspended VM\n";
5255         }
5256
5257         my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine);
5258
5259         my $migration_ip;
5260         my $get_migration_ip = sub {
5261             my ($cidr, $nodename) = @_;
5262
5263             return $migration_ip if defined($migration_ip);
5264
5265             if (!defined($cidr)) {
5266                 my $dc_conf = PVE::Cluster::cfs_read_file('datacenter.cfg');
5267                 $cidr = $dc_conf->{migration}->{network};
5268             }
5269
5270             if (defined($cidr)) {
5271                 my $ips = PVE::Network::get_local_ip_from_cidr($cidr);
5272
5273                 die "could not get IP: no address configured on local " .
5274                     "node for network '$cidr'\n" if scalar(@$ips) == 0;
5275
5276                 die "could not get IP: multiple addresses configured on local " .
5277                     "node for network '$cidr'\n" if scalar(@$ips) > 1;
5278
5279                 $migration_ip = @$ips[0];
5280             }
5281
5282             $migration_ip = PVE::Cluster::remote_node_ip($nodename, 1)
5283                 if !defined($migration_ip);
5284
5285             return $migration_ip;
5286         };
5287
5288         my $migrate_uri;
5289         if ($statefile) {
5290             if ($statefile eq 'tcp') {
5291                 my $localip = "localhost";
5292                 my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
5293                 my $nodename = PVE::INotify::nodename();
5294
5295                 if (!defined($migration_type)) {
5296                     if (defined($datacenterconf->{migration}->{type})) {
5297                         $migration_type = $datacenterconf->{migration}->{type};
5298                     } else {
5299                         $migration_type = 'secure';
5300                     }
5301                 }
5302
5303                 if ($migration_type eq 'insecure') {
5304                     $localip = $get_migration_ip->($migration_network, $nodename);
5305                     $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5306                 }
5307
5308                 my $pfamily = PVE::Tools::get_host_address_family($nodename);
5309                 my $migrate_port = PVE::Tools::next_migrate_port($pfamily);
5310                 $migrate_uri = "tcp:${localip}:${migrate_port}";
5311                 push @$cmd, '-incoming', $migrate_uri;
5312                 push @$cmd, '-S';
5313
5314             } elsif ($statefile eq 'unix') {
5315                 # should be default for secure migrations as a ssh TCP forward
5316                 # tunnel is not deterministic reliable ready and fails regurarly
5317                 # to set up in time, so use UNIX socket forwards
5318                 my $socket_addr = "/run/qemu-server/$vmid.migrate";
5319                 unlink $socket_addr;
5320
5321                 $migrate_uri = "unix:$socket_addr";
5322
5323                 push @$cmd, '-incoming', $migrate_uri;
5324                 push @$cmd, '-S';
5325
5326             } elsif (-e $statefile) {
5327                 push @$cmd, '-loadstate', $statefile;
5328             } else {
5329                 my $statepath = PVE::Storage::path($storecfg, $statefile);
5330                 push @$vollist, $statefile;
5331                 push @$cmd, '-loadstate', $statepath;
5332             }
5333         } elsif ($paused) {
5334             push @$cmd, '-S';
5335         }
5336
5337         # host pci devices
5338         for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
5339           my $d = parse_hostpci($conf->{"hostpci$i"});
5340           next if !$d;
5341           my $pcidevices = $d->{pciid};
5342           foreach my $pcidevice (@$pcidevices) {
5343                 my $pciid = $pcidevice->{id};
5344
5345                 my $info = PVE::SysFSTools::pci_device_info("0000:$pciid");
5346                 die "IOMMU not present\n" if !PVE::SysFSTools::check_iommu_support();
5347                 die "no pci device info for device '$pciid'\n" if !$info;
5348
5349                 if ($d->{mdev}) {
5350                     my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $i);
5351                     PVE::SysFSTools::pci_create_mdev_device($pciid, $uuid, $d->{mdev});
5352                 } else {
5353                     die "can't unbind/bind pci group to vfio '$pciid'\n"
5354                         if !PVE::SysFSTools::pci_dev_group_bind_to_vfio($pciid);
5355                     die "can't reset pci device '$pciid'\n"
5356                         if $info->{has_fl_reset} and !PVE::SysFSTools::pci_dev_reset($info);
5357                 }
5358           }
5359         }
5360
5361         PVE::Storage::activate_volumes($storecfg, $vollist);
5362
5363         eval {
5364             run_command(['/bin/systemctl', 'stop', "$vmid.scope"],
5365                 outfunc => sub {}, errfunc => sub {});
5366         };
5367         # Issues with the above 'stop' not being fully completed are extremely rare, a very low
5368         # timeout should be more than enough here...
5369         PVE::Systemd::wait_for_unit_removed("$vmid.scope", 5);
5370
5371         my $cpuunits = defined($conf->{cpuunits}) ? $conf->{cpuunits}
5372                                                   : $defaults->{cpuunits};
5373
5374         my $start_timeout = ($conf->{hugepages} || $is_suspended) ? 300 : 30;
5375         my %run_params = (timeout => $statefile ? undef : $start_timeout, umask => 0077);
5376
5377         my %properties = (
5378             Slice => 'qemu.slice',
5379             KillMode => 'none',
5380             CPUShares => $cpuunits
5381         );
5382
5383         if (my $cpulimit = $conf->{cpulimit}) {
5384             $properties{CPUQuota} = int($cpulimit * 100);
5385         }
5386         $properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
5387
5388         my $run_qemu = sub {
5389             PVE::Tools::run_fork sub {
5390                 PVE::Systemd::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
5391                 run_command($cmd, %run_params);
5392             };
5393         };
5394
5395         if ($conf->{hugepages}) {
5396
5397             my $code = sub {
5398                 my $hugepages_topology = PVE::QemuServer::Memory::hugepages_topology($conf);
5399                 my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
5400
5401                 PVE::QemuServer::Memory::hugepages_mount();
5402                 PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
5403
5404                 eval { $run_qemu->() };
5405                 if (my $err = $@) {
5406                     PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology);
5407                     die $err;
5408                 }
5409
5410                 PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology);
5411             };
5412             eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
5413
5414         } else {
5415             eval { $run_qemu->() };
5416         }
5417
5418         if (my $err = $@) {
5419             # deactivate volumes if start fails
5420             eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
5421             die "start failed: $err";
5422         }
5423
5424         print "migration listens on $migrate_uri\n" if $migrate_uri;
5425
5426         if ($statefile && $statefile ne 'tcp' && $statefile ne 'unix')  {
5427             eval { mon_cmd($vmid, "cont"); };
5428             warn $@ if $@;
5429         }
5430
5431         #start nbd server for storage migration
5432         if ($targetstorage) {
5433             my $nodename = PVE::INotify::nodename();
5434             my $localip = $get_migration_ip->($migration_network, $nodename);
5435             my $pfamily = PVE::Tools::get_host_address_family($nodename);
5436             my $storage_migrate_port = PVE::Tools::next_migrate_port($pfamily);
5437
5438             mon_cmd($vmid, "nbd-server-start", addr => { type => 'inet', data => { host => "${localip}", port => "${storage_migrate_port}" } } );
5439
5440             $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5441
5442             foreach my $opt (sort keys %$local_volumes) {
5443                 my $volid = $local_volumes->{$opt};
5444                 mon_cmd($vmid, "nbd-server-add", device => "drive-$opt", writable => JSON::true );
5445                 my $migrate_storage_uri = "nbd:${localip}:${storage_migrate_port}:exportname=drive-$opt";
5446                 print "storage migration listens on $migrate_storage_uri volume:$volid\n";
5447             }
5448         }
5449
5450         if ($migratedfrom) {
5451             eval {
5452                 set_migration_caps($vmid);
5453             };
5454             warn $@ if $@;
5455
5456             if ($spice_port) {
5457                 print "spice listens on port $spice_port\n";
5458                 if ($spice_ticket) {
5459                     mon_cmd($vmid, "set_password", protocol => 'spice', password => $spice_ticket);
5460                     mon_cmd($vmid, "expire_password", protocol => 'spice', time => "+30");
5461                 }
5462             }
5463
5464         } else {
5465             mon_cmd($vmid, "balloon", value => $conf->{balloon}*1024*1024)
5466                 if !$statefile && $conf->{balloon};
5467
5468             foreach my $opt (keys %$conf) {
5469                 next if $opt !~  m/^net\d+$/;
5470                 my $nicconf = parse_net($conf->{$opt});
5471                 qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
5472             }
5473         }
5474
5475         mon_cmd($vmid, 'qom-set',
5476                     path => "machine/peripheral/balloon0",
5477                     property => "guest-stats-polling-interval",
5478                     value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
5479
5480         if ($is_suspended && (my $vmstate = $conf->{vmstate})) {
5481             print "Resumed VM, removing state\n";
5482             delete $conf->@{qw(lock vmstate runningmachine)};
5483             PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
5484             PVE::Storage::vdisk_free($storecfg, $vmstate);
5485             PVE::QemuConfig->write_config($vmid, $conf);
5486         }
5487
5488         PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'post-start');
5489     });
5490 }
5491
5492 sub vm_commandline {
5493     my ($storecfg, $vmid, $snapname) = @_;
5494
5495     my $conf = PVE::QemuConfig->load_config($vmid);
5496     my $forcemachine;
5497
5498     if ($snapname) {
5499         my $snapshot = $conf->{snapshots}->{$snapname};
5500         die "snapshot '$snapname' does not exist\n" if !defined($snapshot);
5501
5502         # check for a 'runningmachine' in snapshot
5503         $forcemachine = $snapshot->{runningmachine} if $snapshot->{runningmachine};
5504
5505         $snapshot->{digest} = $conf->{digest}; # keep file digest for API
5506
5507         $conf = $snapshot;
5508     }
5509
5510     my $defaults = load_defaults();
5511
5512     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine);
5513
5514     return PVE::Tools::cmd2string($cmd);
5515 }
5516
5517 sub vm_reset {
5518     my ($vmid, $skiplock) = @_;
5519
5520     PVE::QemuConfig->lock_config($vmid, sub {
5521
5522         my $conf = PVE::QemuConfig->load_config($vmid);
5523
5524         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5525
5526         mon_cmd($vmid, "system_reset");
5527     });
5528 }
5529
5530 sub get_vm_volumes {
5531     my ($conf) = @_;
5532
5533     my $vollist = [];
5534     foreach_volid($conf, sub {
5535         my ($volid, $attr) = @_;
5536
5537         return if $volid =~ m|^/|;
5538
5539         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5540         return if !$sid;
5541
5542         push @$vollist, $volid;
5543     });
5544
5545     return $vollist;
5546 }
5547
5548 sub vm_stop_cleanup {
5549     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
5550
5551     eval {
5552
5553         if (!$keepActive) {
5554             my $vollist = get_vm_volumes($conf);
5555             PVE::Storage::deactivate_volumes($storecfg, $vollist);
5556         }
5557
5558         foreach my $ext (qw(mon qmp pid vnc qga)) {
5559             unlink "/var/run/qemu-server/${vmid}.$ext";
5560         }
5561
5562         if ($conf->{ivshmem}) {
5563             my $ivshmem = PVE::JSONSchema::parse_property_string($ivshmem_fmt, $conf->{ivshmem});
5564             # just delete it for now, VMs which have this already open do not
5565             # are affected, but new VMs will get a separated one. If this
5566             # becomes an issue we either add some sort of ref-counting or just
5567             # add a "don't delete on stop" flag to the ivshmem format.
5568             unlink '/dev/shm/pve-shm-' . ($ivshmem->{name} // $vmid);
5569         }
5570
5571         foreach my $key (keys %$conf) {
5572             next if $key !~ m/^hostpci(\d+)$/;
5573             my $hostpciindex = $1;
5574             my $d = parse_hostpci($conf->{$key});
5575             my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $hostpciindex);
5576
5577             foreach my $pci (@{$d->{pciid}}) {
5578                 my $pciid = $pci->{id};
5579                 PVE::SysFSTools::pci_cleanup_mdev_device($pciid, $uuid);
5580             }
5581         }
5582
5583         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
5584     };
5585     warn $@ if $@; # avoid errors - just warn
5586 }
5587
5588 # call only in locked context
5589 sub _do_vm_stop {
5590     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive) = @_;
5591
5592     my $pid = check_running($vmid, $nocheck);
5593     return if !$pid;
5594
5595     my $conf;
5596     if (!$nocheck) {
5597         $conf = PVE::QemuConfig->load_config($vmid);
5598         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5599         if (!defined($timeout) && $shutdown && $conf->{startup}) {
5600             my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
5601             $timeout = $opts->{down} if $opts->{down};
5602         }
5603         PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-stop');
5604     }
5605
5606     eval {
5607         if ($shutdown) {
5608             if (defined($conf) && parse_guest_agent($conf)->{enabled}) {
5609                 mon_cmd($vmid, "guest-shutdown", timeout => $timeout);
5610             } else {
5611                 mon_cmd($vmid, "system_powerdown");
5612             }
5613         } else {
5614             mon_cmd($vmid, "quit");
5615         }
5616     };
5617     my $err = $@;
5618
5619     if (!$err) {
5620         $timeout = 60 if !defined($timeout);
5621
5622         my $count = 0;
5623         while (($count < $timeout) && check_running($vmid, $nocheck)) {
5624             $count++;
5625             sleep 1;
5626         }
5627
5628         if ($count >= $timeout) {
5629             if ($force) {
5630                 warn "VM still running - terminating now with SIGTERM\n";
5631                 kill 15, $pid;
5632             } else {
5633                 die "VM quit/powerdown failed - got timeout\n";
5634             }
5635         } else {
5636             vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5637             return;
5638         }
5639     } else {
5640         if ($force) {
5641             warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
5642             kill 15, $pid;
5643         } else {
5644             die "VM quit/powerdown failed\n";
5645         }
5646     }
5647
5648     # wait again
5649     $timeout = 10;
5650
5651     my $count = 0;
5652     while (($count < $timeout) && check_running($vmid, $nocheck)) {
5653         $count++;
5654         sleep 1;
5655     }
5656
5657     if ($count >= $timeout) {
5658         warn "VM still running - terminating now with SIGKILL\n";
5659         kill 9, $pid;
5660         sleep 1;
5661     }
5662
5663     vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5664 }
5665
5666 # Note: use $nocheck to skip tests if VM configuration file exists.
5667 # We need that when migration VMs to other nodes (files already moved)
5668 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
5669 sub vm_stop {
5670     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
5671
5672     $force = 1 if !defined($force) && !$shutdown;
5673
5674     if ($migratedfrom){
5675         my $pid = check_running($vmid, $nocheck, $migratedfrom);
5676         kill 15, $pid if $pid;
5677         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
5678         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
5679         return;
5680     }
5681
5682     PVE::QemuConfig->lock_config($vmid, sub {
5683         _do_vm_stop($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive);
5684    });
5685 }
5686
5687 sub vm_reboot {
5688     my ($vmid, $timeout) = @_;
5689
5690     PVE::QemuConfig->lock_config($vmid, sub {
5691         eval {
5692
5693             # only reboot if running, as qmeventd starts it again on a stop event
5694             return if !check_running($vmid);
5695
5696             create_reboot_request($vmid);
5697
5698             my $storecfg = PVE::Storage::config();
5699             _do_vm_stop($storecfg, $vmid, undef, undef, $timeout, 1);
5700
5701         };
5702         if (my $err = $@) {
5703             # avoid that the next normal shutdown will be confused for a reboot
5704             clear_reboot_request($vmid);
5705             die $err;
5706         }
5707    });
5708 }
5709
5710 sub vm_suspend {
5711     my ($vmid, $skiplock, $includestate, $statestorage) = @_;
5712
5713     my $conf;
5714     my $path;
5715     my $storecfg;
5716     my $vmstate;
5717
5718     PVE::QemuConfig->lock_config($vmid, sub {
5719
5720         $conf = PVE::QemuConfig->load_config($vmid);
5721
5722         my $is_backing_up = PVE::QemuConfig->has_lock($conf, 'backup');
5723         PVE::QemuConfig->check_lock($conf)
5724             if !($skiplock || $is_backing_up);
5725
5726         die "cannot suspend to disk during backup\n"
5727             if $is_backing_up && $includestate;
5728
5729         if ($includestate) {
5730             $conf->{lock} = 'suspending';
5731             my $date = strftime("%Y-%m-%d", localtime(time()));
5732             $storecfg = PVE::Storage::config();
5733             $vmstate = PVE::QemuConfig->__snapshot_save_vmstate($vmid, $conf, "suspend-$date", $storecfg, $statestorage, 1);
5734             $path = PVE::Storage::path($storecfg, $vmstate);
5735             PVE::QemuConfig->write_config($vmid, $conf);
5736         } else {
5737             mon_cmd($vmid, "stop");
5738         }
5739     });
5740
5741     if ($includestate) {
5742         # save vm state
5743         PVE::Storage::activate_volumes($storecfg, [$vmstate]);
5744
5745         eval {
5746             mon_cmd($vmid, "savevm-start", statefile => $path);
5747             for(;;) {
5748                 my $state = mon_cmd($vmid, "query-savevm");
5749                 if (!$state->{status}) {
5750                     die "savevm not active\n";
5751                 } elsif ($state->{status} eq 'active') {
5752                     sleep(1);
5753                     next;
5754                 } elsif ($state->{status} eq 'completed') {
5755                     print "State saved, quitting\n";
5756                     last;
5757                 } elsif ($state->{status} eq 'failed' && $state->{error}) {
5758                     die "query-savevm failed with error '$state->{error}'\n"
5759                 } else {
5760                     die "query-savevm returned status '$state->{status}'\n";
5761                 }
5762             }
5763         };
5764         my $err = $@;
5765
5766         PVE::QemuConfig->lock_config($vmid, sub {
5767             $conf = PVE::QemuConfig->load_config($vmid);
5768             if ($err) {
5769                 # cleanup, but leave suspending lock, to indicate something went wrong
5770                 eval {
5771                     mon_cmd($vmid, "savevm-end");
5772                     PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
5773                     PVE::Storage::vdisk_free($storecfg, $vmstate);
5774                     delete $conf->@{qw(vmstate runningmachine)};
5775                     PVE::QemuConfig->write_config($vmid, $conf);
5776                 };
5777                 warn $@ if $@;
5778                 die $err;
5779             }
5780
5781             die "lock changed unexpectedly\n"
5782                 if !PVE::QemuConfig->has_lock($conf, 'suspending');
5783
5784             mon_cmd($vmid, "quit");
5785             $conf->{lock} = 'suspended';
5786             PVE::QemuConfig->write_config($vmid, $conf);
5787         });
5788     }
5789 }
5790
5791 sub vm_resume {
5792     my ($vmid, $skiplock, $nocheck) = @_;
5793
5794     PVE::QemuConfig->lock_config($vmid, sub {
5795         my $res = mon_cmd($vmid, 'query-status');
5796         my $resume_cmd = 'cont';
5797
5798         if ($res->{status} && $res->{status} eq 'suspended') {
5799             $resume_cmd = 'system_wakeup';
5800         }
5801
5802         if (!$nocheck) {
5803
5804             my $conf = PVE::QemuConfig->load_config($vmid);
5805
5806             PVE::QemuConfig->check_lock($conf)
5807                 if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
5808         }
5809
5810         mon_cmd($vmid, $resume_cmd);
5811     });
5812 }
5813
5814 sub vm_sendkey {
5815     my ($vmid, $skiplock, $key) = @_;
5816
5817     PVE::QemuConfig->lock_config($vmid, sub {
5818
5819         my $conf = PVE::QemuConfig->load_config($vmid);
5820
5821         # there is no qmp command, so we use the human monitor command
5822         my $res = PVE::QemuServer::Monitor::hmp_cmd($vmid, "sendkey $key");
5823         die $res if $res ne '';
5824     });
5825 }
5826
5827 # vzdump restore implementaion
5828
5829 sub tar_archive_read_firstfile {
5830     my $archive = shift;
5831
5832     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
5833
5834     # try to detect archive type first
5835     my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
5836         die "unable to open file '$archive'\n";
5837     my $firstfile = <$fh>;
5838     kill 15, $pid;
5839     close $fh;
5840
5841     die "ERROR: archive contaions no data\n" if !$firstfile;
5842     chomp $firstfile;
5843
5844     return $firstfile;
5845 }
5846
5847 sub tar_restore_cleanup {
5848     my ($storecfg, $statfile) = @_;
5849
5850     print STDERR "starting cleanup\n";
5851
5852     if (my $fd = IO::File->new($statfile, "r")) {
5853         while (defined(my $line = <$fd>)) {
5854             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5855                 my $volid = $2;
5856                 eval {
5857                     if ($volid =~ m|^/|) {
5858                         unlink $volid || die 'unlink failed\n';
5859                     } else {
5860                         PVE::Storage::vdisk_free($storecfg, $volid);
5861                     }
5862                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
5863                 };
5864                 print STDERR "unable to cleanup '$volid' - $@" if $@;
5865             } else {
5866                 print STDERR "unable to parse line in statfile - $line";
5867             }
5868         }
5869         $fd->close();
5870     }
5871 }
5872
5873 sub restore_archive {
5874     my ($archive, $vmid, $user, $opts) = @_;
5875
5876     my $format = $opts->{format};
5877     my $comp;
5878
5879     if ($archive =~ m/\.tgz$/ || $archive =~ m/\.tar\.gz$/) {
5880         $format = 'tar' if !$format;
5881         $comp = 'gzip';
5882     } elsif ($archive =~ m/\.tar$/) {
5883         $format = 'tar' if !$format;
5884     } elsif ($archive =~ m/.tar.lzo$/) {
5885         $format = 'tar' if !$format;
5886         $comp = 'lzop';
5887     } elsif ($archive =~ m/\.vma$/) {
5888         $format = 'vma' if !$format;
5889     } elsif ($archive =~ m/\.vma\.gz$/) {
5890         $format = 'vma' if !$format;
5891         $comp = 'gzip';
5892     } elsif ($archive =~ m/\.vma\.lzo$/) {
5893         $format = 'vma' if !$format;
5894         $comp = 'lzop';
5895     } else {
5896         $format = 'vma' if !$format; # default
5897     }
5898
5899     # try to detect archive format
5900     if ($format eq 'tar') {
5901         return restore_tar_archive($archive, $vmid, $user, $opts);
5902     } else {
5903         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
5904     }
5905 }
5906
5907 sub restore_update_config_line {
5908     my ($outfd, $cookie, $vmid, $map, $line, $unique) = @_;
5909
5910     return if $line =~ m/^\#qmdump\#/;
5911     return if $line =~ m/^\#vzdump\#/;
5912     return if $line =~ m/^lock:/;
5913     return if $line =~ m/^unused\d+:/;
5914     return if $line =~ m/^parent:/;
5915
5916     my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
5917     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
5918         # try to convert old 1.X settings
5919         my ($id, $ind, $ethcfg) = ($1, $2, $3);
5920         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
5921             my ($model, $macaddr) = split(/\=/, $devconfig);
5922             $macaddr = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if !$macaddr || $unique;
5923             my $net = {
5924                 model => $model,
5925                 bridge => "vmbr$ind",
5926                 macaddr => $macaddr,
5927             };
5928             my $netstr = print_net($net);
5929
5930             print $outfd "net$cookie->{netcount}: $netstr\n";
5931             $cookie->{netcount}++;
5932         }
5933     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
5934         my ($id, $netstr) = ($1, $2);
5935         my $net = parse_net($netstr);
5936         $net->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if $net->{macaddr};
5937         $netstr = print_net($net);
5938         print $outfd "$id: $netstr\n";
5939     } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk)\d+):\s*(\S+)\s*$/) {
5940         my $virtdev = $1;
5941         my $value = $3;
5942         my $di = parse_drive($virtdev, $value);
5943         if (defined($di->{backup}) && !$di->{backup}) {
5944             print $outfd "#$line";
5945         } elsif ($map->{$virtdev}) {
5946             delete $di->{format}; # format can change on restore
5947             $di->{file} = $map->{$virtdev};
5948             $value = print_drive($vmid, $di);
5949             print $outfd "$virtdev: $value\n";
5950         } else {
5951             print $outfd $line;
5952         }
5953     } elsif (($line =~ m/^vmgenid: (.*)/)) {
5954         my $vmgenid = $1;
5955         if ($vmgenid ne '0') {
5956             # always generate a new vmgenid if there was a valid one setup
5957             $vmgenid = generate_uuid();
5958         }
5959         print $outfd "vmgenid: $vmgenid\n";
5960     } elsif (($line =~ m/^(smbios1: )(.*)/) && $unique) {
5961         my ($uuid, $uuid_str);
5962         UUID::generate($uuid);
5963         UUID::unparse($uuid, $uuid_str);
5964         my $smbios1 = parse_smbios1($2);
5965         $smbios1->{uuid} = $uuid_str;
5966         print $outfd $1.print_smbios1($smbios1)."\n";
5967     } else {
5968         print $outfd $line;
5969     }
5970 }
5971
5972 sub scan_volids {
5973     my ($cfg, $vmid) = @_;
5974
5975     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid);
5976
5977     my $volid_hash = {};
5978     foreach my $storeid (keys %$info) {
5979         foreach my $item (@{$info->{$storeid}}) {
5980             next if !($item->{volid} && $item->{size});
5981             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
5982             $volid_hash->{$item->{volid}} = $item;
5983         }
5984     }
5985
5986     return $volid_hash;
5987 }
5988
5989 sub is_volume_in_use {
5990     my ($storecfg, $conf, $skip_drive, $volid) = @_;
5991
5992     my $path = PVE::Storage::path($storecfg, $volid);
5993
5994     my $scan_config = sub {
5995         my ($cref, $snapname) = @_;
5996
5997         foreach my $key (keys %$cref) {
5998             my $value = $cref->{$key};
5999             if (is_valid_drivename($key)) {
6000                 next if $skip_drive && $key eq $skip_drive;
6001                 my $drive = parse_drive($key, $value);
6002                 next if !$drive || !$drive->{file} || drive_is_cdrom($drive);
6003                 return 1 if $volid eq $drive->{file};
6004                 if ($drive->{file} =~ m!^/!) {
6005                     return 1 if $drive->{file} eq $path;
6006                 } else {
6007                     my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file}, 1);
6008                     next if !$storeid;
6009                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid, 1);
6010                     next if !$scfg;
6011                     return 1 if $path eq PVE::Storage::path($storecfg, $drive->{file}, $snapname);
6012                 }
6013             }
6014         }
6015
6016         return 0;
6017     };
6018
6019     return 1 if &$scan_config($conf);
6020
6021     undef $skip_drive;
6022
6023     foreach my $snapname (keys %{$conf->{snapshots}}) {
6024         return 1 if &$scan_config($conf->{snapshots}->{$snapname}, $snapname);
6025     }
6026
6027     return 0;
6028 }
6029
6030 sub update_disksize {
6031     my ($vmid, $conf, $volid_hash) = @_;
6032
6033     my $changes;
6034     my $prefix = "VM $vmid:";
6035
6036     # used and unused disks
6037     my $referenced = {};
6038
6039     # Note: it is allowed to define multiple storages with same path (alias), so
6040     # we need to check both 'volid' and real 'path' (two different volid can point
6041     # to the same path).
6042
6043     my $referencedpath = {};
6044
6045     # update size info
6046     foreach my $opt (keys %$conf) {
6047         if (is_valid_drivename($opt)) {
6048             my $drive = parse_drive($opt, $conf->{$opt});
6049             my $volid = $drive->{file};
6050             next if !$volid;
6051
6052             $referenced->{$volid} = 1;
6053             if ($volid_hash->{$volid} &&
6054                 (my $path = $volid_hash->{$volid}->{path})) {
6055                 $referencedpath->{$path} = 1;
6056             }
6057
6058             next if drive_is_cdrom($drive);
6059             next if !$volid_hash->{$volid};
6060
6061             $drive->{size} = $volid_hash->{$volid}->{size};
6062             my $new = print_drive($vmid, $drive);
6063             if ($new ne $conf->{$opt}) {
6064                 $changes = 1;
6065                 $conf->{$opt} = $new;
6066                 print "$prefix update disk '$opt' information.\n";
6067             }
6068         }
6069     }
6070
6071     # remove 'unusedX' entry if volume is used
6072     foreach my $opt (keys %$conf) {
6073         next if $opt !~ m/^unused\d+$/;
6074         my $volid = $conf->{$opt};
6075         my $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
6076         if ($referenced->{$volid} || ($path && $referencedpath->{$path})) {
6077             print "$prefix remove entry '$opt', its volume '$volid' is in use.\n";
6078             $changes = 1;
6079             delete $conf->{$opt};
6080         }
6081
6082         $referenced->{$volid} = 1;
6083         $referencedpath->{$path} = 1 if $path;
6084     }
6085
6086     foreach my $volid (sort keys %$volid_hash) {
6087         next if $volid =~ m/vm-$vmid-state-/;
6088         next if $referenced->{$volid};
6089         my $path = $volid_hash->{$volid}->{path};
6090         next if !$path; # just to be sure
6091         next if $referencedpath->{$path};
6092         $changes = 1;
6093         my $key = PVE::QemuConfig->add_unused_volume($conf, $volid);
6094         print "$prefix add unreferenced volume '$volid' as '$key' to config.\n";
6095         $referencedpath->{$path} = 1; # avoid to add more than once (aliases)
6096     }
6097
6098     return $changes;
6099 }
6100
6101 sub rescan {
6102     my ($vmid, $nolock, $dryrun) = @_;
6103
6104     my $cfg = PVE::Storage::config();
6105
6106     # FIXME: Remove once our RBD plugin can handle CT and VM on a single storage
6107     # see: https://pve.proxmox.com/pipermail/pve-devel/2018-July/032900.html
6108     foreach my $stor (keys %{$cfg->{ids}}) {
6109         delete($cfg->{ids}->{$stor}) if ! $cfg->{ids}->{$stor}->{content}->{images};
6110     }
6111
6112     print "rescan volumes...\n";
6113     my $volid_hash = scan_volids($cfg, $vmid);
6114
6115     my $updatefn =  sub {
6116         my ($vmid) = @_;
6117
6118         my $conf = PVE::QemuConfig->load_config($vmid);
6119
6120         PVE::QemuConfig->check_lock($conf);
6121
6122         my $vm_volids = {};
6123         foreach my $volid (keys %$volid_hash) {
6124             my $info = $volid_hash->{$volid};
6125             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
6126         }
6127
6128         my $changes = update_disksize($vmid, $conf, $vm_volids);
6129
6130         PVE::QemuConfig->write_config($vmid, $conf) if $changes && !$dryrun;
6131     };
6132
6133     if (defined($vmid)) {
6134         if ($nolock) {
6135             &$updatefn($vmid);
6136         } else {
6137             PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6138         }
6139     } else {
6140         my $vmlist = config_list();
6141         foreach my $vmid (keys %$vmlist) {
6142             if ($nolock) {
6143                 &$updatefn($vmid);
6144             } else {
6145                 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6146             }
6147         }
6148     }
6149 }
6150
6151 sub restore_vma_archive {
6152     my ($archive, $vmid, $user, $opts, $comp) = @_;
6153
6154     my $readfrom = $archive;
6155
6156     my $cfg = PVE::Storage::config();
6157     my $commands = [];
6158     my $bwlimit = $opts->{bwlimit};
6159
6160     my $dbg_cmdstring = '';
6161     my $add_pipe = sub {
6162         my ($cmd) = @_;
6163         push @$commands, $cmd;
6164         $dbg_cmdstring .= ' | ' if length($dbg_cmdstring);
6165         $dbg_cmdstring .= PVE::Tools::cmd2string($cmd);
6166         $readfrom = '-';
6167     };
6168
6169     my $input = undef;
6170     if ($archive eq '-') {
6171         $input = '<&STDIN';
6172     } else {
6173         # If we use a backup from a PVE defined storage we also consider that
6174         # storage's rate limit:
6175         my (undef, $volid) = PVE::Storage::path_to_volume_id($cfg, $archive);
6176         if (defined($volid)) {
6177             my ($sid, undef) = PVE::Storage::parse_volume_id($volid);
6178             my $readlimit = PVE::Storage::get_bandwidth_limit('restore', [$sid], $bwlimit);
6179             if ($readlimit) {
6180                 print STDERR "applying read rate limit: $readlimit\n";
6181                 my $cstream = ['cstream', '-t', $readlimit*1024, '--', $readfrom];
6182                 $add_pipe->($cstream);
6183             }
6184         }
6185     }
6186
6187     if ($comp) {
6188         my $cmd;
6189         if ($comp eq 'gzip') {
6190             $cmd = ['zcat', $readfrom];
6191         } elsif ($comp eq 'lzop') {
6192             $cmd = ['lzop', '-d', '-c', $readfrom];
6193         } else {
6194             die "unknown compression method '$comp'\n";
6195         }
6196         $add_pipe->($cmd);
6197     }
6198
6199     my $tmpdir = "/var/tmp/vzdumptmp$$";
6200     rmtree $tmpdir;
6201
6202     # disable interrupts (always do cleanups)
6203     local $SIG{INT} =
6204         local $SIG{TERM} =
6205         local $SIG{QUIT} =
6206         local $SIG{HUP} = sub { warn "got interrupt - ignored\n"; };
6207
6208     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
6209     POSIX::mkfifo($mapfifo, 0600);
6210     my $fifofh;
6211
6212     my $openfifo = sub {
6213         open($fifofh, '>', $mapfifo) || die $!;
6214     };
6215
6216     $add_pipe->(['vma', 'extract', '-v', '-r', $mapfifo, $readfrom, $tmpdir]);
6217
6218     my $oldtimeout;
6219     my $timeout = 5;
6220
6221     my $devinfo = {};
6222
6223     my $rpcenv = PVE::RPCEnvironment::get();
6224
6225     my $conffile = PVE::QemuConfig->config_file($vmid);
6226     my $tmpfn = "$conffile.$$.tmp";
6227
6228     # Note: $oldconf is undef if VM does not exists
6229     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
6230     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
6231
6232     my %storage_limits;
6233
6234     my $print_devmap = sub {
6235         my $virtdev_hash = {};
6236
6237         my $cfgfn = "$tmpdir/qemu-server.conf";
6238
6239         # we can read the config - that is already extracted
6240         my $fh = IO::File->new($cfgfn, "r") ||
6241             "unable to read qemu-server.conf - $!\n";
6242
6243         my $fwcfgfn = "$tmpdir/qemu-server.fw";
6244         if (-f $fwcfgfn) {
6245             my $pve_firewall_dir = '/etc/pve/firewall';
6246             mkdir $pve_firewall_dir; # make sure the dir exists
6247             PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
6248         }
6249
6250         while (defined(my $line = <$fh>)) {
6251             if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
6252                 my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
6253                 die "archive does not contain data for drive '$virtdev'\n"
6254                     if !$devinfo->{$devname};
6255                 if (defined($opts->{storage})) {
6256                     $storeid = $opts->{storage} || 'local';
6257                 } elsif (!$storeid) {
6258                     $storeid = 'local';
6259                 }
6260                 $format = 'raw' if !$format;
6261                 $devinfo->{$devname}->{devname} = $devname;
6262                 $devinfo->{$devname}->{virtdev} = $virtdev;
6263                 $devinfo->{$devname}->{format} = $format;
6264                 $devinfo->{$devname}->{storeid} = $storeid;
6265
6266                 # check permission on storage
6267                 my $pool = $opts->{pool}; # todo: do we need that?
6268                 if ($user ne 'root@pam') {
6269                     $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
6270                 }
6271
6272                 $storage_limits{$storeid} = $bwlimit;
6273
6274                 $virtdev_hash->{$virtdev} = $devinfo->{$devname};
6275             } elsif ($line =~ m/^((?:ide|sata|scsi)\d+):\s*(.*)\s*$/) {
6276                 my $virtdev = $1;
6277                 my $drive = parse_drive($virtdev, $2);
6278                 if (drive_is_cloudinit($drive)) {
6279                     my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6280                     my $scfg = PVE::Storage::storage_config($cfg, $storeid);
6281                     my $format = qemu_img_format($scfg, $volname); # has 'raw' fallback
6282
6283                     my $d = {
6284                         format => $format,
6285                         storeid => $opts->{storage} // $storeid,
6286                         size => PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE,
6287                         file => $drive->{file}, # to make drive_is_cloudinit check possible
6288                         name => "vm-$vmid-cloudinit",
6289                         is_cloudinit => 1,
6290                     };
6291                     $virtdev_hash->{$virtdev} = $d;
6292                 }
6293             }
6294         }
6295
6296         foreach my $key (keys %storage_limits) {
6297             my $limit = PVE::Storage::get_bandwidth_limit('restore', [$key], $bwlimit);
6298             next if !$limit;
6299             print STDERR "rate limit for storage $key: $limit KiB/s\n";
6300             $storage_limits{$key} = $limit * 1024;
6301         }
6302
6303         foreach my $devname (keys %$devinfo) {
6304             die "found no device mapping information for device '$devname'\n"
6305                 if !$devinfo->{$devname}->{virtdev};
6306         }
6307
6308         # create empty/temp config
6309         if ($oldconf) {
6310             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
6311             foreach_drive($oldconf, sub {
6312                 my ($ds, $drive) = @_;
6313
6314                 return if drive_is_cdrom($drive, 1);
6315
6316                 my $volid = $drive->{file};
6317                 return if !$volid || $volid =~ m|^/|;
6318
6319                 my ($path, $owner) = PVE::Storage::path($cfg, $volid);
6320                 return if !$path || !$owner || ($owner != $vmid);
6321
6322                 # Note: only delete disk we want to restore
6323                 # other volumes will become unused
6324                 if ($virtdev_hash->{$ds}) {
6325                     eval { PVE::Storage::vdisk_free($cfg, $volid); };
6326                     if (my $err = $@) {
6327                         warn $err;
6328                     }
6329                 }
6330             });
6331
6332             # delete vmstate files, after the restore we have no snapshots anymore
6333             foreach my $snapname (keys %{$oldconf->{snapshots}}) {
6334                 my $snap = $oldconf->{snapshots}->{$snapname};
6335                 if ($snap->{vmstate}) {
6336                     eval { PVE::Storage::vdisk_free($cfg, $snap->{vmstate}); };
6337                     if (my $err = $@) {
6338                         warn $err;
6339                     }
6340                 }
6341             }
6342         }
6343
6344         my $map = {};
6345         foreach my $virtdev (sort keys %$virtdev_hash) {
6346             my $d = $virtdev_hash->{$virtdev};
6347             my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
6348             my $storeid = $d->{storeid};
6349             my $scfg = PVE::Storage::storage_config($cfg, $storeid);
6350
6351             my $map_opts = '';
6352             if (my $limit = $storage_limits{$storeid}) {
6353                 $map_opts .= "throttling.bps=$limit:throttling.group=$storeid:";
6354             }
6355
6356             # test if requested format is supported
6357             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($cfg, $storeid);
6358             my $supported = grep { $_ eq $d->{format} } @$validFormats;
6359             $d->{format} = $defFormat if !$supported;
6360
6361             my $name;
6362             if ($d->{is_cloudinit}) {
6363                 $name = $d->{name};
6364                 $name .= ".$d->{format}" if $d->{format} ne 'raw';
6365             }
6366
6367             my $volid = PVE::Storage::vdisk_alloc($cfg, $storeid, $vmid, $d->{format}, $name, $alloc_size);
6368             print STDERR "new volume ID is '$volid'\n";
6369             $d->{volid} = $volid;
6370
6371             PVE::Storage::activate_volumes($cfg, [$volid]);
6372
6373             my $write_zeros = 1;
6374             if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
6375                 $write_zeros = 0;
6376             }
6377
6378             if (!$d->{is_cloudinit}) {
6379                 my $path = PVE::Storage::path($cfg, $volid);
6380
6381                 print $fifofh "${map_opts}format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
6382
6383                 print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
6384             }
6385             $map->{$virtdev} = $volid;
6386         }
6387
6388         $fh->seek(0, 0) || die "seek failed - $!\n";
6389
6390         my $outfd = new IO::File ($tmpfn, "w") ||
6391             die "unable to write config for VM $vmid\n";
6392
6393         my $cookie = { netcount => 0 };
6394         while (defined(my $line = <$fh>)) {
6395             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
6396         }
6397
6398         $fh->close();
6399         $outfd->close();
6400     };
6401
6402     eval {
6403         # enable interrupts
6404         local $SIG{INT} =
6405             local $SIG{TERM} =
6406             local $SIG{QUIT} =
6407             local $SIG{HUP} =
6408             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6409         local $SIG{ALRM} = sub { die "got timeout\n"; };
6410
6411         $oldtimeout = alarm($timeout);
6412
6413         my $parser = sub {
6414             my $line = shift;
6415
6416             print "$line\n";
6417
6418             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
6419                 my ($dev_id, $size, $devname) = ($1, $2, $3);
6420                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
6421             } elsif ($line =~ m/^CTIME: /) {
6422                 # we correctly received the vma config, so we can disable
6423                 # the timeout now for disk allocation (set to 10 minutes, so
6424                 # that we always timeout if something goes wrong)
6425                 alarm(600);
6426                 &$print_devmap();
6427                 print $fifofh "done\n";
6428                 my $tmp = $oldtimeout || 0;
6429                 $oldtimeout = undef;
6430                 alarm($tmp);
6431                 close($fifofh);
6432             }
6433         };
6434
6435         print "restore vma archive: $dbg_cmdstring\n";
6436         run_command($commands, input => $input, outfunc => $parser, afterfork => $openfifo);
6437     };
6438     my $err = $@;
6439
6440     alarm($oldtimeout) if $oldtimeout;
6441
6442     my $vollist = [];
6443     foreach my $devname (keys %$devinfo) {
6444         my $volid = $devinfo->{$devname}->{volid};
6445         push @$vollist, $volid if $volid;
6446     }
6447
6448     PVE::Storage::deactivate_volumes($cfg, $vollist);
6449
6450     unlink $mapfifo;
6451
6452     if ($err) {
6453         rmtree $tmpdir;
6454         unlink $tmpfn;
6455
6456         foreach my $devname (keys %$devinfo) {
6457             my $volid = $devinfo->{$devname}->{volid};
6458             next if !$volid;
6459             eval {
6460                 if ($volid =~ m|^/|) {
6461                     unlink $volid || die 'unlink failed\n';
6462                 } else {
6463                     PVE::Storage::vdisk_free($cfg, $volid);
6464                 }
6465                 print STDERR "temporary volume '$volid' sucessfuly removed\n";
6466             };
6467             print STDERR "unable to cleanup '$volid' - $@" if $@;
6468         }
6469         die $err;
6470     }
6471
6472     rmtree $tmpdir;
6473
6474     rename($tmpfn, $conffile) ||
6475         die "unable to commit configuration file '$conffile'\n";
6476
6477     PVE::Cluster::cfs_update(); # make sure we read new file
6478
6479     eval { rescan($vmid, 1); };
6480     warn $@ if $@;
6481 }
6482
6483 sub restore_tar_archive {
6484     my ($archive, $vmid, $user, $opts) = @_;
6485
6486     if ($archive ne '-') {
6487         my $firstfile = tar_archive_read_firstfile($archive);
6488         die "ERROR: file '$archive' dos not lock like a QemuServer vzdump backup\n"
6489             if $firstfile ne 'qemu-server.conf';
6490     }
6491
6492     my $storecfg = PVE::Storage::config();
6493
6494     # avoid zombie disks when restoring over an existing VM -> cleanup first
6495     # pass keep_empty_config=1 to keep the config (thus VMID) reserved for us
6496     # skiplock=1 because qmrestore has set the 'create' lock itself already
6497     my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
6498     destroy_vm($storecfg, $vmid, 1, { lock => 'restore' }) if -f $vmcfgfn;
6499
6500     my $tocmd = "/usr/lib/qemu-server/qmextract";
6501
6502     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
6503     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
6504     $tocmd .= ' --prealloc' if $opts->{prealloc};
6505     $tocmd .= ' --info' if $opts->{info};
6506
6507     # tar option "xf" does not autodetect compression when read from STDIN,
6508     # so we pipe to zcat
6509     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
6510         PVE::Tools::shellquote("--to-command=$tocmd");
6511
6512     my $tmpdir = "/var/tmp/vzdumptmp$$";
6513     mkpath $tmpdir;
6514
6515     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
6516     local $ENV{VZDUMP_VMID} = $vmid;
6517     local $ENV{VZDUMP_USER} = $user;
6518
6519     my $conffile = PVE::QemuConfig->config_file($vmid);
6520     my $tmpfn = "$conffile.$$.tmp";
6521
6522     # disable interrupts (always do cleanups)
6523     local $SIG{INT} =
6524         local $SIG{TERM} =
6525         local $SIG{QUIT} =
6526         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
6527
6528     eval {
6529         # enable interrupts
6530         local $SIG{INT} =
6531             local $SIG{TERM} =
6532             local $SIG{QUIT} =
6533             local $SIG{HUP} =
6534             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6535
6536         if ($archive eq '-') {
6537             print "extracting archive from STDIN\n";
6538             run_command($cmd, input => "<&STDIN");
6539         } else {
6540             print "extracting archive '$archive'\n";
6541             run_command($cmd);
6542         }
6543
6544         return if $opts->{info};
6545
6546         # read new mapping
6547         my $map = {};
6548         my $statfile = "$tmpdir/qmrestore.stat";
6549         if (my $fd = IO::File->new($statfile, "r")) {
6550             while (defined (my $line = <$fd>)) {
6551                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
6552                     $map->{$1} = $2 if $1;
6553                 } else {
6554                     print STDERR "unable to parse line in statfile - $line\n";
6555                 }
6556             }
6557             $fd->close();
6558         }
6559
6560         my $confsrc = "$tmpdir/qemu-server.conf";
6561
6562         my $srcfd = new IO::File($confsrc, "r") ||
6563             die "unable to open file '$confsrc'\n";
6564
6565         my $outfd = new IO::File ($tmpfn, "w") ||
6566             die "unable to write config for VM $vmid\n";
6567
6568         my $cookie = { netcount => 0 };
6569         while (defined (my $line = <$srcfd>)) {
6570             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
6571         }
6572
6573         $srcfd->close();
6574         $outfd->close();
6575     };
6576     if (my $err = $@) {
6577         unlink $tmpfn;
6578         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
6579         die $err;
6580     }
6581
6582     rmtree $tmpdir;
6583
6584     rename $tmpfn, $conffile ||
6585         die "unable to commit configuration file '$conffile'\n";
6586
6587     PVE::Cluster::cfs_update(); # make sure we read new file
6588
6589     eval { rescan($vmid, 1); };
6590     warn $@ if $@;
6591 };
6592
6593 sub foreach_storage_used_by_vm {
6594     my ($conf, $func) = @_;
6595
6596     my $sidhash = {};
6597
6598     foreach_drive($conf, sub {
6599         my ($ds, $drive) = @_;
6600         return if drive_is_cdrom($drive);
6601
6602         my $volid = $drive->{file};
6603
6604         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
6605         $sidhash->{$sid} = $sid if $sid;
6606     });
6607
6608     foreach my $sid (sort keys %$sidhash) {
6609         &$func($sid);
6610     }
6611 }
6612
6613 sub do_snapshots_with_qemu {
6614     my ($storecfg, $volid) = @_;
6615
6616     my $storage_name = PVE::Storage::parse_volume_id($volid);
6617     my $scfg = $storecfg->{ids}->{$storage_name};
6618
6619     if ($qemu_snap_storage->{$scfg->{type}} && !$scfg->{krbd}){
6620         return 1;
6621     }
6622
6623     if ($volid =~ m/\.(qcow2|qed)$/){
6624         return 1;
6625     }
6626
6627     return undef;
6628 }
6629
6630 sub qga_check_running {
6631     my ($vmid, $nowarn) = @_;
6632
6633     eval { mon_cmd($vmid, "guest-ping", timeout => 3); };
6634     if ($@) {
6635         warn "Qemu Guest Agent is not running - $@" if !$nowarn;
6636         return 0;
6637     }
6638     return 1;
6639 }
6640
6641 sub template_create {
6642     my ($vmid, $conf, $disk) = @_;
6643
6644     my $storecfg = PVE::Storage::config();
6645
6646     foreach_drive($conf, sub {
6647         my ($ds, $drive) = @_;
6648
6649         return if drive_is_cdrom($drive);
6650         return if $disk && $ds ne $disk;
6651
6652         my $volid = $drive->{file};
6653         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
6654
6655         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
6656         $drive->{file} = $voliddst;
6657         $conf->{$ds} = print_drive($vmid, $drive);
6658         PVE::QemuConfig->write_config($vmid, $conf);
6659     });
6660 }
6661
6662 sub convert_iscsi_path {
6663     my ($path) = @_;
6664
6665     if ($path =~ m|^iscsi://([^/]+)/([^/]+)/(.+)$|) {
6666         my $portal = $1;
6667         my $target = $2;
6668         my $lun = $3;
6669
6670         my $initiator_name = get_initiator_name();
6671
6672         return "file.driver=iscsi,file.transport=tcp,file.initiator-name=$initiator_name,".
6673                "file.portal=$portal,file.target=$target,file.lun=$lun,driver=raw";
6674     }
6675
6676     die "cannot convert iscsi path '$path', unkown format\n";
6677 }
6678
6679 sub qemu_img_convert {
6680     my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized) = @_;
6681
6682     my $storecfg = PVE::Storage::config();
6683     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
6684     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
6685
6686     die "destination '$dst_volid' is not a valid volid form qemu-img convert\n" if !$dst_storeid;
6687
6688     my $cachemode;
6689     my $src_path;
6690     my $src_is_iscsi = 0;
6691     my $src_format = 'raw';
6692
6693     if ($src_storeid) {
6694         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
6695         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
6696         $src_format = qemu_img_format($src_scfg, $src_volname);
6697         $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
6698         $src_is_iscsi = ($src_path =~ m|^iscsi://|);
6699         $cachemode = 'none' if $src_scfg->{type} eq 'zfspool';
6700     } elsif (-f $src_volid) {
6701         $src_path = $src_volid;
6702         if ($src_path =~ m/\.($QEMU_FORMAT_RE)$/) {
6703             $src_format = $1;
6704         }
6705     }
6706
6707     die "source '$src_volid' is not a valid volid nor path for qemu-img convert\n" if !$src_path;
6708
6709     my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6710     my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
6711     my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6712     my $dst_is_iscsi = ($dst_path =~ m|^iscsi://|);
6713
6714     my $cmd = [];
6715     push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
6716     push @$cmd, '-l', "snapshot.name=$snapname" if($snapname && $src_format eq "qcow2");
6717     push @$cmd, '-t', 'none' if $dst_scfg->{type} eq 'zfspool';
6718     push @$cmd, '-T', $cachemode if defined($cachemode);
6719
6720     if ($src_is_iscsi) {
6721         push @$cmd, '--image-opts';
6722         $src_path = convert_iscsi_path($src_path);
6723     } else {
6724         push @$cmd, '-f', $src_format;
6725     }
6726
6727     if ($dst_is_iscsi) {
6728         push @$cmd, '--target-image-opts';
6729         $dst_path = convert_iscsi_path($dst_path);
6730     } else {
6731         push @$cmd, '-O', $dst_format;
6732     }
6733
6734     push @$cmd, $src_path;
6735
6736     if (!$dst_is_iscsi && $is_zero_initialized) {
6737         push @$cmd, "zeroinit:$dst_path";
6738     } else {
6739         push @$cmd, $dst_path;
6740     }
6741
6742     my $parser = sub {
6743         my $line = shift;
6744         if($line =~ m/\((\S+)\/100\%\)/){
6745             my $percent = $1;
6746             my $transferred = int($size * $percent / 100);
6747             my $remaining = $size - $transferred;
6748
6749             print "transferred: $transferred bytes remaining: $remaining bytes total: $size bytes progression: $percent %\n";
6750         }
6751
6752     };
6753
6754     eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
6755     my $err = $@;
6756     die "copy failed: $err" if $err;
6757 }
6758
6759 sub qemu_img_format {
6760     my ($scfg, $volname) = @_;
6761
6762     if ($scfg->{path} && $volname =~ m/\.($QEMU_FORMAT_RE)$/) {
6763         return $1;
6764     } else {
6765         return "raw";
6766     }
6767 }
6768
6769 sub qemu_drive_mirror {
6770     my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $skipcomplete, $qga, $bwlimit) = @_;
6771
6772     $jobs = {} if !$jobs;
6773
6774     my $qemu_target;
6775     my $format;
6776     $jobs->{"drive-$drive"} = {};
6777
6778     if ($dst_volid =~ /^nbd:/) {
6779         $qemu_target = $dst_volid;
6780         $format = "nbd";
6781     } else {
6782         my $storecfg = PVE::Storage::config();
6783         my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
6784
6785         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6786
6787         $format = qemu_img_format($dst_scfg, $dst_volname);
6788
6789         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6790
6791         $qemu_target = $is_zero_initialized ? "zeroinit:$dst_path" : $dst_path;
6792     }
6793
6794     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $qemu_target };
6795     $opts->{format} = $format if $format;
6796
6797     if (defined($bwlimit)) {
6798         $opts->{speed} = $bwlimit * 1024;
6799         print "drive mirror is starting for drive-$drive with bandwidth limit: ${bwlimit} KB/s\n";
6800     } else {
6801         print "drive mirror is starting for drive-$drive\n";
6802     }
6803
6804     # if a job already runs for this device we get an error, catch it for cleanup
6805     eval { mon_cmd($vmid, "drive-mirror", %$opts); };
6806     if (my $err = $@) {
6807         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6808         warn "$@\n" if $@;
6809         die "mirroring error: $err\n";
6810     }
6811
6812     qemu_drive_mirror_monitor ($vmid, $vmiddst, $jobs, $skipcomplete, $qga);
6813 }
6814
6815 sub qemu_drive_mirror_monitor {
6816     my ($vmid, $vmiddst, $jobs, $skipcomplete, $qga) = @_;
6817
6818     eval {
6819         my $err_complete = 0;
6820
6821         while (1) {
6822             die "storage migration timed out\n" if $err_complete > 300;
6823
6824             my $stats = mon_cmd($vmid, "query-block-jobs");
6825
6826             my $running_mirror_jobs = {};
6827             foreach my $stat (@$stats) {
6828                 next if $stat->{type} ne 'mirror';
6829                 $running_mirror_jobs->{$stat->{device}} = $stat;
6830             }
6831
6832             my $readycounter = 0;
6833
6834             foreach my $job (keys %$jobs) {
6835
6836                 if(defined($jobs->{$job}->{complete}) && !defined($running_mirror_jobs->{$job})) {
6837                     print "$job : finished\n";
6838                     delete $jobs->{$job};
6839                     next;
6840                 }
6841
6842                 die "$job: mirroring has been cancelled\n" if !defined($running_mirror_jobs->{$job});
6843
6844                 my $busy = $running_mirror_jobs->{$job}->{busy};
6845                 my $ready = $running_mirror_jobs->{$job}->{ready};
6846                 if (my $total = $running_mirror_jobs->{$job}->{len}) {
6847                     my $transferred = $running_mirror_jobs->{$job}->{offset} || 0;
6848                     my $remaining = $total - $transferred;
6849                     my $percent = sprintf "%.2f", ($transferred * 100 / $total);
6850
6851                     print "$job: transferred: $transferred bytes remaining: $remaining bytes total: $total bytes progression: $percent % busy: $busy ready: $ready \n";
6852                 }
6853
6854                 $readycounter++ if $running_mirror_jobs->{$job}->{ready};
6855             }
6856
6857             last if scalar(keys %$jobs) == 0;
6858
6859             if ($readycounter == scalar(keys %$jobs)) {
6860                 print "all mirroring jobs are ready \n";
6861                 last if $skipcomplete; #do the complete later
6862
6863                 if ($vmiddst && $vmiddst != $vmid) {
6864                     my $agent_running = $qga && qga_check_running($vmid);
6865                     if ($agent_running) {
6866                         print "freeze filesystem\n";
6867                         eval { mon_cmd($vmid, "guest-fsfreeze-freeze"); };
6868                     } else {
6869                         print "suspend vm\n";
6870                         eval { PVE::QemuServer::vm_suspend($vmid, 1); };
6871                     }
6872
6873                     # if we clone a disk for a new target vm, we don't switch the disk
6874                     PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs);
6875
6876                     if ($agent_running) {
6877                         print "unfreeze filesystem\n";
6878                         eval { mon_cmd($vmid, "guest-fsfreeze-thaw"); };
6879                     } else {
6880                         print "resume vm\n";
6881                         eval {  PVE::QemuServer::vm_resume($vmid, 1, 1); };
6882                     }
6883
6884                     last;
6885                 } else {
6886
6887                     foreach my $job (keys %$jobs) {
6888                         # try to switch the disk if source and destination are on the same guest
6889                         print "$job: Completing block job...\n";
6890
6891                         eval { mon_cmd($vmid, "block-job-complete", device => $job) };
6892                         if ($@ =~ m/cannot be completed/) {
6893                             print "$job: Block job cannot be completed, try again.\n";
6894                             $err_complete++;
6895                         }else {
6896                             print "$job: Completed successfully.\n";
6897                             $jobs->{$job}->{complete} = 1;
6898                         }
6899                     }
6900                 }
6901             }
6902             sleep 1;
6903         }
6904     };
6905     my $err = $@;
6906
6907     if ($err) {
6908         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6909         die "mirroring error: $err";
6910     }
6911
6912 }
6913
6914 sub qemu_blockjobs_cancel {
6915     my ($vmid, $jobs) = @_;
6916
6917     foreach my $job (keys %$jobs) {
6918         print "$job: Cancelling block job\n";
6919         eval { mon_cmd($vmid, "block-job-cancel", device => $job); };
6920         $jobs->{$job}->{cancel} = 1;
6921     }
6922
6923     while (1) {
6924         my $stats = mon_cmd($vmid, "query-block-jobs");
6925
6926         my $running_jobs = {};
6927         foreach my $stat (@$stats) {
6928             $running_jobs->{$stat->{device}} = $stat;
6929         }
6930
6931         foreach my $job (keys %$jobs) {
6932
6933             if (defined($jobs->{$job}->{cancel}) && !defined($running_jobs->{$job})) {
6934                 print "$job: Done.\n";
6935                 delete $jobs->{$job};
6936             }
6937         }
6938
6939         last if scalar(keys %$jobs) == 0;
6940
6941         sleep 1;
6942     }
6943 }
6944
6945 sub clone_disk {
6946     my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
6947         $newvmid, $storage, $format, $full, $newvollist, $jobs, $skipcomplete, $qga, $bwlimit) = @_;
6948
6949     my $newvolid;
6950
6951     if (!$full) {
6952         print "create linked clone of drive $drivename ($drive->{file})\n";
6953         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
6954         push @$newvollist, $newvolid;
6955     } else {
6956
6957         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6958         $storeid = $storage if $storage;
6959
6960         my $dst_format = resolve_dst_disk_format($storecfg, $storeid, $volname, $format);
6961         my ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 3);
6962
6963         print "create full clone of drive $drivename ($drive->{file})\n";
6964         my $name = undef;
6965         if (drive_is_cloudinit($drive)) {
6966             $name = "vm-$newvmid-cloudinit";
6967             $name .= ".$dst_format" if $dst_format ne 'raw';
6968             $snapname = undef;
6969             $size = PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE;
6970         }
6971         $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $newvmid, $dst_format, $name, ($size/1024));
6972         push @$newvollist, $newvolid;
6973
6974         PVE::Storage::activate_volumes($storecfg, [$newvolid]);
6975
6976         if (drive_is_cloudinit($drive)) {
6977             goto no_data_clone;
6978         }
6979
6980         my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
6981         if (!$running || $snapname) {
6982             # TODO: handle bwlimits
6983             qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit);
6984         } else {
6985
6986             my $kvmver = get_running_qemu_version ($vmid);
6987             if (!min_version($kvmver, 2, 7)) {
6988                 die "drive-mirror with iothread requires qemu version 2.7 or higher\n"
6989                     if $drive->{iothread};
6990             }
6991
6992             qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid, $sparseinit, $jobs, $skipcomplete, $qga, $bwlimit);
6993         }
6994     }
6995
6996 no_data_clone:
6997     my ($size) = PVE::Storage::volume_size_info($storecfg, $newvolid, 3);
6998
6999     my $disk = $drive;
7000     $disk->{format} = undef;
7001     $disk->{file} = $newvolid;
7002     $disk->{size} = $size;
7003
7004     return $disk;
7005 }
7006
7007 sub get_running_qemu_version {
7008     my ($vmid) = @_;
7009     my $res = mon_cmd($vmid, "query-version");
7010     return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
7011 }
7012
7013 sub qemu_use_old_bios_files {
7014     my ($machine_type) = @_;
7015
7016     return if !$machine_type;
7017
7018     my $use_old_bios_files = undef;
7019
7020     if ($machine_type =~ m/^(\S+)\.pxe$/) {
7021         $machine_type = $1;
7022         $use_old_bios_files = 1;
7023     } else {
7024         my $version = PVE::QemuServer::Machine::extract_version($machine_type) // kvm_user_version();
7025         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
7026         # load new efi bios files on migration. So this hack is required to allow
7027         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
7028         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
7029         $use_old_bios_files = !min_version($version, 2, 4);
7030     }
7031
7032     return ($use_old_bios_files, $machine_type);
7033 }
7034
7035 sub create_efidisk($$$$$) {
7036     my ($storecfg, $storeid, $vmid, $fmt, $arch) = @_;
7037
7038     my (undef, $ovmf_vars) = get_ovmf_files($arch);
7039     die "EFI vars default image not found\n" if ! -f $ovmf_vars;
7040
7041     my $vars_size_b = -s $ovmf_vars;
7042     my $vars_size = PVE::Tools::convert_size($vars_size_b, 'b' => 'kb');
7043     my $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $fmt, undef, $vars_size);
7044     PVE::Storage::activate_volumes($storecfg, [$volid]);
7045
7046     qemu_img_convert($ovmf_vars, $volid, $vars_size_b, undef, 0);
7047
7048     return ($volid, $vars_size);
7049 }
7050
7051 sub vm_iothreads_list {
7052     my ($vmid) = @_;
7053
7054     my $res = mon_cmd($vmid, 'query-iothreads');
7055
7056     my $iothreads = {};
7057     foreach my $iothread (@$res) {
7058         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
7059     }
7060
7061     return $iothreads;
7062 }
7063
7064 sub scsihw_infos {
7065     my ($conf, $drive) = @_;
7066
7067     my $maxdev = 0;
7068
7069     if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)) {
7070         $maxdev = 7;
7071     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
7072         $maxdev = 1;
7073     } else {
7074         $maxdev = 256;
7075     }
7076
7077     my $controller = int($drive->{index} / $maxdev);
7078     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single') ? "virtioscsi" : "scsihw";
7079
7080     return ($maxdev, $controller, $controller_prefix);
7081 }
7082
7083 sub add_hyperv_enlightenments {
7084     my ($cpuFlags, $winversion, $machine_version, $bios, $gpu_passthrough, $hv_vendor_id) = @_;
7085
7086     return if $winversion < 6;
7087     return if $bios && $bios eq 'ovmf' && $winversion < 8;
7088
7089     if ($gpu_passthrough || defined($hv_vendor_id)) {
7090         $hv_vendor_id //= 'proxmox';
7091         push @$cpuFlags , "hv_vendor_id=$hv_vendor_id";
7092     }
7093
7094     if (min_version($machine_version, 2, 3)) {
7095         push @$cpuFlags , 'hv_spinlocks=0x1fff';
7096         push @$cpuFlags , 'hv_vapic';
7097         push @$cpuFlags , 'hv_time';
7098     } else {
7099         push @$cpuFlags , 'hv_spinlocks=0xffff';
7100     }
7101
7102     if (min_version($machine_version, 2, 6)) {
7103         push @$cpuFlags , 'hv_reset';
7104         push @$cpuFlags , 'hv_vpindex';
7105         push @$cpuFlags , 'hv_runtime';
7106     }
7107
7108     if ($winversion >= 7) {
7109         push @$cpuFlags , 'hv_relaxed';
7110
7111         if (min_version($machine_version, 2, 12)) {
7112             push @$cpuFlags , 'hv_synic';
7113             push @$cpuFlags , 'hv_stimer';
7114         }
7115
7116         if (min_version($machine_version, 3, 1)) {
7117             push @$cpuFlags , 'hv_ipi';
7118         }
7119     }
7120 }
7121
7122 sub windows_version {
7123     my ($ostype) = @_;
7124
7125     return 0 if !$ostype;
7126
7127     my $winversion = 0;
7128
7129     if($ostype eq 'wxp' || $ostype eq 'w2k3' || $ostype eq 'w2k') {
7130         $winversion = 5;
7131     } elsif($ostype eq 'w2k8' || $ostype eq 'wvista') {
7132         $winversion = 6;
7133     } elsif ($ostype =~ m/^win(\d+)$/) {
7134         $winversion = $1;
7135     }
7136
7137     return $winversion;
7138 }
7139
7140 sub resolve_dst_disk_format {
7141         my ($storecfg, $storeid, $src_volname, $format) = @_;
7142         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
7143
7144         if (!$format) {
7145             # if no target format is specified, use the source disk format as hint
7146             if ($src_volname) {
7147                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
7148                 $format = qemu_img_format($scfg, $src_volname);
7149             } else {
7150                 return $defFormat;
7151             }
7152         }
7153
7154         # test if requested format is supported - else use default
7155         my $supported = grep { $_ eq $format } @$validFormats;
7156         $format = $defFormat if !$supported;
7157         return $format;
7158 }
7159
7160 sub resolve_first_disk {
7161     my $conf = shift;
7162     my @disks = PVE::QemuServer::valid_drive_names();
7163     my $firstdisk;
7164     foreach my $ds (reverse @disks) {
7165         next if !$conf->{$ds};
7166         my $disk = PVE::QemuServer::parse_drive($ds, $conf->{$ds});
7167         next if PVE::QemuServer::drive_is_cdrom($disk);
7168         $firstdisk = $ds;
7169     }
7170     return $firstdisk;
7171 }
7172
7173 sub generate_uuid {
7174     my ($uuid, $uuid_str);
7175     UUID::generate($uuid);
7176     UUID::unparse($uuid, $uuid_str);
7177     return $uuid_str;
7178 }
7179
7180 sub generate_smbios1_uuid {
7181     return "uuid=".generate_uuid();
7182 }
7183
7184 sub nbd_stop {
7185     my ($vmid) = @_;
7186
7187     mon_cmd($vmid, 'nbd-server-stop');
7188 }
7189
7190 sub create_reboot_request {
7191     my ($vmid) = @_;
7192     open(my $fh, '>', "/run/qemu-server/$vmid.reboot")
7193         or die "failed to create reboot trigger file: $!\n";
7194     close($fh);
7195 }
7196
7197 sub clear_reboot_request {
7198     my ($vmid) = @_;
7199     my $path = "/run/qemu-server/$vmid.reboot";
7200     my $res = 0;
7201
7202     $res = unlink($path);
7203     die "could not remove reboot request for $vmid: $!"
7204         if !$res && $! != POSIX::ENOENT;
7205
7206     return $res;
7207 }
7208
7209 # bash completion helper
7210
7211 sub complete_backup_archives {
7212     my ($cmdname, $pname, $cvalue) = @_;
7213
7214     my $cfg = PVE::Storage::config();
7215
7216     my $storeid;
7217
7218     if ($cvalue =~ m/^([^:]+):/) {
7219         $storeid = $1;
7220     }
7221
7222     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
7223
7224     my $res = [];
7225     foreach my $id (keys %$data) {
7226         foreach my $item (@{$data->{$id}}) {
7227             next if $item->{format} !~ m/^vma\.(gz|lzo)$/;
7228             push @$res, $item->{volid} if defined($item->{volid});
7229         }
7230     }
7231
7232     return $res;
7233 }
7234
7235 my $complete_vmid_full = sub {
7236     my ($running) = @_;
7237
7238     my $idlist = vmstatus();
7239
7240     my $res = [];
7241
7242     foreach my $id (keys %$idlist) {
7243         my $d = $idlist->{$id};
7244         if (defined($running)) {
7245             next if $d->{template};
7246             next if $running && $d->{status} ne 'running';
7247             next if !$running && $d->{status} eq 'running';
7248         }
7249         push @$res, $id;
7250
7251     }
7252     return $res;
7253 };
7254
7255 sub complete_vmid {
7256     return &$complete_vmid_full();
7257 }
7258
7259 sub complete_vmid_stopped {
7260     return &$complete_vmid_full(0);
7261 }
7262
7263 sub complete_vmid_running {
7264     return &$complete_vmid_full(1);
7265 }
7266
7267 sub complete_storage {
7268
7269     my $cfg = PVE::Storage::config();
7270     my $ids = $cfg->{ids};
7271
7272     my $res = [];
7273     foreach my $sid (keys %$ids) {
7274         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
7275         next if !$ids->{$sid}->{content}->{images};
7276         push @$res, $sid;
7277     }
7278
7279     return $res;
7280 }
7281
7282 sub complete_migration_storage {
7283     my ($cmd, $param, $current_value, $all_args) = @_;
7284
7285     my $targetnode = @$all_args[1];
7286
7287     my $cfg = PVE::Storage::config();
7288     my $ids = $cfg->{ids};
7289
7290     my $res = [];
7291     foreach my $sid (keys %$ids) {
7292         next if !PVE::Storage::storage_check_enabled($cfg, $sid, $targetnode, 1);
7293         next if !$ids->{$sid}->{content}->{images};
7294         push @$res, $sid;
7295     }
7296
7297     return $res;
7298 }
7299
7300 1;