PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5
   6 use Cwd 'abs_path';
   7 use Digest::SHA;
   8 use Fcntl ':flock';
   9 use Fcntl;
  10 use File::Basename;
  11 use File::Copy qw(copy);
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use IO::Dir;
  16 use IO::File;
  17 use IO::Handle;
  18 use IO::Select;
  19 use IO::Socket::UNIX;
  20 use IPC::Open3;
  21 use JSON;
  22 use MIME::Base64;
  23 use POSIX;
  24 use Storable qw(dclone);
  25 use Time::HiRes qw(gettimeofday);
  26 use URI::Escape;
  27 use UUID;
  28
  29 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file cfs_lock_file);
  30 use PVE::DataCenterConfig;
  31 use PVE::Exception qw(raise raise_param_exc);
  32 use PVE::GuestHelpers;
  33 use PVE::INotify;
  34 use PVE::JSONSchema qw(get_standard_option);
  35 use PVE::ProcFSTools;
  36 use PVE::RPCEnvironment;
  37 use PVE::SafeSyslog;
  38 use PVE::Storage;
  39 use PVE::SysFSTools;
  40 use PVE::Systemd;
  41 use PVE::Tools qw(run_command lock_file lock_file_full file_read_firstline dir_glob_foreach get_host_arch $IPV6RE);
  42
  43 use PVE::QMPClient;
  44 use PVE::QemuConfig;
  45 use PVE::QemuServer::Helpers;
  46 use PVE::QemuServer::Cloudinit;
  47 use PVE::QemuServer::Memory;
  48 use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr print_pcie_root_port);
  49 use PVE::QemuServer::USB qw(parse_usb_device);
  50
  51 my $EDK2_FW_BASE = '/usr/share/pve-edk2-firmware/';
  52 my $OVMF = {
  53     x86_64 => [
  54         "$EDK2_FW_BASE/OVMF_CODE.fd",
  55         "$EDK2_FW_BASE/OVMF_VARS.fd"
  56     ],
  57     aarch64 => [
  58         "$EDK2_FW_BASE/AAVMF_CODE.fd",
  59         "$EDK2_FW_BASE/AAVMF_VARS.fd"
  60     ],
  61 };
  62
  63 my $qemu_snap_storage = { rbd => 1 };
  64
  65 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  66
  67 my $QEMU_FORMAT_RE = qr/raw|cow|qcow|qcow2|qed|vmdk|cloop/;
  68
  69 # Note about locking: we use flock on the config file protect
  70 # against concurent actions.
  71 # Aditionaly, we have a 'lock' setting in the config file. This
  72 # can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
  73 # allowed when such lock is set. But you can ignore this kind of
  74 # lock with the --skiplock flag.
  75
  76 cfs_register_file('/qemu-server/',
  77                   \&parse_vm_config,
  78                   \&write_vm_config);
  79
  80 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
  81     description => "Some command save/restore state from this location.",
  82     type => 'string',
  83     maxLength => 128,
  84     optional => 1,
  85 });
  86
  87 PVE::JSONSchema::register_standard_option('pve-qm-image-format', {
  88     type => 'string',
  89     enum => [qw(raw cow qcow qed qcow2 vmdk cloop)],
  90     description => "The drive's backing file's data format.",
  91     optional => 1,
  92 });
  93
  94 PVE::JSONSchema::register_standard_option('pve-qemu-machine', {
  95         description => "Specifies the Qemu machine type.",
  96         type => 'string',
  97         pattern => '(pc|pc(-i440fx)?-\d+(\.\d+)+(\.pxe)?|q35|pc-q35-\d+(\.\d+)+(\.pxe)?|virt(?:-\d+(\.\d+)+)?)',
  98         maxLength => 40,
  99         optional => 1,
 100 });
 101
 102 #no warnings 'redefine';
 103
 104 sub cgroups_write {
 105    my ($controller, $vmid, $option, $value) = @_;
 106
 107    my $path = "/sys/fs/cgroup/$controller/qemu.slice/$vmid.scope/$option";
 108    PVE::ProcFSTools::write_proc_entry($path, $value);
 109
 110 }
 111
 112 my $nodename = PVE::INotify::nodename();
 113
 114 my $cpu_vendor_list = {
 115     # Intel CPUs
 116     486 => 'GenuineIntel',
 117     pentium => 'GenuineIntel',
 118     pentium2  => 'GenuineIntel',
 119     pentium3  => 'GenuineIntel',
 120     coreduo => 'GenuineIntel',
 121     core2duo => 'GenuineIntel',
 122     Conroe  => 'GenuineIntel',
 123     Penryn  => 'GenuineIntel',
 124     Nehalem  => 'GenuineIntel',
 125     'Nehalem-IBRS'  => 'GenuineIntel',
 126     Westmere => 'GenuineIntel',
 127     'Westmere-IBRS' => 'GenuineIntel',
 128     SandyBridge => 'GenuineIntel',
 129     'SandyBridge-IBRS' => 'GenuineIntel',
 130     IvyBridge => 'GenuineIntel',
 131     'IvyBridge-IBRS' => 'GenuineIntel',
 132     Haswell => 'GenuineIntel',
 133     'Haswell-IBRS' => 'GenuineIntel',
 134     'Haswell-noTSX' => 'GenuineIntel',
 135     'Haswell-noTSX-IBRS' => 'GenuineIntel',
 136     Broadwell => 'GenuineIntel',
 137     'Broadwell-IBRS' => 'GenuineIntel',
 138     'Broadwell-noTSX' => 'GenuineIntel',
 139     'Broadwell-noTSX-IBRS' => 'GenuineIntel',
 140     'Skylake-Client' => 'GenuineIntel',
 141     'Skylake-Client-IBRS' => 'GenuineIntel',
 142     'Skylake-Server' => 'GenuineIntel',
 143     'Skylake-Server-IBRS' => 'GenuineIntel',
 144     'Cascadelake-Server' => 'GenuineIntel',
 145     KnightsMill => 'GenuineIntel',
 146
 147
 148     # AMD CPUs
 149     athlon => 'AuthenticAMD',
 150     phenom  => 'AuthenticAMD',
 151     Opteron_G1  => 'AuthenticAMD',
 152     Opteron_G2  => 'AuthenticAMD',
 153     Opteron_G3  => 'AuthenticAMD',
 154     Opteron_G4  => 'AuthenticAMD',
 155     Opteron_G5  => 'AuthenticAMD',
 156     EPYC => 'AuthenticAMD',
 157     'EPYC-IBPB' => 'AuthenticAMD',
 158
 159     # generic types, use vendor from host node
 160     host => 'default',
 161     kvm32 => 'default',
 162     kvm64 => 'default',
 163     qemu32 => 'default',
 164     qemu64 => 'default',
 165     max => 'default',
 166 };
 167
 168 my @supported_cpu_flags = (
 169     'pcid',
 170     'spec-ctrl',
 171     'ibpb',
 172     'ssbd',
 173     'virt-ssbd',
 174     'amd-ssbd',
 175     'amd-no-ssb',
 176     'pdpe1gb',
 177     'md-clear',
 178     'hv-tlbflush',
 179     'hv-evmcs',
 180     'aes'
 181 );
 182 my $cpu_flag = qr/[+-](@{[join('|', @supported_cpu_flags)]})/;
 183
 184 my $cpu_fmt = {
 185     cputype => {
 186         description => "Emulated CPU type.",
 187         type => 'string',
 188         enum => [ sort { "\L$a" cmp "\L$b" } keys %$cpu_vendor_list ],
 189         default => 'kvm64',
 190         default_key => 1,
 191     },
 192     hidden => {
 193         description => "Do not identify as a KVM virtual machine.",
 194         type => 'boolean',
 195         optional => 1,
 196         default => 0
 197     },
 198     'hv-vendor-id' => {
 199         type => 'string',
 200         pattern => qr/[a-zA-Z0-9]{1,12}/,
 201         format_description => 'vendor-id',
 202         description => 'The Hyper-V vendor ID. Some drivers or programs inside Windows guests need a specific ID.',
 203         optional => 1,
 204     },
 205     flags => {
 206         description => "List of additional CPU flags separated by ';'."
 207                      . " Use '+FLAG' to enable, '-FLAG' to disable a flag."
 208                      . " Currently supported flags: @{[join(', ', @supported_cpu_flags)]}.",
 209         format_description => '+FLAG[;-FLAG...]',
 210         type => 'string',
 211         pattern => qr/$cpu_flag(;$cpu_flag)*/,
 212         optional => 1,
 213     },
 214 };
 215
 216 my $watchdog_fmt = {
 217     model => {
 218         default_key => 1,
 219         type => 'string',
 220         enum => [qw(i6300esb ib700)],
 221         description => "Watchdog type to emulate.",
 222         default => 'i6300esb',
 223         optional => 1,
 224     },
 225     action => {
 226         type => 'string',
 227         enum => [qw(reset shutdown poweroff pause debug none)],
 228         description => "The action to perform if after activation the guest fails to poll the watchdog in time.",
 229         optional => 1,
 230     },
 231 };
 232 PVE::JSONSchema::register_format('pve-qm-watchdog', $watchdog_fmt);
 233
 234 my $agent_fmt = {
 235     enabled => {
 236         description => "Enable/disable Qemu GuestAgent.",
 237         type => 'boolean',
 238         default => 0,
 239         default_key => 1,
 240     },
 241     fstrim_cloned_disks => {
 242         description => "Run fstrim after cloning/moving a disk.",
 243         type => 'boolean',
 244         optional => 1,
 245         default => 0
 246     },
 247     type => {
 248         description => "Select the agent type",
 249         type => 'string',
 250         default => 'virtio',
 251         optional => 1,
 252         enum => [qw(virtio isa)],
 253     },
 254 };
 255
 256 my $vga_fmt = {
 257     type => {
 258         description => "Select the VGA type.",
 259         type => 'string',
 260         default => 'std',
 261         optional => 1,
 262         default_key => 1,
 263         enum => [qw(cirrus qxl qxl2 qxl3 qxl4 none serial0 serial1 serial2 serial3 std virtio vmware)],
 264     },
 265     memory => {
 266         description => "Sets the VGA memory (in MiB). Has no effect with serial display.",
 267         type => 'integer',
 268         optional => 1,
 269         minimum => 4,
 270         maximum => 512,
 271     },
 272 };
 273
 274 my $ivshmem_fmt = {
 275     size => {
 276         type => 'integer',
 277         minimum => 1,
 278         description => "The size of the file in MB.",
 279     },
 280     name => {
 281         type => 'string',
 282         pattern => '[a-zA-Z0-9\-]+',
 283         optional => 1,
 284         format_description => 'string',
 285         description => "The name of the file. Will be prefixed with 'pve-shm-'. Default is the VMID. Will be deleted when the VM is stopped.",
 286     },
 287 };
 288
 289 my $audio_fmt = {
 290     device => {
 291         type => 'string',
 292         enum => [qw(ich9-intel-hda intel-hda AC97)],
 293         description =>  "Configure an audio device."
 294     },
 295     driver =>  {
 296         type => 'string',
 297         enum => ['spice'],
 298         default => 'spice',
 299         optional => 1,
 300         description => "Driver backend for the audio device."
 301     },
 302 };
 303
 304 my $spice_enhancements_fmt = {
 305     foldersharing => {
 306         type => 'boolean',
 307         optional => 1,
 308         default => '0',
 309         description =>  "Enable folder sharing via SPICE. Needs Spice-WebDAV daemon installed in the VM."
 310     },
 311     videostreaming =>  {
 312         type => 'string',
 313         enum => ['off', 'all', 'filter'],
 314         default => 'off',
 315         optional => 1,
 316         description => "Enable video streaming. Uses compression for detected video streams."
 317     },
 318 };
 319
 320 my $confdesc = {
 321     onboot => {
 322         optional => 1,
 323         type => 'boolean',
 324         description => "Specifies whether a VM will be started during system bootup.",
 325         default => 0,
 326     },
 327     autostart => {
 328         optional => 1,
 329         type => 'boolean',
 330         description => "Automatic restart after crash (currently ignored).",
 331         default => 0,
 332     },
 333     hotplug => {
 334         optional => 1,
 335         type => 'string', format => 'pve-hotplug-features',
 336         description => "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
 337         default => 'network,disk,usb',
 338     },
 339     reboot => {
 340         optional => 1,
 341         type => 'boolean',
 342         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 343         default => 1,
 344     },
 345     lock => {
 346         optional => 1,
 347         type => 'string',
 348         description => "Lock/unlock the VM.",
 349         enum => [qw(backup clone create migrate rollback snapshot snapshot-delete suspending suspended)],
 350     },
 351     cpulimit => {
 352         optional => 1,
 353         type => 'number',
 354         description => "Limit of CPU usage.",
 355         verbose_description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
 356         minimum => 0,
 357         maximum => 128,
 358         default => 0,
 359     },
 360     cpuunits => {
 361         optional => 1,
 362         type => 'integer',
 363         description => "CPU weight for a VM.",
 364         verbose_description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.",
 365         minimum => 2,
 366         maximum => 262144,
 367         default => 1024,
 368     },
 369     memory => {
 370         optional => 1,
 371         type => 'integer',
 372         description => "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
 373         minimum => 16,
 374         default => 512,
 375     },
 376     balloon => {
 377         optional => 1,
 378         type => 'integer',
 379         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 380         minimum => 0,
 381     },
 382     shares => {
 383         optional => 1,
 384         type => 'integer',
 385         description => "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning. Auto-ballooning is done by pvestatd.",
 386         minimum => 0,
 387         maximum => 50000,
 388         default => 1000,
 389     },
 390     keyboard => {
 391         optional => 1,
 392         type => 'string',
 393         description => "Keybord layout for vnc server. Default is read from the '/etc/pve/datacenter.cfg' configuration file.".
 394                        "It should not be necessary to set it.",
 395         enum => PVE::Tools::kvmkeymaplist(),
 396         default => undef,
 397     },
 398     name => {
 399         optional => 1,
 400         type => 'string', format => 'dns-name',
 401         description => "Set a name for the VM. Only used on the configuration web interface.",
 402     },
 403     scsihw => {
 404         optional => 1,
 405         type => 'string',
 406         description => "SCSI controller model",
 407         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 408         default => 'lsi',
 409     },
 410     description => {
 411         optional => 1,
 412         type => 'string',
 413         description => "Description for the VM. Only used on the configuration web interface. This is saved as comment inside the configuration file.",
 414     },
 415     ostype => {
 416         optional => 1,
 417         type => 'string',
 418         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 l24 l26 solaris)],
 419         description => "Specify guest operating system.",
 420         verbose_description => <<EODESC,
 421 Specify guest operating system. This is used to enable special
 422 optimization/features for specific operating systems:
 423
 424 [horizontal]
 425 other;; unspecified OS
 426 wxp;; Microsoft Windows XP
 427 w2k;; Microsoft Windows 2000
 428 w2k3;; Microsoft Windows 2003
 429 w2k8;; Microsoft Windows 2008
 430 wvista;; Microsoft Windows Vista
 431 win7;; Microsoft Windows 7
 432 win8;; Microsoft Windows 8/2012/2012r2
 433 win10;; Microsoft Windows 10/2016
 434 l24;; Linux 2.4 Kernel
 435 l26;; Linux 2.6 - 5.X Kernel
 436 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
 437 EODESC
 438     },
 439     boot => {
 440         optional => 1,
 441         type => 'string',
 442         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n).",
 443         pattern => '[acdn]{1,4}',
 444         default => 'cdn',
 445     },
 446     bootdisk => {
 447         optional => 1,
 448         type => 'string', format => 'pve-qm-bootdisk',
 449         description => "Enable booting from specified disk.",
 450         pattern => '(ide|sata|scsi|virtio)\d+',
 451     },
 452     smp => {
 453         optional => 1,
 454         type => 'integer',
 455         description => "The number of CPUs. Please use option -sockets instead.",
 456         minimum => 1,
 457         default => 1,
 458     },
 459     sockets => {
 460         optional => 1,
 461         type => 'integer',
 462         description => "The number of CPU sockets.",
 463         minimum => 1,
 464         default => 1,
 465     },
 466     cores => {
 467         optional => 1,
 468         type => 'integer',
 469         description => "The number of cores per socket.",
 470         minimum => 1,
 471         default => 1,
 472     },
 473     numa => {
 474         optional => 1,
 475         type => 'boolean',
 476         description => "Enable/disable NUMA.",
 477         default => 0,
 478     },
 479     hugepages => {
 480         optional => 1,
 481         type => 'string',
 482         description => "Enable/disable hugepages memory.",
 483         enum => [qw(any 2 1024)],
 484     },
 485     vcpus => {
 486         optional => 1,
 487         type => 'integer',
 488         description => "Number of hotplugged vcpus.",
 489         minimum => 1,
 490         default => 0,
 491     },
 492     acpi => {
 493         optional => 1,
 494         type => 'boolean',
 495         description => "Enable/disable ACPI.",
 496         default => 1,
 497     },
 498     agent => {
 499         optional => 1,
 500         description => "Enable/disable Qemu GuestAgent and its properties.",
 501         type => 'string',
 502         format => $agent_fmt,
 503     },
 504     kvm => {
 505         optional => 1,
 506         type => 'boolean',
 507         description => "Enable/disable KVM hardware virtualization.",
 508         default => 1,
 509     },
 510     tdf => {
 511         optional => 1,
 512         type => 'boolean',
 513         description => "Enable/disable time drift fix.",
 514         default => 0,
 515     },
 516     localtime => {
 517         optional => 1,
 518         type => 'boolean',
 519         description => "Set the real time clock to local time. This is enabled by default if ostype indicates a Microsoft OS.",
 520     },
 521     freeze => {
 522         optional => 1,
 523         type => 'boolean',
 524         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 525     },
 526     vga => {
 527         optional => 1,
 528         type => 'string', format => $vga_fmt,
 529         description => "Configure the VGA hardware.",
 530         verbose_description => "Configure the VGA Hardware. If you want to use ".
 531             "high resolution modes (>= 1280x1024x16) you may need to increase " .
 532             "the vga memory option. Since QEMU 2.9 the default VGA display type " .
 533             "is 'std' for all OS types besides some Windows versions (XP and " .
 534             "older) which use 'cirrus'. The 'qxl' option enables the SPICE " .
 535             "display server. For win* OS you can select how many independent " .
 536             "displays you want, Linux guests can add displays them self.\n".
 537             "You can also run without any graphic card, using a serial device as terminal.",
 538     },
 539     watchdog => {
 540         optional => 1,
 541         type => 'string', format => 'pve-qm-watchdog',
 542         description => "Create a virtual hardware watchdog device.",
 543         verbose_description => "Create a virtual hardware watchdog device. Once enabled" .
 544             " (by a guest action), the watchdog must be periodically polled " .
 545             "by an agent inside the guest or else the watchdog will reset " .
 546             "the guest (or execute the respective action specified)",
 547     },
 548     startdate => {
 549         optional => 1,
 550         type => 'string',
 551         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 552         description => "Set the initial date of the real time clock. Valid format for date are: 'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 553         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 554         default => 'now',
 555     },
 556     startup =>  get_standard_option('pve-startup-order'),
 557     template => {
 558         optional => 1,
 559         type => 'boolean',
 560         description => "Enable/disable Template.",
 561         default => 0,
 562     },
 563     args => {
 564         optional => 1,
 565         type => 'string',
 566         description => "Arbitrary arguments passed to kvm.",
 567         verbose_description => <<EODESCR,
 568 Arbitrary arguments passed to kvm, for example:
 569
 570 args: -no-reboot -no-hpet
 571
 572 NOTE: this option is for experts only.
 573 EODESCR
 574     },
 575     tablet => {
 576         optional => 1,
 577         type => 'boolean',
 578         default => 1,
 579         description => "Enable/disable the USB tablet device.",
 580         verbose_description => "Enable/disable the USB tablet device. This device is " .
 581             "usually needed to allow absolute mouse positioning with VNC. " .
 582             "Else the mouse runs out of sync with normal VNC clients. " .
 583             "If you're running lots of console-only guests on one host, " .
 584             "you may consider disabling this to save some context switches. " .
 585             "This is turned off by default if you use spice (-vga=qxl).",
 586     },
 587     migrate_speed => {
 588         optional => 1,
 589         type => 'integer',
 590         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 591         minimum => 0,
 592         default => 0,
 593     },
 594     migrate_downtime => {
 595         optional => 1,
 596         type => 'number',
 597         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 598         minimum => 0,
 599         default => 0.1,
 600     },
 601     cdrom => {
 602         optional => 1,
 603         type => 'string', format => 'pve-qm-ide',
 604         typetext => '<volume>',
 605         description => "This is an alias for option -ide2",
 606     },
 607     cpu => {
 608         optional => 1,
 609         description => "Emulated CPU type.",
 610         type => 'string',
 611         format => $cpu_fmt,
 612     },
 613     parent => get_standard_option('pve-snapshot-name', {
 614         optional => 1,
 615         description => "Parent snapshot name. This is used internally, and should not be modified.",
 616     }),
 617     snaptime => {
 618         optional => 1,
 619         description => "Timestamp for snapshots.",
 620         type => 'integer',
 621         minimum => 0,
 622     },
 623     vmstate => {
 624         optional => 1,
 625         type => 'string', format => 'pve-volume-id',
 626         description => "Reference to a volume which stores the VM state. This is used internally for snapshots.",
 627     },
 628     vmstatestorage => get_standard_option('pve-storage-id', {
 629         description => "Default storage for VM state volumes/files.",
 630         optional => 1,
 631     }),
 632     runningmachine => get_standard_option('pve-qemu-machine', {
 633         description => "Specifies the Qemu machine type of the running vm. This is used internally for snapshots.",
 634     }),
 635     machine => get_standard_option('pve-qemu-machine'),
 636     arch => {
 637         description => "Virtual processor architecture. Defaults to the host.",
 638         optional => 1,
 639         type => 'string',
 640         enum => [qw(x86_64 aarch64)],
 641     },
 642     smbios1 => {
 643         description => "Specify SMBIOS type 1 fields.",
 644         type => 'string', format => 'pve-qm-smbios1',
 645         maxLength => 512,
 646         optional => 1,
 647     },
 648     protection => {
 649         optional => 1,
 650         type => 'boolean',
 651         description => "Sets the protection flag of the VM. This will disable the remove VM and remove disk operations.",
 652         default => 0,
 653     },
 654     bios => {
 655         optional => 1,
 656         type => 'string',
 657         enum => [ qw(seabios ovmf) ],
 658         description => "Select BIOS implementation.",
 659         default => 'seabios',
 660     },
 661     vmgenid => {
 662         type => 'string',
 663         pattern => '(?:[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}|[01])',
 664         format_description => 'UUID',
 665         description => "Set VM Generation ID. Use '1' to autogenerate on create or update, pass '0' to disable explicitly.",
 666         verbose_description => "The VM generation ID (vmgenid) device exposes a".
 667             " 128-bit integer value identifier to the guest OS. This allows to".
 668             " notify the guest operating system when the virtual machine is".
 669             " executed with a different configuration (e.g. snapshot execution".
 670             " or creation from a template). The guest operating system notices".
 671             " the change, and is then able to react as appropriate by marking".
 672             " its copies of distributed databases as dirty, re-initializing its".
 673             " random number generator, etc.\n".
 674             "Note that auto-creation only works when done throug API/CLI create".
 675             " or update methods, but not when manually editing the config file.",
 676         default => "1 (autogenerated)",
 677         optional => 1,
 678     },
 679     hookscript => {
 680         type => 'string',
 681         format => 'pve-volume-id',
 682         optional => 1,
 683         description => "Script that will be executed during various steps in the vms lifetime.",
 684     },
 685     ivshmem => {
 686         type => 'string',
 687         format => $ivshmem_fmt,
 688         description => "Inter-VM shared memory. Useful for direct communication between VMs, or to the host.",
 689         optional => 1,
 690     },
 691     audio0 => {
 692         type => 'string',
 693         format => $audio_fmt,
 694         description => "Configure a audio device, useful in combination with QXL/Spice.",
 695         optional => 1
 696     },
 697     spice_enhancements => {
 698         type => 'string',
 699         format => $spice_enhancements_fmt,
 700         description => "Configure additional enhancements for SPICE.",
 701         optional => 1
 702     },
 703 };
 704
 705 my $cicustom_fmt = {
 706     meta => {
 707         type => 'string',
 708         optional => 1,
 709         description => 'Specify a custom file containing all meta data passed to the VM via cloud-init. This is provider specific meaning configdrive2 and nocloud differ.',
 710         format => 'pve-volume-id',
 711         format_description => 'volume',
 712     },
 713     network => {
 714         type => 'string',
 715         optional => 1,
 716         description => 'Specify a custom file containing all network data passed to the VM via cloud-init.',
 717         format => 'pve-volume-id',
 718         format_description => 'volume',
 719     },
 720     user => {
 721         type => 'string',
 722         optional => 1,
 723         description => 'Specify a custom file containing all user data passed to the VM via cloud-init.',
 724         format => 'pve-volume-id',
 725         format_description => 'volume',
 726     },
 727 };
 728 PVE::JSONSchema::register_format('pve-qm-cicustom', $cicustom_fmt);
 729
 730 my $confdesc_cloudinit = {
 731     citype => {
 732         optional => 1,
 733         type => 'string',
 734         description => 'Specifies the cloud-init configuration format. The default depends on the configured operating system type (`ostype`. We use the `nocloud` format for Linux, and `configdrive2` for windows.',
 735         enum => ['configdrive2', 'nocloud'],
 736     },
 737     ciuser => {
 738         optional => 1,
 739         type => 'string',
 740         description => "cloud-init: User name to change ssh keys and password for instead of the image's configured default user.",
 741     },
 742     cipassword => {
 743         optional => 1,
 744         type => 'string',
 745         description => 'cloud-init: Password to assign the user. Using this is generally not recommended. Use ssh keys instead. Also note that older cloud-init versions do not support hashed passwords.',
 746     },
 747     cicustom => {
 748         optional => 1,
 749         type => 'string',
 750         description => 'cloud-init: Specify custom files to replace the automatically generated ones at start.',
 751         format => 'pve-qm-cicustom',
 752     },
 753     searchdomain => {
 754         optional => 1,
 755         type => 'string',
 756         description => "cloud-init: Sets DNS search domains for a container. Create will automatically use the setting from the host if neither searchdomain nor nameserver are set.",
 757     },
 758     nameserver => {
 759         optional => 1,
 760         type => 'string', format => 'address-list',
 761         description => "cloud-init: Sets DNS server IP address for a container. Create will automatically use the setting from the host if neither searchdomain nor nameserver are set.",
 762     },
 763     sshkeys => {
 764         optional => 1,
 765         type => 'string',
 766         format => 'urlencoded',
 767         description => "cloud-init: Setup public SSH keys (one key per line, OpenSSH format).",
 768     },
 769 };
 770
 771 # what about other qemu settings ?
 772 #cpu => 'string',
 773 #machine => 'string',
 774 #fda => 'file',
 775 #fdb => 'file',
 776 #mtdblock => 'file',
 777 #sd => 'file',
 778 #pflash => 'file',
 779 #snapshot => 'bool',
 780 #bootp => 'file',
 781 ##tftp => 'dir',
 782 ##smb => 'dir',
 783 #kernel => 'file',
 784 #append => 'string',
 785 #initrd => 'file',
 786 ##soundhw => 'string',
 787
 788 while (my ($k, $v) = each %$confdesc) {
 789     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 790 }
 791
 792 my $MAX_IDE_DISKS = 4;
 793 my $MAX_SCSI_DISKS = 14;
 794 my $MAX_VIRTIO_DISKS = 16;
 795 my $MAX_SATA_DISKS = 6;
 796 my $MAX_USB_DEVICES = 5;
 797 my $MAX_NETS = 32;
 798 my $MAX_UNUSED_DISKS = 256;
 799 my $MAX_HOSTPCI_DEVICES = 16;
 800 my $MAX_SERIAL_PORTS = 4;
 801 my $MAX_PARALLEL_PORTS = 3;
 802 my $MAX_NUMA = 8;
 803
 804 my $numa_fmt = {
 805     cpus => {
 806         type => "string",
 807         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 808         description => "CPUs accessing this NUMA node.",
 809         format_description => "id[-id];...",
 810     },
 811     memory => {
 812         type => "number",
 813         description => "Amount of memory this NUMA node provides.",
 814         optional => 1,
 815     },
 816     hostnodes => {
 817         type => "string",
 818         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 819         description => "Host NUMA nodes to use.",
 820         format_description => "id[-id];...",
 821         optional => 1,
 822     },
 823     policy => {
 824         type => 'string',
 825         enum => [qw(preferred bind interleave)],
 826         description => "NUMA allocation policy.",
 827         optional => 1,
 828     },
 829 };
 830 PVE::JSONSchema::register_format('pve-qm-numanode', $numa_fmt);
 831 my $numadesc = {
 832     optional => 1,
 833     type => 'string', format => $numa_fmt,
 834     description => "NUMA topology.",
 835 };
 836 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 837
 838 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 839     $confdesc->{"numa$i"} = $numadesc;
 840 }
 841
 842 my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000',  'pcnet',  'virtio',
 843                       'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3',
 844                       'e1000-82540em', 'e1000-82544gc', 'e1000-82545em'];
 845 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 846
 847 my $net_fmt_bridge_descr = <<__EOD__;
 848 Bridge to attach the network device to. The Proxmox VE standard bridge
 849 is called 'vmbr0'.
 850
 851 If you do not specify a bridge, we create a kvm user (NATed) network
 852 device, which provides DHCP and DNS services. The following addresses
 853 are used:
 854
 855  10.0.2.2   Gateway
 856  10.0.2.3   DNS Server
 857  10.0.2.4   SMB Server
 858
 859 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 860 __EOD__
 861
 862 my $net_fmt = {
 863     macaddr  => get_standard_option('mac-addr', {
 864         description => "MAC address. That address must be unique withing your network. This is automatically generated if not specified.",
 865     }),
 866     model => {
 867         type => 'string',
 868         description => "Network Card Model. The 'virtio' model provides the best performance with very low CPU overhead. If your guest does not support this driver, it is usually best to use 'e1000'.",
 869         enum => $nic_model_list,
 870         default_key => 1,
 871     },
 872     (map { $_ => { keyAlias => 'model', alias => 'macaddr' }} @$nic_model_list),
 873     bridge => {
 874         type => 'string',
 875         description => $net_fmt_bridge_descr,
 876         format_description => 'bridge',
 877         optional => 1,
 878     },
 879     queues => {
 880         type => 'integer',
 881         minimum => 0, maximum => 16,
 882         description => 'Number of packet queues to be used on the device.',
 883         optional => 1,
 884     },
 885     rate => {
 886         type => 'number',
 887         minimum => 0,
 888         description => "Rate limit in mbps (megabytes per second) as floating point number.",
 889         optional => 1,
 890     },
 891     tag => {
 892         type => 'integer',
 893         minimum => 1, maximum => 4094,
 894         description => 'VLAN tag to apply to packets on this interface.',
 895         optional => 1,
 896     },
 897     trunks => {
 898         type => 'string',
 899         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 900         description => 'VLAN trunks to pass through this interface.',
 901         format_description => 'vlanid[;vlanid...]',
 902         optional => 1,
 903     },
 904     firewall => {
 905         type => 'boolean',
 906         description => 'Whether this interface should be protected by the firewall.',
 907         optional => 1,
 908     },
 909     link_down => {
 910         type => 'boolean',
 911         description => 'Whether this interface should be disconnected (like pulling the plug).',
 912         optional => 1,
 913     },
 914 };
 915
 916 my $netdesc = {
 917     optional => 1,
 918     type => 'string', format => $net_fmt,
 919     description => "Specify network devices.",
 920 };
 921
 922 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 923
 924 my $ipconfig_fmt = {
 925     ip => {
 926         type => 'string',
 927         format => 'pve-ipv4-config',
 928         format_description => 'IPv4Format/CIDR',
 929         description => 'IPv4 address in CIDR format.',
 930         optional => 1,
 931         default => 'dhcp',
 932     },
 933     gw => {
 934         type => 'string',
 935         format => 'ipv4',
 936         format_description => 'GatewayIPv4',
 937         description => 'Default gateway for IPv4 traffic.',
 938         optional => 1,
 939         requires => 'ip',
 940     },
 941     ip6 => {
 942         type => 'string',
 943         format => 'pve-ipv6-config',
 944         format_description => 'IPv6Format/CIDR',
 945         description => 'IPv6 address in CIDR format.',
 946         optional => 1,
 947         default => 'dhcp',
 948     },
 949     gw6 => {
 950         type => 'string',
 951         format => 'ipv6',
 952         format_description => 'GatewayIPv6',
 953         description => 'Default gateway for IPv6 traffic.',
 954         optional => 1,
 955         requires => 'ip6',
 956     },
 957 };
 958 PVE::JSONSchema::register_format('pve-qm-ipconfig', $ipconfig_fmt);
 959 my $ipconfigdesc = {
 960     optional => 1,
 961     type => 'string', format => 'pve-qm-ipconfig',
 962     description => <<'EODESCR',
 963 cloud-init: Specify IP addresses and gateways for the corresponding interface.
 964
 965 IP addresses use CIDR notation, gateways are optional but need an IP of the same type specified.
 966
 967 The special string 'dhcp' can be used for IP addresses to use DHCP, in which case no explicit gateway should be provided.
 968 For IPv6 the special string 'auto' can be used to use stateless autoconfiguration.
 969
 970 If cloud-init is enabled and neither an IPv4 nor an IPv6 address is specified, it defaults to using dhcp on IPv4.
 971 EODESCR
 972 };
 973 PVE::JSONSchema::register_standard_option("pve-qm-ipconfig", $netdesc);
 974
 975 for (my $i = 0; $i < $MAX_NETS; $i++)  {
 976     $confdesc->{"net$i"} = $netdesc;
 977     $confdesc_cloudinit->{"ipconfig$i"} = $ipconfigdesc;
 978 }
 979
 980 foreach my $key (keys %$confdesc_cloudinit) {
 981     $confdesc->{$key} = $confdesc_cloudinit->{$key};
 982 }
 983
 984 PVE::JSONSchema::register_format('pve-volume-id-or-qm-path', \&verify_volume_id_or_qm_path);
 985 sub verify_volume_id_or_qm_path {
 986     my ($volid, $noerr) = @_;
 987
 988     if ($volid eq 'none' || $volid eq 'cdrom' || $volid =~ m|^/|) {
 989         return $volid;
 990     }
 991
 992     # if its neither 'none' nor 'cdrom' nor a path, check if its a volume-id
 993     $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') };
 994     if ($@) {
 995         return undef if $noerr;
 996         die $@;
 997     }
 998     return $volid;
 999 }
1000
1001 my $drivename_hash;
1002
1003 my %drivedesc_base = (
1004     volume => { alias => 'file' },
1005     file => {
1006         type => 'string',
1007         format => 'pve-volume-id-or-qm-path',
1008         default_key => 1,
1009         format_description => 'volume',
1010         description => "The drive's backing volume.",
1011     },
1012     media => {
1013         type => 'string',
1014         enum => [qw(cdrom disk)],
1015         description => "The drive's media type.",
1016         default => 'disk',
1017         optional => 1
1018     },
1019     cyls => {
1020         type => 'integer',
1021         description => "Force the drive's physical geometry to have a specific cylinder count.",
1022         optional => 1
1023     },
1024     heads => {
1025         type => 'integer',
1026         description => "Force the drive's physical geometry to have a specific head count.",
1027         optional => 1
1028     },
1029     secs => {
1030         type => 'integer',
1031         description => "Force the drive's physical geometry to have a specific sector count.",
1032         optional => 1
1033     },
1034     trans => {
1035         type => 'string',
1036         enum => [qw(none lba auto)],
1037         description => "Force disk geometry bios translation mode.",
1038         optional => 1,
1039     },
1040     snapshot => {
1041         type => 'boolean',
1042         description => "Controls qemu's snapshot mode feature."
1043             . " If activated, changes made to the disk are temporary and will"
1044             . " be discarded when the VM is shutdown.",
1045         optional => 1,
1046     },
1047     cache => {
1048         type => 'string',
1049         enum => [qw(none writethrough writeback unsafe directsync)],
1050         description => "The drive's cache mode",
1051         optional => 1,
1052     },
1053     format => get_standard_option('pve-qm-image-format'),
1054     size => {
1055         type => 'string',
1056         format => 'disk-size',
1057         format_description => 'DiskSize',
1058         description => "Disk size. This is purely informational and has no effect.",
1059         optional => 1,
1060     },
1061     backup => {
1062         type => 'boolean',
1063         description => "Whether the drive should be included when making backups.",
1064         optional => 1,
1065     },
1066     replicate => {
1067         type => 'boolean',
1068         description => 'Whether the drive should considered for replication jobs.',
1069         optional => 1,
1070         default => 1,
1071     },
1072     rerror => {
1073         type => 'string',
1074         enum => [qw(ignore report stop)],
1075         description => 'Read error action.',
1076         optional => 1,
1077     },
1078     werror => {
1079         type => 'string',
1080         enum => [qw(enospc ignore report stop)],
1081         description => 'Write error action.',
1082         optional => 1,
1083     },
1084     aio => {
1085         type => 'string',
1086         enum => [qw(native threads)],
1087         description => 'AIO type to use.',
1088         optional => 1,
1089     },
1090     discard => {
1091         type => 'string',
1092         enum => [qw(ignore on)],
1093         description => 'Controls whether to pass discard/trim requests to the underlying storage.',
1094         optional => 1,
1095     },
1096     detect_zeroes => {
1097         type => 'boolean',
1098         description => 'Controls whether to detect and try to optimize writes of zeroes.',
1099         optional => 1,
1100     },
1101     serial => {
1102         type => 'string',
1103         format => 'urlencoded',
1104         format_description => 'serial',
1105         maxLength => 20*3, # *3 since it's %xx url enoded
1106         description => "The drive's reported serial number, url-encoded, up to 20 bytes long.",
1107         optional => 1,
1108     },
1109     shared => {
1110         type => 'boolean',
1111         description => 'Mark this locally-managed volume as available on all nodes',
1112         verbose_description => "Mark this locally-managed volume as available on all nodes.\n\nWARNING: This option does not share the volume automatically, it assumes it is shared already!",
1113         optional => 1,
1114         default => 0,
1115     }
1116 );
1117
1118 my %iothread_fmt = ( iothread => {
1119         type => 'boolean',
1120         description => "Whether to use iothreads for this drive",
1121         optional => 1,
1122 });
1123
1124 my %model_fmt = (
1125     model => {
1126         type => 'string',
1127         format => 'urlencoded',
1128         format_description => 'model',
1129         maxLength => 40*3, # *3 since it's %xx url enoded
1130         description => "The drive's reported model name, url-encoded, up to 40 bytes long.",
1131         optional => 1,
1132     },
1133 );
1134
1135 my %queues_fmt = (
1136     queues => {
1137         type => 'integer',
1138         description => "Number of queues.",
1139         minimum => 2,
1140         optional => 1
1141     }
1142 );
1143
1144 my %scsiblock_fmt = (
1145     scsiblock => {
1146         type => 'boolean',
1147         description => "whether to use scsi-block for full passthrough of host block device\n\nWARNING: can lead to I/O errors in combination with low memory or high memory fragmentation on host",
1148         optional => 1,
1149         default => 0,
1150     },
1151 );
1152
1153 my %ssd_fmt = (
1154     ssd => {
1155         type => 'boolean',
1156         description => "Whether to expose this drive as an SSD, rather than a rotational hard disk.",
1157         optional => 1,
1158     },
1159 );
1160
1161 my %wwn_fmt = (
1162     wwn => {
1163         type => 'string',
1164         pattern => qr/^(0x)[0-9a-fA-F]{16}/,
1165         format_description => 'wwn',
1166         description => "The drive's worldwide name, encoded as 16 bytes hex string, prefixed by '0x'.",
1167         optional => 1,
1168     },
1169 );
1170
1171 my $add_throttle_desc = sub {
1172     my ($key, $type, $what, $unit, $longunit, $minimum) = @_;
1173     my $d = {
1174         type => $type,
1175         format_description => $unit,
1176         description => "Maximum $what in $longunit.",
1177         optional => 1,
1178     };
1179     $d->{minimum} = $minimum if defined($minimum);
1180     $drivedesc_base{$key} = $d;
1181 };
1182 # throughput: (leaky bucket)
1183 $add_throttle_desc->('bps',     'integer', 'r/w speed',   'bps',  'bytes per second');
1184 $add_throttle_desc->('bps_rd',  'integer', 'read speed',  'bps',  'bytes per second');
1185 $add_throttle_desc->('bps_wr',  'integer', 'write speed', 'bps',  'bytes per second');
1186 $add_throttle_desc->('mbps',    'number',  'r/w speed',   'mbps', 'megabytes per second');
1187 $add_throttle_desc->('mbps_rd', 'number',  'read speed',  'mbps', 'megabytes per second');
1188 $add_throttle_desc->('mbps_wr', 'number',  'write speed', 'mbps', 'megabytes per second');
1189 $add_throttle_desc->('iops',    'integer', 'r/w I/O',     'iops', 'operations per second');
1190 $add_throttle_desc->('iops_rd', 'integer', 'read I/O',    'iops', 'operations per second');
1191 $add_throttle_desc->('iops_wr', 'integer', 'write I/O',   'iops', 'operations per second');
1192
1193 # pools: (pool of IO before throttling starts taking effect)
1194 $add_throttle_desc->('mbps_max',    'number',  'unthrottled r/w pool',       'mbps', 'megabytes per second');
1195 $add_throttle_desc->('mbps_rd_max', 'number',  'unthrottled read pool',      'mbps', 'megabytes per second');
1196 $add_throttle_desc->('mbps_wr_max', 'number',  'unthrottled write pool',     'mbps', 'megabytes per second');
1197 $add_throttle_desc->('iops_max',    'integer', 'unthrottled r/w I/O pool',   'iops', 'operations per second');
1198 $add_throttle_desc->('iops_rd_max', 'integer', 'unthrottled read I/O pool',  'iops', 'operations per second');
1199 $add_throttle_desc->('iops_wr_max', 'integer', 'unthrottled write I/O pool', 'iops', 'operations per second');
1200
1201 # burst lengths
1202 $add_throttle_desc->('bps_max_length',     'integer', 'length of I/O bursts',       'seconds', 'seconds', 1);
1203 $add_throttle_desc->('bps_rd_max_length',  'integer', 'length of read I/O bursts',  'seconds', 'seconds', 1);
1204 $add_throttle_desc->('bps_wr_max_length',  'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
1205 $add_throttle_desc->('iops_max_length',    'integer', 'length of I/O bursts',       'seconds', 'seconds', 1);
1206 $add_throttle_desc->('iops_rd_max_length', 'integer', 'length of read I/O bursts',  'seconds', 'seconds', 1);
1207 $add_throttle_desc->('iops_wr_max_length', 'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
1208
1209 # legacy support
1210 $drivedesc_base{'bps_rd_length'} = { alias => 'bps_rd_max_length' };
1211 $drivedesc_base{'bps_wr_length'} = { alias => 'bps_wr_max_length' };
1212 $drivedesc_base{'iops_rd_length'} = { alias => 'iops_rd_max_length' };
1213 $drivedesc_base{'iops_wr_length'} = { alias => 'iops_wr_max_length' };
1214
1215 my $ide_fmt = {
1216     %drivedesc_base,
1217     %model_fmt,
1218     %ssd_fmt,
1219     %wwn_fmt,
1220 };
1221 PVE::JSONSchema::register_format("pve-qm-ide", $ide_fmt);
1222
1223 my $idedesc = {
1224     optional => 1,
1225     type => 'string', format => $ide_fmt,
1226     description => "Use volume as IDE hard disk or CD-ROM (n is 0 to " .($MAX_IDE_DISKS -1) . ").",
1227 };
1228 PVE::JSONSchema::register_standard_option("pve-qm-ide", $idedesc);
1229
1230 my $scsi_fmt = {
1231     %drivedesc_base,
1232     %iothread_fmt,
1233     %queues_fmt,
1234     %scsiblock_fmt,
1235     %ssd_fmt,
1236     %wwn_fmt,
1237 };
1238 my $scsidesc = {
1239     optional => 1,
1240     type => 'string', format => $scsi_fmt,
1241     description => "Use volume as SCSI hard disk or CD-ROM (n is 0 to " . ($MAX_SCSI_DISKS - 1) . ").",
1242 };
1243 PVE::JSONSchema::register_standard_option("pve-qm-scsi", $scsidesc);
1244
1245 my $sata_fmt = {
1246     %drivedesc_base,
1247     %ssd_fmt,
1248     %wwn_fmt,
1249 };
1250 my $satadesc = {
1251     optional => 1,
1252     type => 'string', format => $sata_fmt,
1253     description => "Use volume as SATA hard disk or CD-ROM (n is 0 to " . ($MAX_SATA_DISKS - 1). ").",
1254 };
1255 PVE::JSONSchema::register_standard_option("pve-qm-sata", $satadesc);
1256
1257 my $virtio_fmt = {
1258     %drivedesc_base,
1259     %iothread_fmt,
1260 };
1261 my $virtiodesc = {
1262     optional => 1,
1263     type => 'string', format => $virtio_fmt,
1264     description => "Use volume as VIRTIO hard disk (n is 0 to " . ($MAX_VIRTIO_DISKS - 1) . ").",
1265 };
1266 PVE::JSONSchema::register_standard_option("pve-qm-virtio", $virtiodesc);
1267
1268 my $alldrive_fmt = {
1269     %drivedesc_base,
1270     %iothread_fmt,
1271     %model_fmt,
1272     %queues_fmt,
1273     %scsiblock_fmt,
1274     %ssd_fmt,
1275     %wwn_fmt,
1276 };
1277
1278 my $efidisk_fmt = {
1279     volume => { alias => 'file' },
1280     file => {
1281         type => 'string',
1282         format => 'pve-volume-id-or-qm-path',
1283         default_key => 1,
1284         format_description => 'volume',
1285         description => "The drive's backing volume.",
1286     },
1287     format => get_standard_option('pve-qm-image-format'),
1288     size => {
1289         type => 'string',
1290         format => 'disk-size',
1291         format_description => 'DiskSize',
1292         description => "Disk size. This is purely informational and has no effect.",
1293         optional => 1,
1294     },
1295 };
1296
1297 my $efidisk_desc = {
1298     optional => 1,
1299     type => 'string', format => $efidisk_fmt,
1300     description => "Configure a Disk for storing EFI vars",
1301 };
1302
1303 PVE::JSONSchema::register_standard_option("pve-qm-efidisk", $efidisk_desc);
1304
1305 my $usb_fmt = {
1306     host => {
1307         default_key => 1,
1308         type => 'string', format => 'pve-qm-usb-device',
1309         format_description => 'HOSTUSBDEVICE|spice',
1310         description => <<EODESCR,
1311 The Host USB device or port or the value 'spice'. HOSTUSBDEVICE syntax is:
1312
1313  'bus-port(.port)*' (decimal numbers) or
1314  'vendor_id:product_id' (hexadeciaml numbers) or
1315  'spice'
1316
1317 You can use the 'lsusb -t' command to list existing usb devices.
1318
1319 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1320
1321 The value 'spice' can be used to add a usb redirection devices for spice.
1322 EODESCR
1323     },
1324     usb3 => {
1325         optional => 1,
1326         type => 'boolean',
1327         description => "Specifies whether if given host option is a USB3 device or port.",
1328         default => 0,
1329     },
1330 };
1331
1332 my $usbdesc = {
1333     optional => 1,
1334     type => 'string', format => $usb_fmt,
1335     description => "Configure an USB device (n is 0 to 4).",
1336 };
1337 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
1338
1339 my $PCIRE = qr/[a-f0-9]{2}:[a-f0-9]{2}(?:\.[a-f0-9])?/;
1340 my $hostpci_fmt = {
1341     host => {
1342         default_key => 1,
1343         type => 'string',
1344         pattern => qr/$PCIRE(;$PCIRE)*/,
1345         format_description => 'HOSTPCIID[;HOSTPCIID2...]',
1346         description => <<EODESCR,
1347 Host PCI device pass through. The PCI ID of a host's PCI device or a list
1348 of PCI virtual functions of the host. HOSTPCIID syntax is:
1349
1350 'bus:dev.func' (hexadecimal numbers)
1351
1352 You can us the 'lspci' command to list existing PCI devices.
1353 EODESCR
1354     },
1355     rombar => {
1356         type => 'boolean',
1357         description =>  "Specify whether or not the device's ROM will be visible in the guest's memory map.",
1358         optional => 1,
1359         default => 1,
1360     },
1361     romfile => {
1362         type => 'string',
1363         pattern => '[^,;]+',
1364         format_description => 'string',
1365         description => "Custom pci device rom filename (must be located in /usr/share/kvm/).",
1366         optional => 1,
1367     },
1368     pcie => {
1369         type => 'boolean',
1370         description =>  "Choose the PCI-express bus (needs the 'q35' machine model).",
1371         optional => 1,
1372         default => 0,
1373     },
1374     'x-vga' => {
1375         type => 'boolean',
1376         description =>  "Enable vfio-vga device support.",
1377         optional => 1,
1378         default => 0,
1379     },
1380     'mdev' => {
1381         type => 'string',
1382         format_description => 'string',
1383         pattern => '[^/\.:]+',
1384         optional => 1,
1385         description => <<EODESCR
1386 The type of mediated device to use.
1387 An instance of this type will be created on startup of the VM and
1388 will be cleaned up when the VM stops.
1389 EODESCR
1390     }
1391 };
1392 PVE::JSONSchema::register_format('pve-qm-hostpci', $hostpci_fmt);
1393
1394 my $hostpcidesc = {
1395         optional => 1,
1396         type => 'string', format => 'pve-qm-hostpci',
1397         description => "Map host PCI devices into guest.",
1398         verbose_description =>  <<EODESCR,
1399 Map host PCI devices into guest.
1400
1401 NOTE: This option allows direct access to host hardware. So it is no longer
1402 possible to migrate such machines - use with special care.
1403
1404 CAUTION: Experimental! User reported problems with this option.
1405 EODESCR
1406 };
1407 PVE::JSONSchema::register_standard_option("pve-qm-hostpci", $hostpcidesc);
1408
1409 my $serialdesc = {
1410         optional => 1,
1411         type => 'string',
1412         pattern => '(/dev/.+|socket)',
1413         description =>  "Create a serial device inside the VM (n is 0 to 3)",
1414         verbose_description =>  <<EODESCR,
1415 Create a serial device inside the VM (n is 0 to 3), and pass through a
1416 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1417 host side (use 'qm terminal' to open a terminal connection).
1418
1419 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines - use with special care.
1420
1421 CAUTION: Experimental! User reported problems with this option.
1422 EODESCR
1423 };
1424
1425 my $paralleldesc= {
1426         optional => 1,
1427         type => 'string',
1428         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
1429         description =>  "Map host parallel devices (n is 0 to 2).",
1430         verbose_description =>  <<EODESCR,
1431 Map host parallel devices (n is 0 to 2).
1432
1433 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1434
1435 CAUTION: Experimental! User reported problems with this option.
1436 EODESCR
1437 };
1438
1439 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
1440     $confdesc->{"parallel$i"} = $paralleldesc;
1441 }
1442
1443 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
1444     $confdesc->{"serial$i"} = $serialdesc;
1445 }
1446
1447 for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
1448     $confdesc->{"hostpci$i"} = $hostpcidesc;
1449 }
1450
1451 for (my $i = 0; $i < $MAX_IDE_DISKS; $i++)  {
1452     $drivename_hash->{"ide$i"} = 1;
1453     $confdesc->{"ide$i"} = $idedesc;
1454 }
1455
1456 for (my $i = 0; $i < $MAX_SATA_DISKS; $i++)  {
1457     $drivename_hash->{"sata$i"} = 1;
1458     $confdesc->{"sata$i"} = $satadesc;
1459 }
1460
1461 for (my $i = 0; $i < $MAX_SCSI_DISKS; $i++)  {
1462     $drivename_hash->{"scsi$i"} = 1;
1463     $confdesc->{"scsi$i"} = $scsidesc ;
1464 }
1465
1466 for (my $i = 0; $i < $MAX_VIRTIO_DISKS; $i++)  {
1467     $drivename_hash->{"virtio$i"} = 1;
1468     $confdesc->{"virtio$i"} = $virtiodesc;
1469 }
1470
1471 $drivename_hash->{efidisk0} = 1;
1472 $confdesc->{efidisk0} = $efidisk_desc;
1473
1474 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
1475     $confdesc->{"usb$i"} = $usbdesc;
1476 }
1477
1478 my $unuseddesc = {
1479     optional => 1,
1480     type => 'string', format => 'pve-volume-id',
1481     description => "Reference to unused volumes. This is used internally, and should not be modified manually.",
1482 };
1483
1484 for (my $i = 0; $i < $MAX_UNUSED_DISKS; $i++)  {
1485     $confdesc->{"unused$i"} = $unuseddesc;
1486 }
1487
1488 my $kvm_api_version = 0;
1489
1490 sub kvm_version {
1491     return $kvm_api_version if $kvm_api_version;
1492
1493     open my $fh, '<', '/dev/kvm'
1494         or return undef;
1495
1496     # 0xae00 => KVM_GET_API_VERSION
1497     $kvm_api_version = ioctl($fh, 0xae00, 0);
1498
1499     return $kvm_api_version;
1500 }
1501
1502 my $kvm_user_version = {};
1503 my $kvm_mtime = {};
1504
1505 sub kvm_user_version {
1506     my ($binary) = @_;
1507
1508     $binary //= get_command_for_arch(get_host_arch()); # get the native arch by default
1509     my $st = stat($binary);
1510
1511     my $cachedmtime = $kvm_mtime->{$binary} // -1;
1512     return $kvm_user_version->{$binary} if $kvm_user_version->{$binary} &&
1513         $cachedmtime == $st->mtime;
1514
1515     $kvm_user_version->{$binary} = 'unknown';
1516     $kvm_mtime->{$binary} = $st->mtime;
1517
1518     my $code = sub {
1519         my $line = shift;
1520         if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1521             $kvm_user_version->{$binary} = $2;
1522         }
1523     };
1524
1525     eval { run_command([$binary, '--version'], outfunc => $code); };
1526     warn $@ if $@;
1527
1528     return $kvm_user_version->{$binary};
1529
1530 }
1531
1532 sub kernel_has_vhost_net {
1533     return -c '/dev/vhost-net';
1534 }
1535
1536 sub valid_drive_names {
1537     # order is important - used to autoselect boot disk
1538     return ((map { "ide$_" } (0 .. ($MAX_IDE_DISKS - 1))),
1539             (map { "scsi$_" } (0 .. ($MAX_SCSI_DISKS - 1))),
1540             (map { "virtio$_" } (0 .. ($MAX_VIRTIO_DISKS - 1))),
1541             (map { "sata$_" } (0 .. ($MAX_SATA_DISKS - 1))),
1542             'efidisk0');
1543 }
1544
1545 sub is_valid_drivename {
1546     my $dev = shift;
1547
1548     return defined($drivename_hash->{$dev});
1549 }
1550
1551 sub option_exists {
1552     my $key = shift;
1553     return defined($confdesc->{$key});
1554 }
1555
1556 my $cdrom_path;
1557 sub get_cdrom_path {
1558
1559     return  $cdrom_path if $cdrom_path;
1560
1561     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
1562     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
1563     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
1564 }
1565
1566 sub get_iso_path {
1567     my ($storecfg, $vmid, $cdrom) = @_;
1568
1569     if ($cdrom eq 'cdrom') {
1570         return get_cdrom_path();
1571     } elsif ($cdrom eq 'none') {
1572         return '';
1573     } elsif ($cdrom =~ m|^/|) {
1574         return $cdrom;
1575     } else {
1576         return PVE::Storage::path($storecfg, $cdrom);
1577     }
1578 }
1579
1580 # try to convert old style file names to volume IDs
1581 sub filename_to_volume_id {
1582     my ($vmid, $file, $media) = @_;
1583
1584      if (!($file eq 'none' || $file eq 'cdrom' ||
1585           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1586
1587         return undef if $file =~ m|/|;
1588
1589         if ($media && $media eq 'cdrom') {
1590             $file = "local:iso/$file";
1591         } else {
1592             $file = "local:$vmid/$file";
1593         }
1594     }
1595
1596     return $file;
1597 }
1598
1599 sub verify_media_type {
1600     my ($opt, $vtype, $media) = @_;
1601
1602     return if !$media;
1603
1604     my $etype;
1605     if ($media eq 'disk') {
1606         $etype = 'images';
1607     } elsif ($media eq 'cdrom') {
1608         $etype = 'iso';
1609     } else {
1610         die "internal error";
1611     }
1612
1613     return if ($vtype eq $etype);
1614
1615     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1616 }
1617
1618 sub cleanup_drive_path {
1619     my ($opt, $storecfg, $drive) = @_;
1620
1621     # try to convert filesystem paths to volume IDs
1622
1623     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1624         ($drive->{file} !~ m|^/dev/.+|) &&
1625         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1626         ($drive->{file} !~ m/^\d+$/)) {
1627         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1628         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"}) if !$vtype;
1629         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1630         verify_media_type($opt, $vtype, $drive->{media});
1631         $drive->{file} = $volid;
1632     }
1633
1634     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1635 }
1636
1637 sub parse_hotplug_features {
1638     my ($data) = @_;
1639
1640     my $res = {};
1641
1642     return $res if $data eq '0';
1643
1644     $data = $confdesc->{hotplug}->{default} if $data eq '1';
1645
1646     foreach my $feature (PVE::Tools::split_list($data)) {
1647         if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
1648             $res->{$1} = 1;
1649         } else {
1650             die "invalid hotplug feature '$feature'\n";
1651         }
1652     }
1653     return $res;
1654 }
1655
1656 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1657 sub pve_verify_hotplug_features {
1658     my ($value, $noerr) = @_;
1659
1660     return $value if parse_hotplug_features($value);
1661
1662     return undef if $noerr;
1663
1664     die "unable to parse hotplug option\n";
1665 }
1666
1667 # ideX = [volume=]volume-id[,media=d][,cyls=c,heads=h,secs=s[,trans=t]]
1668 #        [,snapshot=on|off][,cache=on|off][,format=f][,backup=yes|no]
1669 #        [,rerror=ignore|report|stop][,werror=enospc|ignore|report|stop]
1670 #        [,aio=native|threads][,discard=ignore|on][,detect_zeroes=on|off]
1671 #        [,iothread=on][,serial=serial][,model=model]
1672
1673 sub parse_drive {
1674     my ($key, $data) = @_;
1675
1676     my ($interface, $index);
1677
1678     if ($key =~ m/^([^\d]+)(\d+)$/) {
1679         $interface = $1;
1680         $index = $2;
1681     } else {
1682         return undef;
1683     }
1684
1685     my $desc = $key =~ /^unused\d+$/ ? $alldrive_fmt
1686                                      : $confdesc->{$key}->{format};
1687     if (!$desc) {
1688         warn "invalid drive key: $key\n";
1689         return undef;
1690     }
1691     my $res = eval { PVE::JSONSchema::parse_property_string($desc, $data) };
1692     return undef if !$res;
1693     $res->{interface} = $interface;
1694     $res->{index} = $index;
1695
1696     my $error = 0;
1697     foreach my $opt (qw(bps bps_rd bps_wr)) {
1698         if (my $bps = defined(delete $res->{$opt})) {
1699             if (defined($res->{"m$opt"})) {
1700                 warn "both $opt and m$opt specified\n";
1701                 ++$error;
1702                 next;
1703             }
1704             $res->{"m$opt"} = sprintf("%.3f", $bps / (1024*1024.0));
1705         }
1706     }
1707
1708     # can't use the schema's 'requires' because of the mbps* => bps* "transforming aliases"
1709     for my $requirement (
1710         [mbps_max => 'mbps'],
1711         [mbps_rd_max => 'mbps_rd'],
1712         [mbps_wr_max => 'mbps_wr'],
1713         [miops_max => 'miops'],
1714         [miops_rd_max => 'miops_rd'],
1715         [miops_wr_max => 'miops_wr'],
1716         [bps_max_length => 'mbps_max'],
1717         [bps_rd_max_length => 'mbps_rd_max'],
1718         [bps_wr_max_length => 'mbps_wr_max'],
1719         [iops_max_length => 'iops_max'],
1720         [iops_rd_max_length => 'iops_rd_max'],
1721         [iops_wr_max_length => 'iops_wr_max']) {
1722         my ($option, $requires) = @$requirement;
1723         if ($res->{$option} && !$res->{$requires}) {
1724             warn "$option requires $requires\n";
1725             ++$error;
1726         }
1727     }
1728
1729     return undef if $error;
1730
1731     return undef if $res->{mbps_rd} && $res->{mbps};
1732     return undef if $res->{mbps_wr} && $res->{mbps};
1733     return undef if $res->{iops_rd} && $res->{iops};
1734     return undef if $res->{iops_wr} && $res->{iops};
1735
1736     if ($res->{media} && ($res->{media} eq 'cdrom')) {
1737         return undef if $res->{snapshot} || $res->{trans} || $res->{format};
1738         return undef if $res->{heads} || $res->{secs} || $res->{cyls};
1739         return undef if $res->{interface} eq 'virtio';
1740     }
1741
1742     if (my $size = $res->{size}) {
1743         return undef if !defined($res->{size} = PVE::JSONSchema::parse_size($size));
1744     }
1745
1746     return $res;
1747 }
1748
1749 sub print_drive {
1750     my ($vmid, $drive) = @_;
1751     my $data = { %$drive };
1752     delete $data->{$_} for qw(index interface);
1753     return PVE::JSONSchema::print_property_string($data, $alldrive_fmt);
1754 }
1755
1756 sub scsi_inquiry {
1757     my($fh, $noerr) = @_;
1758
1759     my $SG_IO = 0x2285;
1760     my $SG_GET_VERSION_NUM = 0x2282;
1761
1762     my $versionbuf = "\x00" x 8;
1763     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1764     if (!$ret) {
1765         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1766         return undef;
1767     }
1768     my $version = unpack("I", $versionbuf);
1769     if ($version < 30000) {
1770         die "scsi generic interface too old\n"  if !$noerr;
1771         return undef;
1772     }
1773
1774     my $buf = "\x00" x 36;
1775     my $sensebuf = "\x00" x 8;
1776     my $cmd = pack("C x3 C x1", 0x12, 36);
1777
1778     # see /usr/include/scsi/sg.h
1779     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1780
1781     my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1782                       length($sensebuf), 0, length($buf), $buf,
1783                       $cmd, $sensebuf, 6000);
1784
1785     $ret = ioctl($fh, $SG_IO, $packet);
1786     if (!$ret) {
1787         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1788         return undef;
1789     }
1790
1791     my @res = unpack($sg_io_hdr_t, $packet);
1792     if ($res[17] || $res[18]) {
1793         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1794         return undef;
1795     }
1796
1797     my $res = {};
1798     (my $byte0, my $byte1, $res->{vendor},
1799      $res->{product}, $res->{revision}) = unpack("C C x6 A8 A16 A4", $buf);
1800
1801     $res->{removable} = $byte1 & 128 ? 1 : 0;
1802     $res->{type} = $byte0 & 31;
1803
1804     return $res;
1805 }
1806
1807 sub path_is_scsi {
1808     my ($path) = @_;
1809
1810     my $fh = IO::File->new("+<$path") || return undef;
1811     my $res = scsi_inquiry($fh, 1);
1812     close($fh);
1813
1814     return $res;
1815 }
1816
1817 sub machine_type_is_q35 {
1818     my ($conf) = @_;
1819
1820     return $conf->{machine} && ($conf->{machine} =~ m/q35/) ? 1 : 0;
1821 }
1822
1823 sub print_tabletdevice_full {
1824     my ($conf, $arch) = @_;
1825
1826     my $q35 = machine_type_is_q35($conf);
1827
1828     # we use uhci for old VMs because tablet driver was buggy in older qemu
1829     my $usbbus;
1830     if (machine_type_is_q35($conf) || $arch eq 'aarch64') {
1831         $usbbus = 'ehci';
1832     } else {
1833         $usbbus = 'uhci';
1834     }
1835
1836     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1837 }
1838
1839 sub print_keyboarddevice_full {
1840     my ($conf, $arch, $machine) = @_;
1841
1842     return undef if $arch ne 'aarch64';
1843
1844     return "usb-kbd,id=keyboard,bus=ehci.0,port=2";
1845 }
1846
1847 sub print_drivedevice_full {
1848     my ($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type) = @_;
1849
1850     my $device = '';
1851     my $maxdev = 0;
1852
1853     if ($drive->{interface} eq 'virtio') {
1854         my $pciaddr = print_pci_addr("$drive->{interface}$drive->{index}", $bridges, $arch, $machine_type);
1855         $device = "virtio-blk-pci,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}$pciaddr";
1856         $device .= ",iothread=iothread-$drive->{interface}$drive->{index}" if $drive->{iothread};
1857     } elsif ($drive->{interface} eq 'scsi') {
1858
1859         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1860         my $unit = $drive->{index} % $maxdev;
1861         my $devicetype = 'hd';
1862         my $path = '';
1863         if (drive_is_cdrom($drive)) {
1864             $devicetype = 'cd';
1865         } else {
1866             if ($drive->{file} =~ m|^/|) {
1867                 $path = $drive->{file};
1868                 if (my $info = path_is_scsi($path)) {
1869                     if ($info->{type} == 0 && $drive->{scsiblock}) {
1870                         $devicetype = 'block';
1871                     } elsif ($info->{type} == 1) { # tape
1872                         $devicetype = 'generic';
1873                     }
1874                 }
1875             } else {
1876                  $path = PVE::Storage::path($storecfg, $drive->{file});
1877             }
1878
1879             # for compatibility only, we prefer scsi-hd (#2408, #2355, #2380)
1880             if ($path =~ m/^iscsi\:\/\// &&
1881                !qemu_machine_feature_enabled($machine_type, undef, 4, 1)) {
1882                 $devicetype = 'generic';
1883             }
1884         }
1885
1886         if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)){
1887             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1888         } else {
1889             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0,lun=$drive->{index},drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1890         }
1891
1892         if ($drive->{ssd} && ($devicetype eq 'block' || $devicetype eq 'hd')) {
1893             $device .= ",rotation_rate=1";
1894         }
1895         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1896
1897     } elsif ($drive->{interface} eq 'ide' || $drive->{interface} eq 'sata') {
1898         my $maxdev = ($drive->{interface} eq 'sata') ? $MAX_SATA_DISKS : 2;
1899         my $controller = int($drive->{index} / $maxdev);
1900         my $unit = $drive->{index} % $maxdev;
1901         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1902
1903         $device = "ide-$devicetype";
1904         if ($drive->{interface} eq 'ide') {
1905             $device .= ",bus=ide.$controller,unit=$unit";
1906         } else {
1907             $device .= ",bus=ahci$controller.$unit";
1908         }
1909         $device .= ",drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1910
1911         if ($devicetype eq 'hd') {
1912             if (my $model = $drive->{model}) {
1913                 $model = URI::Escape::uri_unescape($model);
1914                 $device .= ",model=$model";
1915             }
1916             if ($drive->{ssd}) {
1917                 $device .= ",rotation_rate=1";
1918             }
1919         }
1920         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1921     } elsif ($drive->{interface} eq 'usb') {
1922         die "implement me";
1923         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1924     } else {
1925         die "unsupported interface type";
1926     }
1927
1928     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1929
1930     if (my $serial = $drive->{serial}) {
1931         $serial = URI::Escape::uri_unescape($serial);
1932         $device .= ",serial=$serial";
1933     }
1934
1935
1936     return $device;
1937 }
1938
1939 sub get_initiator_name {
1940     my $initiator;
1941
1942     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return undef;
1943     while (defined(my $line = <$fh>)) {
1944         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1945         $initiator = $1;
1946         last;
1947     }
1948     $fh->close();
1949
1950     return $initiator;
1951 }
1952
1953 sub print_drive_full {
1954     my ($storecfg, $vmid, $drive) = @_;
1955
1956     my $path;
1957     my $volid = $drive->{file};
1958     my $format;
1959
1960     if (drive_is_cdrom($drive)) {
1961         $path = get_iso_path($storecfg, $vmid, $volid);
1962     } else {
1963         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1964         if ($storeid) {
1965             $path = PVE::Storage::path($storecfg, $volid);
1966             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
1967             $format = qemu_img_format($scfg, $volname);
1968         } else {
1969             $path = $volid;
1970             $format = "raw";
1971         }
1972    }
1973
1974     my $opts = '';
1975     my @qemu_drive_options = qw(heads secs cyls trans media format cache rerror werror aio discard);
1976     foreach my $o (@qemu_drive_options) {
1977         $opts .= ",$o=$drive->{$o}" if defined($drive->{$o});
1978     }
1979
1980     # snapshot only accepts on|off
1981     if (defined($drive->{snapshot})) {
1982         my $v = $drive->{snapshot} ? 'on' : 'off';
1983         $opts .= ",snapshot=$v";
1984     }
1985
1986     foreach my $type (['', '-total'], [_rd => '-read'], [_wr => '-write']) {
1987         my ($dir, $qmpname) = @$type;
1988         if (my $v = $drive->{"mbps$dir"}) {
1989             $opts .= ",throttling.bps$qmpname=".int($v*1024*1024);
1990         }
1991         if (my $v = $drive->{"mbps${dir}_max"}) {
1992             $opts .= ",throttling.bps$qmpname-max=".int($v*1024*1024);
1993         }
1994         if (my $v = $drive->{"bps${dir}_max_length"}) {
1995             $opts .= ",throttling.bps$qmpname-max-length=$v";
1996         }
1997         if (my $v = $drive->{"iops${dir}"}) {
1998             $opts .= ",throttling.iops$qmpname=$v";
1999         }
2000         if (my $v = $drive->{"iops${dir}_max"}) {
2001             $opts .= ",throttling.iops$qmpname-max=$v";
2002         }
2003         if (my $v = $drive->{"iops${dir}_max_length"}) {
2004             $opts .= ",throttling.iops$qmpname-max-length=$v";
2005         }
2006     }
2007
2008     $opts .= ",format=$format" if $format && !$drive->{format};
2009
2010     my $cache_direct = 0;
2011
2012     if (my $cache = $drive->{cache}) {
2013         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
2014     } elsif (!drive_is_cdrom($drive)) {
2015         $opts .= ",cache=none";
2016         $cache_direct = 1;
2017     }
2018
2019     # aio native works only with O_DIRECT
2020     if (!$drive->{aio}) {
2021         if($cache_direct) {
2022             $opts .= ",aio=native";
2023         } else {
2024             $opts .= ",aio=threads";
2025         }
2026     }
2027
2028     if (!drive_is_cdrom($drive)) {
2029         my $detectzeroes;
2030         if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
2031             $detectzeroes = 'off';
2032         } elsif ($drive->{discard}) {
2033             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
2034         } else {
2035             # This used to be our default with discard not being specified:
2036             $detectzeroes = 'on';
2037         }
2038         $opts .= ",detect-zeroes=$detectzeroes" if $detectzeroes;
2039     }
2040
2041     my $pathinfo = $path ? "file=$path," : '';
2042
2043     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
2044 }
2045
2046 sub print_netdevice_full {
2047     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files, $arch, $machine_type) = @_;
2048
2049     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
2050
2051     my $device = $net->{model};
2052     if ($net->{model} eq 'virtio') {
2053          $device = 'virtio-net-pci';
2054      };
2055
2056     my $pciaddr = print_pci_addr("$netid", $bridges, $arch, $machine_type);
2057     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
2058     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
2059         #Consider we have N queues, the number of vectors needed is 2*N + 2 (plus one config interrupt and control vq)
2060         my $vectors = $net->{queues} * 2 + 2;
2061         $tmpstr .= ",vectors=$vectors,mq=on";
2062     }
2063     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
2064
2065     if ($use_old_bios_files) {
2066         my $romfile;
2067         if ($device eq 'virtio-net-pci') {
2068             $romfile = 'pxe-virtio.rom';
2069         } elsif ($device eq 'e1000') {
2070             $romfile = 'pxe-e1000.rom';
2071         } elsif ($device eq 'ne2k') {
2072             $romfile = 'pxe-ne2k_pci.rom';
2073         } elsif ($device eq 'pcnet') {
2074             $romfile = 'pxe-pcnet.rom';
2075         } elsif ($device eq 'rtl8139') {
2076             $romfile = 'pxe-rtl8139.rom';
2077         }
2078         $tmpstr .= ",romfile=$romfile" if $romfile;
2079     }
2080
2081     return $tmpstr;
2082 }
2083
2084 sub print_netdev_full {
2085     my ($vmid, $conf, $arch, $net, $netid, $hotplug) = @_;
2086
2087     my $i = '';
2088     if ($netid =~ m/^net(\d+)$/) {
2089         $i = int($1);
2090     }
2091
2092     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
2093
2094     my $ifname = "tap${vmid}i$i";
2095
2096     # kvm uses TUNSETIFF ioctl, and that limits ifname length
2097     die "interface name '$ifname' is too long (max 15 character)\n"
2098         if length($ifname) >= 16;
2099
2100     my $vhostparam = '';
2101     if (is_native($arch)) {
2102         $vhostparam = ',vhost=on' if kernel_has_vhost_net() && $net->{model} eq 'virtio';
2103     }
2104
2105     my $vmname = $conf->{name} || "vm$vmid";
2106
2107     my $netdev = "";
2108     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
2109
2110     if ($net->{bridge}) {
2111         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script,downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
2112     } else {
2113         $netdev = "type=user,id=$netid,hostname=$vmname";
2114     }
2115
2116     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
2117
2118     return $netdev;
2119 }
2120
2121
2122 sub print_cpu_device {
2123     my ($conf, $id) = @_;
2124
2125     my $kvm = $conf->{kvm} // 1;
2126     my $cpu = $kvm ? "kvm64" : "qemu64";
2127     if (my $cputype = $conf->{cpu}) {
2128         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
2129             or die "Cannot parse cpu description: $cputype\n";
2130         $cpu = $cpuconf->{cputype};
2131     }
2132
2133     my $cores = $conf->{cores} || 1;
2134
2135     my $current_core = ($id - 1) % $cores;
2136     my $current_socket = int(($id - 1 - $current_core)/$cores);
2137
2138     return "$cpu-x86_64-cpu,id=cpu$id,socket-id=$current_socket,core-id=$current_core,thread-id=0";
2139 }
2140
2141 my $vga_map = {
2142     'cirrus' => 'cirrus-vga',
2143     'std' => 'VGA',
2144     'vmware' => 'vmware-svga',
2145     'virtio' => 'virtio-vga',
2146 };
2147
2148 sub print_vga_device {
2149     my ($conf, $vga, $arch, $kvmver, $machine, $id, $qxlnum, $bridges) = @_;
2150
2151     my $type = $vga_map->{$vga->{type}};
2152     if ($arch eq 'aarch64' && defined($type) && $type eq 'virtio-vga') {
2153         $type = 'virtio-gpu';
2154     }
2155     my $vgamem_mb = $vga->{memory};
2156
2157     my $max_outputs = '';
2158     if ($qxlnum) {
2159         $type = $id ? 'qxl' : 'qxl-vga';
2160
2161         if (!$conf->{ostype} || $conf->{ostype} =~ m/^(?:l\d\d)|(?:other)$/) {
2162             # set max outputs so linux can have up to 4 qxl displays with one device
2163             if (qemu_machine_feature_enabled($machine, $kvmver, 4, 1)) {
2164                 $max_outputs = ",max_outputs=4";
2165             }
2166         }
2167     }
2168
2169     die "no devicetype for $vga->{type}\n" if !$type;
2170
2171     my $memory = "";
2172     if ($vgamem_mb) {
2173         if ($vga->{type} eq 'virtio') {
2174             my $bytes = PVE::Tools::convert_size($vgamem_mb, "mb" => "b");
2175             $memory = ",max_hostmem=$bytes";
2176         } elsif ($qxlnum) {
2177             # from https://www.spice-space.org/multiple-monitors.html
2178             $memory = ",vgamem_mb=$vga->{memory}";
2179             my $ram = $vgamem_mb * 4;
2180             my $vram = $vgamem_mb * 2;
2181             $memory .= ",ram_size_mb=$ram,vram_size_mb=$vram";
2182         } else {
2183             $memory = ",vgamem_mb=$vga->{memory}";
2184         }
2185     } elsif ($qxlnum && $id) {
2186         $memory = ",ram_size=67108864,vram_size=33554432";
2187     }
2188
2189     my $q35 = machine_type_is_q35($conf);
2190     my $vgaid = "vga" . ($id // '');
2191     my $pciaddr;
2192
2193     if ($q35 && $vgaid eq 'vga') {
2194         # the first display uses pcie.0 bus on q35 machines
2195         $pciaddr = print_pcie_addr($vgaid, $bridges, $arch, $machine);
2196     } else {
2197         $pciaddr = print_pci_addr($vgaid, $bridges, $arch, $machine);
2198     }
2199
2200     return "$type,id=${vgaid}${memory}${max_outputs}${pciaddr}";
2201 }
2202
2203 sub drive_is_cloudinit {
2204     my ($drive) = @_;
2205     return $drive->{file} =~ m@[:/]vm-\d+-cloudinit(?:\.$QEMU_FORMAT_RE)?$@;
2206 }
2207
2208 sub drive_is_cdrom {
2209     my ($drive, $exclude_cloudinit) = @_;
2210
2211     return 0 if $exclude_cloudinit && drive_is_cloudinit($drive);
2212
2213     return $drive && $drive->{media} && ($drive->{media} eq 'cdrom');
2214
2215 }
2216
2217 sub parse_number_sets {
2218     my ($set) = @_;
2219     my $res = [];
2220     foreach my $part (split(/;/, $set)) {
2221         if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
2222             die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
2223             push @$res, [ $1, $2 ];
2224         } else {
2225             die "invalid range: $part\n";
2226         }
2227     }
2228     return $res;
2229 }
2230
2231 sub parse_numa {
2232     my ($data) = @_;
2233
2234     my $res = PVE::JSONSchema::parse_property_string($numa_fmt, $data);
2235     $res->{cpus} = parse_number_sets($res->{cpus}) if defined($res->{cpus});
2236     $res->{hostnodes} = parse_number_sets($res->{hostnodes}) if defined($res->{hostnodes});
2237     return $res;
2238 }
2239
2240 sub parse_hostpci {
2241     my ($value) = @_;
2242
2243     return undef if !$value;
2244
2245     my $res = PVE::JSONSchema::parse_property_string($hostpci_fmt, $value);
2246
2247     my @idlist = split(/;/, $res->{host});
2248     delete $res->{host};
2249     foreach my $id (@idlist) {
2250         if ($id =~ m/\./) { # full id 00:00.1
2251             push @{$res->{pciid}}, {
2252                 id => $id,
2253             };
2254         } else { # partial id 00:00
2255             $res->{pciid} = PVE::SysFSTools::lspci($id);
2256         }
2257     }
2258     return $res;
2259 }
2260
2261 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
2262 sub parse_net {
2263     my ($data) = @_;
2264
2265     my $res = eval { PVE::JSONSchema::parse_property_string($net_fmt, $data) };
2266     if ($@) {
2267         warn $@;
2268         return undef;
2269     }
2270     if (!defined($res->{macaddr})) {
2271         my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
2272         $res->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix});
2273     }
2274     return $res;
2275 }
2276
2277 # ipconfigX ip=cidr,gw=ip,ip6=cidr,gw6=ip
2278 sub parse_ipconfig {
2279     my ($data) = @_;
2280
2281     my $res = eval { PVE::JSONSchema::parse_property_string($ipconfig_fmt, $data) };
2282     if ($@) {
2283         warn $@;
2284         return undef;
2285     }
2286
2287     if ($res->{gw} && !$res->{ip}) {
2288         warn 'gateway specified without specifying an IP address';
2289         return undef;
2290     }
2291     if ($res->{gw6} && !$res->{ip6}) {
2292         warn 'IPv6 gateway specified without specifying an IPv6 address';
2293         return undef;
2294     }
2295     if ($res->{gw} && $res->{ip} eq 'dhcp') {
2296         warn 'gateway specified together with DHCP';
2297         return undef;
2298     }
2299     if ($res->{gw6} && $res->{ip6} !~ /^$IPV6RE/) {
2300         # gw6 + auto/dhcp
2301         warn "IPv6 gateway specified together with $res->{ip6} address";
2302         return undef;
2303     }
2304
2305     if (!$res->{ip} && !$res->{ip6}) {
2306         return { ip => 'dhcp', ip6 => 'dhcp' };
2307     }
2308
2309     return $res;
2310 }
2311
2312 sub print_net {
2313     my $net = shift;
2314
2315     return PVE::JSONSchema::print_property_string($net, $net_fmt);
2316 }
2317
2318 sub add_random_macs {
2319     my ($settings) = @_;
2320
2321     foreach my $opt (keys %$settings) {
2322         next if $opt !~ m/^net(\d+)$/;
2323         my $net = parse_net($settings->{$opt});
2324         next if !$net;
2325         $settings->{$opt} = print_net($net);
2326     }
2327 }
2328
2329 sub vm_is_volid_owner {
2330     my ($storecfg, $vmid, $volid) = @_;
2331
2332     if ($volid !~  m|^/|) {
2333         my ($path, $owner);
2334         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
2335         if ($owner && ($owner == $vmid)) {
2336             return 1;
2337         }
2338     }
2339
2340     return undef;
2341 }
2342
2343 sub vmconfig_register_unused_drive {
2344     my ($storecfg, $vmid, $conf, $drive) = @_;
2345
2346     if (drive_is_cloudinit($drive)) {
2347         eval { PVE::Storage::vdisk_free($storecfg, $drive->{file}) };
2348         warn $@ if $@;
2349     } elsif (!drive_is_cdrom($drive)) {
2350         my $volid = $drive->{file};
2351         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
2352             PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
2353         }
2354     }
2355 }
2356
2357 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str][,base64=bool]
2358 my $smbios1_fmt = {
2359     uuid => {
2360         type => 'string',
2361         pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
2362         format_description => 'UUID',
2363         description => "Set SMBIOS1 UUID.",
2364         optional => 1,
2365     },
2366     version => {
2367         type => 'string',
2368         pattern => '[A-Za-z0-9+\/]+={0,2}',
2369         format_description => 'Base64 encoded string',
2370         description => "Set SMBIOS1 version.",
2371         optional => 1,
2372     },
2373     serial => {
2374         type => 'string',
2375         pattern => '[A-Za-z0-9+\/]+={0,2}',
2376         format_description => 'Base64 encoded string',
2377         description => "Set SMBIOS1 serial number.",
2378         optional => 1,
2379     },
2380     manufacturer => {
2381         type => 'string',
2382         pattern => '[A-Za-z0-9+\/]+={0,2}',
2383         format_description => 'Base64 encoded string',
2384         description => "Set SMBIOS1 manufacturer.",
2385         optional => 1,
2386     },
2387     product => {
2388         type => 'string',
2389         pattern => '[A-Za-z0-9+\/]+={0,2}',
2390         format_description => 'Base64 encoded string',
2391         description => "Set SMBIOS1 product ID.",
2392         optional => 1,
2393     },
2394     sku => {
2395         type => 'string',
2396         pattern => '[A-Za-z0-9+\/]+={0,2}',
2397         format_description => 'Base64 encoded string',
2398         description => "Set SMBIOS1 SKU string.",
2399         optional => 1,
2400     },
2401     family => {
2402         type => 'string',
2403         pattern => '[A-Za-z0-9+\/]+={0,2}',
2404         format_description => 'Base64 encoded string',
2405         description => "Set SMBIOS1 family string.",
2406         optional => 1,
2407     },
2408     base64 => {
2409         type => 'boolean',
2410         description => 'Flag to indicate that the SMBIOS values are base64 encoded',
2411         optional => 1,
2412     },
2413 };
2414
2415 sub parse_smbios1 {
2416     my ($data) = @_;
2417
2418     my $res = eval { PVE::JSONSchema::parse_property_string($smbios1_fmt, $data) };
2419     warn $@ if $@;
2420     return $res;
2421 }
2422
2423 sub print_smbios1 {
2424     my ($smbios1) = @_;
2425     return PVE::JSONSchema::print_property_string($smbios1, $smbios1_fmt);
2426 }
2427
2428 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_fmt);
2429
2430 PVE::JSONSchema::register_format('pve-qm-bootdisk', \&verify_bootdisk);
2431 sub verify_bootdisk {
2432     my ($value, $noerr) = @_;
2433
2434     return $value if is_valid_drivename($value);
2435
2436     return undef if $noerr;
2437
2438     die "invalid boot disk '$value'\n";
2439 }
2440
2441 sub parse_watchdog {
2442     my ($value) = @_;
2443
2444     return undef if !$value;
2445
2446     my $res = eval { PVE::JSONSchema::parse_property_string($watchdog_fmt, $value) };
2447     warn $@ if $@;
2448     return $res;
2449 }
2450
2451 sub parse_guest_agent {
2452     my ($value) = @_;
2453
2454     return {} if !defined($value->{agent});
2455
2456     my $res = eval { PVE::JSONSchema::parse_property_string($agent_fmt, $value->{agent}) };
2457     warn $@ if $@;
2458
2459     # if the agent is disabled ignore the other potentially set properties
2460     return {} if !$res->{enabled};
2461     return $res;
2462 }
2463
2464 sub parse_vga {
2465     my ($value) = @_;
2466
2467     return {} if !$value;
2468     my $res = eval { PVE::JSONSchema::parse_property_string($vga_fmt, $value) };
2469     warn $@ if $@;
2470     return $res;
2471 }
2472
2473 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
2474 sub verify_usb_device {
2475     my ($value, $noerr) = @_;
2476
2477     return $value if parse_usb_device($value);
2478
2479     return undef if $noerr;
2480
2481     die "unable to parse usb device\n";
2482 }
2483
2484 # add JSON properties for create and set function
2485 sub json_config_properties {
2486     my $prop = shift;
2487
2488     foreach my $opt (keys %$confdesc) {
2489         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate' || $opt eq 'runningmachine';
2490         $prop->{$opt} = $confdesc->{$opt};
2491     }
2492
2493     return $prop;
2494 }
2495
2496 # return copy of $confdesc_cloudinit to generate documentation
2497 sub cloudinit_config_properties {
2498
2499     return dclone($confdesc_cloudinit);
2500 }
2501
2502 sub check_type {
2503     my ($key, $value) = @_;
2504
2505     die "unknown setting '$key'\n" if !$confdesc->{$key};
2506
2507     my $type = $confdesc->{$key}->{type};
2508
2509     if (!defined($value)) {
2510         die "got undefined value\n";
2511     }
2512
2513     if ($value =~ m/[\n\r]/) {
2514         die "property contains a line feed\n";
2515     }
2516
2517     if ($type eq 'boolean') {
2518         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
2519         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
2520         die "type check ('boolean') failed - got '$value'\n";
2521     } elsif ($type eq 'integer') {
2522         return int($1) if $value =~ m/^(\d+)$/;
2523         die "type check ('integer') failed - got '$value'\n";
2524     } elsif ($type eq 'number') {
2525         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
2526         die "type check ('number') failed - got '$value'\n";
2527     } elsif ($type eq 'string') {
2528         if (my $fmt = $confdesc->{$key}->{format}) {
2529             PVE::JSONSchema::check_format($fmt, $value);
2530             return $value;
2531         }
2532         $value =~ s/^\"(.*)\"$/$1/;
2533         return $value;
2534     } else {
2535         die "internal error"
2536     }
2537 }
2538
2539 sub destroy_vm {
2540     my ($storecfg, $vmid, $skiplock, $replacement_conf) = @_;
2541
2542     my $conf = PVE::QemuConfig->load_config($vmid);
2543
2544     PVE::QemuConfig->check_lock($conf) if !$skiplock;
2545
2546     if ($conf->{template}) {
2547         # check if any base image is still used by a linked clone
2548         foreach_drive($conf, sub {
2549                 my ($ds, $drive) = @_;
2550                 return if drive_is_cdrom($drive);
2551
2552                 my $volid = $drive->{file};
2553                 return if !$volid || $volid =~ m|^/|;
2554
2555                 die "base volume '$volid' is still in use by linked cloned\n"
2556                     if PVE::Storage::volume_is_base_and_used($storecfg, $volid);
2557
2558         });
2559     }
2560
2561     # only remove disks owned by this VM
2562     foreach_drive($conf, sub {
2563         my ($ds, $drive) = @_;
2564         return if drive_is_cdrom($drive, 1);
2565
2566         my $volid = $drive->{file};
2567         return if !$volid || $volid =~ m|^/|;
2568
2569         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
2570         return if !$path || !$owner || ($owner != $vmid);
2571
2572         eval { PVE::Storage::vdisk_free($storecfg, $volid) };
2573         warn "Could not remove disk '$volid', check manually: $@" if $@;
2574     });
2575
2576     # also remove unused disk
2577     my $vmdisks = PVE::Storage::vdisk_list($storecfg, undef, $vmid);
2578     PVE::Storage::foreach_volid($vmdisks, sub {
2579         my ($volid, $sid, $volname, $d) = @_;
2580         eval { PVE::Storage::vdisk_free($storecfg, $volid) };
2581         warn $@ if $@;
2582     });
2583
2584     if (defined $replacement_conf) {
2585         PVE::QemuConfig->write_config($vmid, $replacement_conf);
2586     } else {
2587         PVE::QemuConfig->destroy_config($vmid);
2588     }
2589 }
2590
2591 sub parse_vm_config {
2592     my ($filename, $raw) = @_;
2593
2594     return undef if !defined($raw);
2595
2596     my $res = {
2597         digest => Digest::SHA::sha1_hex($raw),
2598         snapshots => {},
2599         pending => {},
2600     };
2601
2602     $filename =~ m|/qemu-server/(\d+)\.conf$|
2603         || die "got strange filename '$filename'";
2604
2605     my $vmid = $1;
2606
2607     my $conf = $res;
2608     my $descr;
2609     my $section = '';
2610
2611     my @lines = split(/\n/, $raw);
2612     foreach my $line (@lines) {
2613         next if $line =~ m/^\s*$/;
2614
2615         if ($line =~ m/^\[PENDING\]\s*$/i) {
2616             $section = 'pending';
2617             if (defined($descr)) {
2618                 $descr =~ s/\s+$//;
2619                 $conf->{description} = $descr;
2620             }
2621             $descr = undef;
2622             $conf = $res->{$section} = {};
2623             next;
2624
2625         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2626             $section = $1;
2627             if (defined($descr)) {
2628                 $descr =~ s/\s+$//;
2629                 $conf->{description} = $descr;
2630             }
2631             $descr = undef;
2632             $conf = $res->{snapshots}->{$section} = {};
2633             next;
2634         }
2635
2636         if ($line =~ m/^\#(.*)\s*$/) {
2637             $descr = '' if !defined($descr);
2638             $descr .= PVE::Tools::decode_text($1) . "\n";
2639             next;
2640         }
2641
2642         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2643             $descr = '' if !defined($descr);
2644             $descr .= PVE::Tools::decode_text($2);
2645         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2646             $conf->{snapstate} = $1;
2647         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2648             my $key = $1;
2649             my $value = $2;
2650             $conf->{$key} = $value;
2651         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2652             my $value = $1;
2653             if ($section eq 'pending') {
2654                 $conf->{delete} = $value; # we parse this later
2655             } else {
2656                 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
2657             }
2658         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(.+?)\s*$/) {
2659             my $key = $1;
2660             my $value = $2;
2661             eval { $value = check_type($key, $value); };
2662             if ($@) {
2663                 warn "vm $vmid - unable to parse value of '$key' - $@";
2664             } else {
2665                 $key = 'ide2' if $key eq 'cdrom';
2666                 my $fmt = $confdesc->{$key}->{format};
2667                 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2668                     my $v = parse_drive($key, $value);
2669                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2670                         $v->{file} = $volid;
2671                         $value = print_drive($vmid, $v);
2672                     } else {
2673                         warn "vm $vmid - unable to parse value of '$key'\n";
2674                         next;
2675                     }
2676                 }
2677
2678                 $conf->{$key} = $value;
2679             }
2680         }
2681     }
2682
2683     if (defined($descr)) {
2684         $descr =~ s/\s+$//;
2685         $conf->{description} = $descr;
2686     }
2687     delete $res->{snapstate}; # just to be sure
2688
2689     return $res;
2690 }
2691
2692 sub write_vm_config {
2693     my ($filename, $conf) = @_;
2694
2695     delete $conf->{snapstate}; # just to be sure
2696
2697     if ($conf->{cdrom}) {
2698         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2699         $conf->{ide2} = $conf->{cdrom};
2700         delete $conf->{cdrom};
2701     }
2702
2703     # we do not use 'smp' any longer
2704     if ($conf->{sockets}) {
2705         delete $conf->{smp};
2706     } elsif ($conf->{smp}) {
2707         $conf->{sockets} = $conf->{smp};
2708         delete $conf->{cores};
2709         delete $conf->{smp};
2710     }
2711
2712     my $used_volids = {};
2713
2714     my $cleanup_config = sub {
2715         my ($cref, $pending, $snapname) = @_;
2716
2717         foreach my $key (keys %$cref) {
2718             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2719                 $key eq 'snapstate' || $key eq 'pending';
2720             my $value = $cref->{$key};
2721             if ($key eq 'delete') {
2722                 die "propertry 'delete' is only allowed in [PENDING]\n"
2723                     if !$pending;
2724                 # fixme: check syntax?
2725                 next;
2726             }
2727             eval { $value = check_type($key, $value); };
2728             die "unable to parse value of '$key' - $@" if $@;
2729
2730             $cref->{$key} = $value;
2731
2732             if (!$snapname && is_valid_drivename($key)) {
2733                 my $drive = parse_drive($key, $value);
2734                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2735             }
2736         }
2737     };
2738
2739     &$cleanup_config($conf);
2740
2741     &$cleanup_config($conf->{pending}, 1);
2742
2743     foreach my $snapname (keys %{$conf->{snapshots}}) {
2744         die "internal error" if $snapname eq 'pending';
2745         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2746     }
2747
2748     # remove 'unusedX' settings if we re-add a volume
2749     foreach my $key (keys %$conf) {
2750         my $value = $conf->{$key};
2751         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2752             delete $conf->{$key};
2753         }
2754     }
2755
2756     my $generate_raw_config = sub {
2757         my ($conf, $pending) = @_;
2758
2759         my $raw = '';
2760
2761         # add description as comment to top of file
2762         if (defined(my $descr = $conf->{description})) {
2763             if ($descr) {
2764                 foreach my $cl (split(/\n/, $descr)) {
2765                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2766                 }
2767             } else {
2768                 $raw .= "#\n" if $pending;
2769             }
2770         }
2771
2772         foreach my $key (sort keys %$conf) {
2773             next if $key eq 'digest' || $key eq 'description' || $key eq 'pending' || $key eq 'snapshots';
2774             $raw .= "$key: $conf->{$key}\n";
2775         }
2776         return $raw;
2777     };
2778
2779     my $raw = &$generate_raw_config($conf);
2780
2781     if (scalar(keys %{$conf->{pending}})){
2782         $raw .= "\n[PENDING]\n";
2783         $raw .= &$generate_raw_config($conf->{pending}, 1);
2784     }
2785
2786     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2787         $raw .= "\n[$snapname]\n";
2788         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2789     }
2790
2791     return $raw;
2792 }
2793
2794 sub load_defaults {
2795
2796     my $res = {};
2797
2798     # we use static defaults from our JSON schema configuration
2799     foreach my $key (keys %$confdesc) {
2800         if (defined(my $default = $confdesc->{$key}->{default})) {
2801             $res->{$key} = $default;
2802         }
2803     }
2804
2805     return $res;
2806 }
2807
2808 sub config_list {
2809     my $vmlist = PVE::Cluster::get_vmlist();
2810     my $res = {};
2811     return $res if !$vmlist || !$vmlist->{ids};
2812     my $ids = $vmlist->{ids};
2813
2814     foreach my $vmid (keys %$ids) {
2815         my $d = $ids->{$vmid};
2816         next if !$d->{node} || $d->{node} ne $nodename;
2817         next if !$d->{type} || $d->{type} ne 'qemu';
2818         $res->{$vmid}->{exists} = 1;
2819     }
2820     return $res;
2821 }
2822
2823 # test if VM uses local resources (to prevent migration)
2824 sub check_local_resources {
2825     my ($conf, $noerr) = @_;
2826
2827     my @loc_res = ();
2828
2829     push @loc_res, "hostusb" if $conf->{hostusb}; # old syntax
2830     push @loc_res, "hostpci" if $conf->{hostpci}; # old syntax
2831
2832     push @loc_res, "ivshmem" if $conf->{ivshmem};
2833
2834     foreach my $k (keys %$conf) {
2835         next if $k =~ m/^usb/ && ($conf->{$k} =~ m/^spice(?![^,])/);
2836         # sockets are safe: they will recreated be on the target side post-migrate
2837         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2838         push @loc_res, $k if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2839     }
2840
2841     die "VM uses local resources\n" if scalar @loc_res && !$noerr;
2842
2843     return \@loc_res;
2844 }
2845
2846 # check if used storages are available on all nodes (use by migrate)
2847 sub check_storage_availability {
2848     my ($storecfg, $conf, $node) = @_;
2849
2850     foreach_drive($conf, sub {
2851         my ($ds, $drive) = @_;
2852
2853         my $volid = $drive->{file};
2854         return if !$volid;
2855
2856         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2857         return if !$sid;
2858
2859         # check if storage is available on both nodes
2860         my $scfg = PVE::Storage::storage_check_node($storecfg, $sid);
2861         PVE::Storage::storage_check_node($storecfg, $sid, $node);
2862    });
2863 }
2864
2865 # list nodes where all VM images are available (used by has_feature API)
2866 sub shared_nodes {
2867     my ($conf, $storecfg) = @_;
2868
2869     my $nodelist = PVE::Cluster::get_nodelist();
2870     my $nodehash = { map { $_ => 1 } @$nodelist };
2871     my $nodename = PVE::INotify::nodename();
2872
2873     foreach_drive($conf, sub {
2874         my ($ds, $drive) = @_;
2875
2876         my $volid = $drive->{file};
2877         return if !$volid;
2878
2879         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2880         if ($storeid) {
2881             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2882             if ($scfg->{disable}) {
2883                 $nodehash = {};
2884             } elsif (my $avail = $scfg->{nodes}) {
2885                 foreach my $node (keys %$nodehash) {
2886                     delete $nodehash->{$node} if !$avail->{$node};
2887                 }
2888             } elsif (!$scfg->{shared}) {
2889                 foreach my $node (keys %$nodehash) {
2890                     delete $nodehash->{$node} if $node ne $nodename
2891                 }
2892             }
2893         }
2894     });
2895
2896     return $nodehash
2897 }
2898
2899 sub check_local_storage_availability {
2900     my ($conf, $storecfg) = @_;
2901
2902     my $nodelist = PVE::Cluster::get_nodelist();
2903     my $nodehash = { map { $_ => {} } @$nodelist };
2904
2905     foreach_drive($conf, sub {
2906         my ($ds, $drive) = @_;
2907
2908         my $volid = $drive->{file};
2909         return if !$volid;
2910
2911         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2912         if ($storeid) {
2913             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2914
2915             if ($scfg->{disable}) {
2916                 foreach my $node (keys %$nodehash) {
2917                     $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2918                 }
2919             } elsif (my $avail = $scfg->{nodes}) {
2920                 foreach my $node (keys %$nodehash) {
2921                     if (!$avail->{$node}) {
2922                         $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2923                     }
2924                 }
2925             }
2926         }
2927     });
2928
2929     foreach my $node (values %$nodehash) {
2930         if (my $unavail = $node->{unavailable_storages}) {
2931             $node->{unavailable_storages} = [ sort keys %$unavail ];
2932         }
2933     }
2934
2935     return $nodehash
2936 }
2937
2938 sub check_cmdline {
2939     my ($pidfile, $pid) = @_;
2940
2941     my $fh = IO::File->new("/proc/$pid/cmdline", "r");
2942     if (defined($fh)) {
2943         my $line = <$fh>;
2944         $fh->close;
2945         return undef if !$line;
2946         my @param = split(/\0/, $line);
2947
2948         my $cmd = $param[0];
2949         return if !$cmd || ($cmd !~ m|kvm$| && $cmd !~ m@(?:^|/)qemu-system-[^/]+$@);
2950
2951         for (my $i = 0; $i < scalar (@param); $i++) {
2952             my $p = $param[$i];
2953             next if !$p;
2954             if (($p eq '-pidfile') || ($p eq '--pidfile')) {
2955                 my $p = $param[$i+1];
2956                 return 1 if $p && ($p eq $pidfile);
2957                 return undef;
2958             }
2959         }
2960     }
2961     return undef;
2962 }
2963
2964 sub check_running {
2965     my ($vmid, $nocheck, $node) = @_;
2966
2967     my $filename = PVE::QemuConfig->config_file($vmid, $node);
2968
2969     die "unable to find configuration file for VM $vmid - no such machine\n"
2970         if !$nocheck && ! -f $filename;
2971
2972     my $pidfile = PVE::QemuServer::Helpers::pidfile_name($vmid);
2973
2974     if (my $fd = IO::File->new("<$pidfile")) {
2975         my $st = stat($fd);
2976         my $line = <$fd>;
2977         close($fd);
2978
2979         my $mtime = $st->mtime;
2980         if ($mtime > time()) {
2981             warn "file '$filename' modified in future\n";
2982         }
2983
2984         if ($line =~ m/^(\d+)$/) {
2985             my $pid = $1;
2986             if (check_cmdline($pidfile, $pid)) {
2987                 if (my $pinfo = PVE::ProcFSTools::check_process_running($pid)) {
2988                     return $pid;
2989                 }
2990             }
2991         }
2992     }
2993
2994     return undef;
2995 }
2996
2997 sub vzlist {
2998
2999     my $vzlist = config_list();
3000
3001     my $fd = IO::Dir->new($PVE::QemuServer::Helpers::var_run_tmpdir) || return $vzlist;
3002
3003     while (defined(my $de = $fd->read)) {
3004         next if $de !~ m/^(\d+)\.pid$/;
3005         my $vmid = $1;
3006         next if !defined($vzlist->{$vmid});
3007         if (my $pid = check_running($vmid)) {
3008             $vzlist->{$vmid}->{pid} = $pid;
3009         }
3010     }
3011
3012     return $vzlist;
3013 }
3014
3015 sub disksize {
3016     my ($storecfg, $conf) = @_;
3017
3018     my $bootdisk = $conf->{bootdisk};
3019     return undef if !$bootdisk;
3020     return undef if !is_valid_drivename($bootdisk);
3021
3022     return undef if !$conf->{$bootdisk};
3023
3024     my $drive = parse_drive($bootdisk, $conf->{$bootdisk});
3025     return undef if !defined($drive);
3026
3027     return undef if drive_is_cdrom($drive);
3028
3029     my $volid = $drive->{file};
3030     return undef if !$volid;
3031
3032     return $drive->{size};
3033 }
3034
3035 our $vmstatus_return_properties = {
3036     vmid => get_standard_option('pve-vmid'),
3037     status => {
3038         description => "Qemu process status.",
3039         type => 'string',
3040         enum => ['stopped', 'running'],
3041     },
3042     maxmem => {
3043         description => "Maximum memory in bytes.",
3044         type => 'integer',
3045         optional => 1,
3046         renderer => 'bytes',
3047     },
3048     maxdisk => {
3049         description => "Root disk size in bytes.",
3050         type => 'integer',
3051         optional => 1,
3052         renderer => 'bytes',
3053     },
3054     name => {
3055         description => "VM name.",
3056         type => 'string',
3057         optional => 1,
3058     },
3059     qmpstatus => {
3060         description => "Qemu QMP agent status.",
3061         type => 'string',
3062         optional => 1,
3063     },
3064     pid => {
3065         description => "PID of running qemu process.",
3066         type => 'integer',
3067         optional => 1,
3068     },
3069     uptime => {
3070         description => "Uptime.",
3071         type => 'integer',
3072         optional => 1,
3073         renderer => 'duration',
3074     },
3075     cpus => {
3076         description => "Maximum usable CPUs.",
3077         type => 'number',
3078         optional => 1,
3079     },
3080     lock => {
3081         description => "The current config lock, if any.",
3082         type => 'string',
3083         optional => 1,
3084     }
3085 };
3086
3087 my $last_proc_pid_stat;
3088
3089 # get VM status information
3090 # This must be fast and should not block ($full == false)
3091 # We only query KVM using QMP if $full == true (this can be slow)
3092 sub vmstatus {
3093     my ($opt_vmid, $full) = @_;
3094
3095     my $res = {};
3096
3097     my $storecfg = PVE::Storage::config();
3098
3099     my $list = vzlist();
3100     my $defaults = load_defaults();
3101
3102     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
3103
3104     my $cpucount = $cpuinfo->{cpus} || 1;
3105
3106     foreach my $vmid (keys %$list) {
3107         next if $opt_vmid && ($vmid ne $opt_vmid);
3108
3109         my $conf = PVE::QemuConfig->load_config($vmid);
3110
3111         my $d = { vmid => $vmid };
3112         $d->{pid} = $list->{$vmid}->{pid};
3113
3114         # fixme: better status?
3115         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
3116
3117         my $size = disksize($storecfg, $conf);
3118         if (defined($size)) {
3119             $d->{disk} = 0; # no info available
3120             $d->{maxdisk} = $size;
3121         } else {
3122             $d->{disk} = 0;
3123             $d->{maxdisk} = 0;
3124         }
3125
3126         $d->{cpus} = ($conf->{sockets} || $defaults->{sockets})
3127             * ($conf->{cores} || $defaults->{cores});
3128         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
3129         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
3130
3131         $d->{name} = $conf->{name} || "VM $vmid";
3132         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024)
3133             : $defaults->{memory}*(1024*1024);
3134
3135         if ($conf->{balloon}) {
3136             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
3137             $d->{shares} = defined($conf->{shares}) ? $conf->{shares}
3138                 : $defaults->{shares};
3139         }
3140
3141         $d->{uptime} = 0;
3142         $d->{cpu} = 0;
3143         $d->{mem} = 0;
3144
3145         $d->{netout} = 0;
3146         $d->{netin} = 0;
3147
3148         $d->{diskread} = 0;
3149         $d->{diskwrite} = 0;
3150
3151         $d->{template} = PVE::QemuConfig->is_template($conf);
3152
3153         $d->{serial} = 1 if conf_has_serial($conf);
3154         $d->{lock} = $conf->{lock} if $conf->{lock};
3155
3156         $res->{$vmid} = $d;
3157     }
3158
3159     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
3160     foreach my $dev (keys %$netdev) {
3161         next if $dev !~ m/^tap([1-9]\d*)i/;
3162         my $vmid = $1;
3163         my $d = $res->{$vmid};
3164         next if !$d;
3165
3166         $d->{netout} += $netdev->{$dev}->{receive};
3167         $d->{netin} += $netdev->{$dev}->{transmit};
3168
3169         if ($full) {
3170             $d->{nics}->{$dev}->{netout} = $netdev->{$dev}->{receive};
3171             $d->{nics}->{$dev}->{netin} = $netdev->{$dev}->{transmit};
3172         }
3173
3174     }
3175
3176     my $ctime = gettimeofday;
3177
3178     foreach my $vmid (keys %$list) {
3179
3180         my $d = $res->{$vmid};
3181         my $pid = $d->{pid};
3182         next if !$pid;
3183
3184         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
3185         next if !$pstat; # not running
3186
3187         my $used = $pstat->{utime} + $pstat->{stime};
3188
3189         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
3190
3191         if ($pstat->{vsize}) {
3192             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
3193         }
3194
3195         my $old = $last_proc_pid_stat->{$pid};
3196         if (!$old) {
3197             $last_proc_pid_stat->{$pid} = {
3198                 time => $ctime,
3199                 used => $used,
3200                 cpu => 0,
3201             };
3202             next;
3203         }
3204
3205         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
3206
3207         if ($dtime > 1000) {
3208             my $dutime = $used -  $old->{used};
3209
3210             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
3211             $last_proc_pid_stat->{$pid} = {
3212                 time => $ctime,
3213                 used => $used,
3214                 cpu => $d->{cpu},
3215             };
3216         } else {
3217             $d->{cpu} = $old->{cpu};
3218         }
3219     }
3220
3221     return $res if !$full;
3222
3223     my $qmpclient = PVE::QMPClient->new();
3224
3225     my $ballooncb = sub {
3226         my ($vmid, $resp) = @_;
3227
3228         my $info = $resp->{'return'};
3229         return if !$info->{max_mem};
3230
3231         my $d = $res->{$vmid};
3232
3233         # use memory assigned to VM
3234         $d->{maxmem} = $info->{max_mem};
3235         $d->{balloon} = $info->{actual};
3236
3237         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
3238             $d->{mem} = $info->{total_mem} - $info->{free_mem};
3239             $d->{freemem} = $info->{free_mem};
3240         }
3241
3242         $d->{ballooninfo} = $info;
3243     };
3244
3245     my $blockstatscb = sub {
3246         my ($vmid, $resp) = @_;
3247         my $data = $resp->{'return'} || [];
3248         my $totalrdbytes = 0;
3249         my $totalwrbytes = 0;
3250
3251         for my $blockstat (@$data) {
3252             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
3253             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
3254
3255             $blockstat->{device} =~ s/drive-//;
3256             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
3257         }
3258         $res->{$vmid}->{diskread} = $totalrdbytes;
3259         $res->{$vmid}->{diskwrite} = $totalwrbytes;
3260     };
3261
3262     my $statuscb = sub {
3263         my ($vmid, $resp) = @_;
3264
3265         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
3266         # this fails if ballon driver is not loaded, so this must be
3267         # the last commnand (following command are aborted if this fails).
3268         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
3269
3270         my $status = 'unknown';
3271         if (!defined($status = $resp->{'return'}->{status})) {
3272             warn "unable to get VM status\n";
3273             return;
3274         }
3275
3276         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
3277     };
3278
3279     foreach my $vmid (keys %$list) {
3280         next if $opt_vmid && ($vmid ne $opt_vmid);
3281         next if !$res->{$vmid}->{pid}; # not running
3282         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
3283     }
3284
3285     $qmpclient->queue_execute(undef, 2);
3286
3287     foreach my $vmid (keys %$list) {
3288         next if $opt_vmid && ($vmid ne $opt_vmid);
3289         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
3290     }
3291
3292     return $res;
3293 }
3294
3295 sub foreach_drive {
3296     my ($conf, $func, @param) = @_;
3297
3298     foreach my $ds (valid_drive_names()) {
3299         next if !defined($conf->{$ds});
3300
3301         my $drive = parse_drive($ds, $conf->{$ds});
3302         next if !$drive;
3303
3304         &$func($ds, $drive, @param);
3305     }
3306 }
3307
3308 sub foreach_volid {
3309     my ($conf, $func, @param) = @_;
3310
3311     my $volhash = {};
3312
3313     my $test_volid = sub {
3314         my ($volid, $is_cdrom, $replicate, $shared, $snapname, $size) = @_;
3315
3316         return if !$volid;
3317
3318         $volhash->{$volid}->{cdrom} //= 1;
3319         $volhash->{$volid}->{cdrom} = 0 if !$is_cdrom;
3320
3321         $volhash->{$volid}->{replicate} //= 0;
3322         $volhash->{$volid}->{replicate} = 1 if $replicate;
3323
3324         $volhash->{$volid}->{shared} //= 0;
3325         $volhash->{$volid}->{shared} = 1 if $shared;
3326
3327         $volhash->{$volid}->{referenced_in_config} //= 0;
3328         $volhash->{$volid}->{referenced_in_config} = 1 if !defined($snapname);
3329
3330         $volhash->{$volid}->{referenced_in_snapshot}->{$snapname} = 1
3331             if defined($snapname);
3332         $volhash->{$volid}->{size} = $size if $size;
3333     };
3334
3335     foreach_drive($conf, sub {
3336         my ($ds, $drive) = @_;
3337         $test_volid->($drive->{file}, drive_is_cdrom($drive), $drive->{replicate} // 1, $drive->{shared}, undef, $drive->{size});
3338     });
3339
3340     foreach my $snapname (keys %{$conf->{snapshots}}) {
3341         my $snap = $conf->{snapshots}->{$snapname};
3342         $test_volid->($snap->{vmstate}, 0, 1, $snapname);
3343         foreach_drive($snap, sub {
3344             my ($ds, $drive) = @_;
3345             $test_volid->($drive->{file}, drive_is_cdrom($drive), $drive->{replicate} // 1, $drive->{shared}, $snapname);
3346         });
3347     }
3348
3349     foreach my $volid (keys %$volhash) {
3350         &$func($volid, $volhash->{$volid}, @param);
3351     }
3352 }
3353
3354 sub conf_has_serial {
3355     my ($conf) = @_;
3356
3357     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3358         if ($conf->{"serial$i"}) {
3359             return 1;
3360         }
3361     }
3362
3363     return 0;
3364 }
3365
3366 sub conf_has_audio {
3367     my ($conf, $id) = @_;
3368
3369     $id //= 0;
3370     my $audio = $conf->{"audio$id"};
3371     return undef if !defined($audio);
3372
3373     my $audioproperties = PVE::JSONSchema::parse_property_string($audio_fmt, $audio);
3374     my $audiodriver = $audioproperties->{driver} // 'spice';
3375
3376     return {
3377         dev => $audioproperties->{device},
3378         dev_id => "audiodev$id",
3379         backend => $audiodriver,
3380         backend_id => "$audiodriver-backend${id}",
3381     };
3382 }
3383
3384 sub vga_conf_has_spice {
3385     my ($vga) = @_;
3386
3387     my $vgaconf = parse_vga($vga);
3388     my $vgatype = $vgaconf->{type};
3389     return 0 if !$vgatype || $vgatype !~ m/^qxl([234])?$/;
3390
3391     return $1 || 1;
3392 }
3393
3394 sub is_native($) {
3395     my ($arch) = @_;
3396     return get_host_arch() eq $arch;
3397 }
3398
3399 my $default_machines = {
3400     x86_64 => 'pc',
3401     aarch64 => 'virt',
3402 };
3403
3404 sub get_basic_machine_info {
3405     my ($conf, $forcemachine) = @_;
3406
3407     my $arch = $conf->{arch} // get_host_arch();
3408     my $machine = $forcemachine || $conf->{machine} || $default_machines->{$arch};
3409     return ($arch, $machine);
3410 }
3411
3412 sub get_ovmf_files($) {
3413     my ($arch) = @_;
3414
3415     my $ovmf = $OVMF->{$arch}
3416         or die "no OVMF images known for architecture '$arch'\n";
3417
3418     return @$ovmf;
3419 }
3420
3421 my $Arch2Qemu = {
3422     aarch64 => '/usr/bin/qemu-system-aarch64',
3423     x86_64 => '/usr/bin/qemu-system-x86_64',
3424 };
3425 sub get_command_for_arch($) {
3426     my ($arch) = @_;
3427     return '/usr/bin/kvm' if is_native($arch);
3428
3429     my $cmd = $Arch2Qemu->{$arch}
3430         or die "don't know how to emulate architecture '$arch'\n";
3431     return $cmd;
3432 }
3433
3434 sub get_cpu_options {
3435     my ($conf, $arch, $kvm, $machine_type, $kvm_off, $kvmver, $winversion, $gpu_passthrough) = @_;
3436
3437     my $cpuFlags = [];
3438     my $ostype = $conf->{ostype};
3439
3440     my $cpu = $kvm ? "kvm64" : "qemu64";
3441     if ($arch eq 'aarch64') {
3442         $cpu = 'cortex-a57';
3443     }
3444     my $hv_vendor_id;
3445     if (my $cputype = $conf->{cpu}) {
3446         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
3447             or die "Cannot parse cpu description: $cputype\n";
3448         $cpu = $cpuconf->{cputype};
3449         $kvm_off = 1 if $cpuconf->{hidden};
3450         $hv_vendor_id = $cpuconf->{'hv-vendor-id'};
3451
3452         if (defined(my $flags = $cpuconf->{flags})) {
3453             push @$cpuFlags, split(";", $flags);
3454         }
3455     }
3456
3457     push @$cpuFlags , '+lahf_lm' if $cpu eq 'kvm64' && $arch eq 'x86_64';
3458
3459     push @$cpuFlags , '-x2apic' if $ostype && $ostype eq 'solaris';
3460
3461     push @$cpuFlags, '+sep' if $cpu eq 'kvm64' || $cpu eq 'kvm32';
3462
3463     push @$cpuFlags, '-rdtscp' if $cpu =~ m/^Opteron/;
3464
3465     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3) && $arch eq 'x86_64') {
3466
3467         push @$cpuFlags , '+kvm_pv_unhalt' if $kvm;
3468         push @$cpuFlags , '+kvm_pv_eoi' if $kvm;
3469     }
3470
3471     add_hyperv_enlightenments($cpuFlags, $winversion, $machine_type, $kvmver, $conf->{bios}, $gpu_passthrough, $hv_vendor_id) if $kvm;
3472
3473     push @$cpuFlags, 'enforce' if $cpu ne 'host' && $kvm && $arch eq 'x86_64';
3474
3475     push @$cpuFlags, 'kvm=off' if $kvm_off;
3476
3477     if (my $cpu_vendor = $cpu_vendor_list->{$cpu}) {
3478         push @$cpuFlags, "vendor=${cpu_vendor}"
3479             if $cpu_vendor ne 'default';
3480     } elsif ($arch ne 'aarch64') {
3481         die "internal error"; # should not happen
3482     }
3483
3484     $cpu .= "," . join(',', @$cpuFlags) if scalar(@$cpuFlags);
3485
3486     return ('-cpu', $cpu);
3487 }
3488
3489 sub config_to_command {
3490     my ($storecfg, $vmid, $conf, $defaults, $forcemachine) = @_;
3491
3492     my $cmd = [];
3493     my $globalFlags = [];
3494     my $machineFlags = [];
3495     my $rtcFlags = [];
3496     my $devices = [];
3497     my $pciaddr = '';
3498     my $bridges = {};
3499     my $vernum = 0; # unknown
3500     my $ostype = $conf->{ostype};
3501     my $winversion = windows_version($ostype);
3502     my $kvm = $conf->{kvm};
3503
3504     my ($arch, $machine_type) = get_basic_machine_info($conf, $forcemachine);
3505     my $kvm_binary = get_command_for_arch($arch);
3506     my $kvmver = kvm_user_version($kvm_binary);
3507     $kvm //= 1 if is_native($arch);
3508
3509     if ($kvm) {
3510         die "KVM virtualisation configured, but not available. Either disable in VM configuration or enable in BIOS.\n"
3511             if !defined kvm_version();
3512     }
3513
3514     if ($kvmver =~ m/^(\d+)\.(\d+)$/) {
3515         $vernum = $1*1000000+$2*1000;
3516     } elsif ($kvmver =~ m/^(\d+)\.(\d+)\.(\d+)$/) {
3517         $vernum = $1*1000000+$2*1000+$3;
3518     }
3519
3520     die "detected old qemu-kvm binary ($kvmver)\n" if $vernum < 15000;
3521
3522     my $q35 = machine_type_is_q35($conf);
3523     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
3524     my $use_old_bios_files = undef;
3525     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3526
3527     my $cpuunits = defined($conf->{cpuunits}) ?
3528             $conf->{cpuunits} : $defaults->{cpuunits};
3529
3530     push @$cmd, $kvm_binary;
3531
3532     push @$cmd, '-id', $vmid;
3533
3534     my $vmname = $conf->{name} || "vm$vmid";
3535
3536     push @$cmd, '-name', $vmname;
3537
3538     my $use_virtio = 0;
3539
3540     my $qmpsocket = PVE::QemuServer::Helpers::qmp_socket($vmid);
3541     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server,nowait";
3542     push @$cmd, '-mon', "chardev=qmp,mode=control";
3543
3544     if (qemu_machine_feature_enabled($machine_type, $kvmver, 2, 12)) {
3545         push @$cmd, '-chardev', "socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5";
3546         push @$cmd, '-mon', "chardev=qmp-event,mode=control";
3547     }
3548
3549     push @$cmd, '-pidfile' , PVE::QemuServer::Helpers::pidfile_name($vmid);
3550
3551     push @$cmd, '-daemonize';
3552
3553     if ($conf->{smbios1}) {
3554         my $smbios_conf = parse_smbios1($conf->{smbios1});
3555         if ($smbios_conf->{base64}) {
3556             # Do not pass base64 flag to qemu
3557             delete $smbios_conf->{base64};
3558             my $smbios_string = "";
3559             foreach my $key (keys %$smbios_conf) {
3560                 my $value;
3561                 if ($key eq "uuid") {
3562                     $value = $smbios_conf->{uuid}
3563                 } else {
3564                     $value = decode_base64($smbios_conf->{$key});
3565                 }
3566                 # qemu accepts any binary data, only commas need escaping by double comma
3567                 $value =~ s/,/,,/g;
3568                 $smbios_string .= "," . $key . "=" . $value if $value;
3569             }
3570             push @$cmd, '-smbios', "type=1" . $smbios_string;
3571         } else {
3572             push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
3573         }
3574     }
3575
3576     if ($conf->{vmgenid}) {
3577         push @$devices, '-device', 'vmgenid,guid='.$conf->{vmgenid};
3578     }
3579
3580     my ($ovmf_code, $ovmf_vars) = get_ovmf_files($arch);
3581     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3582         die "uefi base image not found\n" if ! -f $ovmf_code;
3583
3584         my $path;
3585         my $format;
3586         if (my $efidisk = $conf->{efidisk0}) {
3587             my $d = PVE::JSONSchema::parse_property_string($efidisk_fmt, $efidisk);
3588             my ($storeid, $volname) = PVE::Storage::parse_volume_id($d->{file}, 1);
3589             $format = $d->{format};
3590             if ($storeid) {
3591                 $path = PVE::Storage::path($storecfg, $d->{file});
3592                 if (!defined($format)) {
3593                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
3594                     $format = qemu_img_format($scfg, $volname);
3595                 }
3596             } else {
3597                 $path = $d->{file};
3598                 die "efidisk format must be specified\n"
3599                     if !defined($format);
3600             }
3601         } else {
3602             warn "no efidisk configured! Using temporary efivars disk.\n";
3603             $path = "/tmp/$vmid-ovmf.fd";
3604             PVE::Tools::file_copy($ovmf_vars, $path, -s $ovmf_vars);
3605             $format = 'raw';
3606         }
3607
3608         push @$cmd, '-drive', "if=pflash,unit=0,format=raw,readonly,file=$ovmf_code";
3609         push @$cmd, '-drive', "if=pflash,unit=1,format=$format,id=drive-efidisk0,file=$path";
3610     }
3611
3612     # load q35 config
3613     if ($q35) {
3614         # we use different pcie-port hardware for qemu >= 4.0 for passthrough
3615         if (qemu_machine_feature_enabled($machine_type, $kvmver, 4, 0)) {
3616             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35-4.0.cfg';
3617         } else {
3618             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35.cfg';
3619         }
3620     }
3621
3622     # add usb controllers
3623     my @usbcontrollers = PVE::QemuServer::USB::get_usb_controllers($conf, $bridges, $arch, $machine_type, $usbdesc->{format}, $MAX_USB_DEVICES);
3624     push @$devices, @usbcontrollers if @usbcontrollers;
3625     my $vga = parse_vga($conf->{vga});
3626
3627     my $qxlnum = vga_conf_has_spice($conf->{vga});
3628     $vga->{type} = 'qxl' if $qxlnum;
3629
3630     if (!$vga->{type}) {
3631         if ($arch eq 'aarch64') {
3632             $vga->{type} = 'virtio';
3633         } elsif (qemu_machine_feature_enabled($machine_type, $kvmver, 2, 9)) {
3634             $vga->{type} = (!$winversion || $winversion >= 6) ? 'std' : 'cirrus';
3635         } else {
3636             $vga->{type} = ($winversion >= 6) ? 'std' : 'cirrus';
3637         }
3638     }
3639
3640     # enable absolute mouse coordinates (needed by vnc)
3641     my $tablet;
3642     if (defined($conf->{tablet})) {
3643         $tablet = $conf->{tablet};
3644     } else {
3645         $tablet = $defaults->{tablet};
3646         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
3647         $tablet = 0 if $vga->{type} =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
3648     }
3649
3650     if ($tablet) {
3651         push @$devices, '-device', print_tabletdevice_full($conf, $arch) if $tablet;
3652         my $kbd = print_keyboarddevice_full($conf, $arch);
3653         push @$devices, '-device', $kbd if defined($kbd);
3654     }
3655
3656     my $kvm_off = 0;
3657     my $gpu_passthrough;
3658
3659     # host pci devices
3660     for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
3661         my $id = "hostpci$i";
3662         my $d = parse_hostpci($conf->{$id});
3663         next if !$d;
3664
3665         if (my $pcie = $d->{pcie}) {
3666             die "q35 machine model is not enabled" if !$q35;
3667             # win7 wants to have the pcie devices directly on the pcie bus
3668             # instead of in the root port
3669             if ($winversion == 7) {
3670                 $pciaddr = print_pcie_addr("${id}bus0");
3671             } else {
3672                 # add more root ports if needed, 4 are present by default
3673                 # by pve-q35 cfgs, rest added here on demand.
3674                 if ($i > 3) {
3675                     push @$devices, '-device', print_pcie_root_port($i);
3676                 }
3677                 $pciaddr = print_pcie_addr($id);
3678             }
3679         } else {
3680             $pciaddr = print_pci_addr($id, $bridges, $arch, $machine_type);
3681         }
3682
3683         my $xvga = '';
3684         if ($d->{'x-vga'}) {
3685             $xvga = ',x-vga=on' if !($conf->{bios} && $conf->{bios} eq 'ovmf');
3686             $kvm_off = 1;
3687             $vga->{type} = 'none' if !defined($conf->{vga});
3688             $gpu_passthrough = 1;
3689         }
3690
3691         my $pcidevices = $d->{pciid};
3692         my $multifunction = 1 if @$pcidevices > 1;
3693
3694         my $sysfspath;
3695         if ($d->{mdev} && scalar(@$pcidevices) == 1) {
3696             my $pci_id = $pcidevices->[0]->{id};
3697             my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $i);
3698             $sysfspath = "/sys/bus/pci/devices/0000:$pci_id/$uuid";
3699         } elsif ($d->{mdev}) {
3700             warn "ignoring mediated device '$id' with multifunction device\n";
3701         }
3702
3703         my $j=0;
3704         foreach my $pcidevice (@$pcidevices) {
3705             my $devicestr = "vfio-pci";
3706
3707             if ($sysfspath) {
3708                 $devicestr .= ",sysfsdev=$sysfspath";
3709             } else {
3710                 $devicestr .= ",host=$pcidevice->{id}";
3711             }
3712
3713             my $mf_addr = $multifunction ? ".$j" : '';
3714             $devicestr .= ",id=${id}${mf_addr}${pciaddr}${mf_addr}";
3715
3716             if ($j == 0) {
3717                 $devicestr .= ',rombar=0' if defined($d->{rombar}) && !$d->{rombar};
3718                 $devicestr .= "$xvga";
3719                 $devicestr .= ",multifunction=on" if $multifunction;
3720                 $devicestr .= ",romfile=/usr/share/kvm/$d->{romfile}" if $d->{romfile};
3721             }
3722
3723             push @$devices, '-device', $devicestr;
3724             $j++;
3725         }
3726     }
3727
3728     # usb devices
3729     my $usb_dev_features = {};
3730     $usb_dev_features->{spice_usb3} = 1 if qemu_machine_feature_enabled($machine_type, $kvmver, 4, 0);
3731
3732     my @usbdevices = PVE::QemuServer::USB::get_usb_devices($conf, $usbdesc->{format}, $MAX_USB_DEVICES, $usb_dev_features);
3733     push @$devices, @usbdevices if @usbdevices;
3734     # serial devices
3735     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3736         if (my $path = $conf->{"serial$i"}) {
3737             if ($path eq 'socket') {
3738                 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
3739                 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server,nowait";
3740                 # On aarch64, serial0 is the UART device. Qemu only allows
3741                 # connecting UART devices via the '-serial' command line, as
3742                 # the device has a fixed slot on the hardware...
3743                 if ($arch eq 'aarch64' && $i == 0) {
3744                     push @$devices, '-serial', "chardev:serial$i";
3745                 } else {
3746                     push @$devices, '-device', "isa-serial,chardev=serial$i";
3747                 }
3748             } else {
3749                 die "no such serial device\n" if ! -c $path;
3750                 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
3751                 push @$devices, '-device', "isa-serial,chardev=serial$i";
3752             }
3753         }
3754     }
3755
3756     # parallel devices
3757     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
3758         if (my $path = $conf->{"parallel$i"}) {
3759             die "no such parallel device\n" if ! -c $path;
3760             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
3761             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
3762             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
3763         }
3764     }
3765
3766     if (my $audio = conf_has_audio($conf)) {
3767
3768         my $audiopciaddr = print_pci_addr("audio0", $bridges, $arch, $machine_type);
3769
3770         my $id = $audio->{dev_id};
3771         if ($audio->{dev} eq 'AC97') {
3772             push @$devices, '-device', "AC97,id=${id}${audiopciaddr}";
3773         } elsif ($audio->{dev} =~ /intel\-hda$/) {
3774             push @$devices, '-device', "$audio->{dev},id=${id}${audiopciaddr}";
3775             push @$devices, '-device', "hda-micro,id=${id}-codec0,bus=${id}.0,cad=0";
3776             push @$devices, '-device', "hda-duplex,id=${id}-codec1,bus=${id}.0,cad=1";
3777         } else {
3778             die "unkown audio device '$audio->{dev}', implement me!";
3779         }
3780
3781         push @$devices, '-audiodev', "$audio->{backend},id=$audio->{backend_id}";
3782     }
3783
3784     my $sockets = 1;
3785     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3786     $sockets = $conf->{sockets} if  $conf->{sockets};
3787
3788     my $cores = $conf->{cores} || 1;
3789
3790     my $maxcpus = $sockets * $cores;
3791
3792     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
3793
3794     my $allowed_vcpus = $cpuinfo->{cpus};
3795
3796     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n"
3797         if ($allowed_vcpus < $maxcpus);
3798
3799     if($hotplug_features->{cpu} && qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 7)) {
3800
3801         push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3802         for (my $i = 2; $i <= $vcpus; $i++)  {
3803             my $cpustr = print_cpu_device($conf,$i);
3804             push @$cmd, '-device', $cpustr;
3805         }
3806
3807     } else {
3808
3809         push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3810     }
3811     push @$cmd, '-nodefaults';
3812
3813     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
3814
3815     my $bootindex_hash = {};
3816     my $i = 1;
3817     foreach my $o (split(//, $bootorder)) {
3818         $bootindex_hash->{$o} = $i*100;
3819         $i++;
3820     }
3821
3822     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
3823
3824     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
3825
3826     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
3827
3828     if ($vga->{type} && $vga->{type} !~ m/^serial\d+$/ && $vga->{type} ne 'none'){
3829         push @$devices, '-device', print_vga_device($conf, $vga, $arch, $kvmver, $machine_type, undef, $qxlnum, $bridges);
3830         my $socket = PVE::QemuServer::Helpers::vnc_socket($vmid);
3831         push @$cmd,  '-vnc', "unix:$socket,password";
3832     } else {
3833         push @$cmd, '-vga', 'none' if $vga->{type} eq 'none';
3834         push @$cmd, '-nographic';
3835     }
3836
3837     # time drift fix
3838     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
3839
3840     my $useLocaltime = $conf->{localtime};
3841
3842     if ($winversion >= 5) { # windows
3843         $useLocaltime = 1 if !defined($conf->{localtime});
3844
3845         # use time drift fix when acpi is enabled
3846         if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
3847             $tdf = 1 if !defined($conf->{tdf});
3848         }
3849     }
3850
3851     if ($winversion >= 6) {
3852         push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3853         push @$cmd, '-no-hpet';
3854     }
3855
3856     push @$rtcFlags, 'driftfix=slew' if $tdf;
3857
3858     if (!$kvm) {
3859         push @$machineFlags, 'accel=tcg';
3860     }
3861
3862     if ($machine_type) {
3863         push @$machineFlags, "type=${machine_type}";
3864     }
3865
3866     if (($conf->{startdate}) && ($conf->{startdate} ne 'now')) {
3867         push @$rtcFlags, "base=$conf->{startdate}";
3868     } elsif ($useLocaltime) {
3869         push @$rtcFlags, 'base=localtime';
3870     }
3871
3872     push @$cmd, get_cpu_options($conf, $arch, $kvm, $machine_type, $kvm_off, $kvmver, $winversion, $gpu_passthrough);
3873
3874     PVE::QemuServer::Memory::config($conf, $vmid, $sockets, $cores, $defaults, $hotplug_features, $cmd);
3875
3876     push @$cmd, '-S' if $conf->{freeze};
3877
3878     push @$cmd, '-k', $conf->{keyboard} if defined($conf->{keyboard});
3879
3880     my $guest_agent = parse_guest_agent($conf);
3881
3882     if ($guest_agent->{enabled}) {
3883         my $qgasocket = PVE::QemuServer::Helpers::qmp_socket($vmid, 1);
3884         push @$devices, '-chardev', "socket,path=$qgasocket,server,nowait,id=qga0";
3885
3886         if (!$guest_agent->{type} || $guest_agent->{type} eq 'virtio') {
3887             my $pciaddr = print_pci_addr("qga0", $bridges, $arch, $machine_type);
3888             push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3889             push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3890         } elsif ($guest_agent->{type} eq 'isa') {
3891             push @$devices, '-device', "isa-serial,chardev=qga0";
3892         }
3893     }
3894
3895     my $spice_port;
3896
3897     if ($qxlnum) {
3898         if ($qxlnum > 1) {
3899             if ($winversion){
3900                 for(my $i = 1; $i < $qxlnum; $i++){
3901                     push @$devices, '-device', print_vga_device($conf, $vga, $arch, $kvmver, $machine_type, $i, $qxlnum, $bridges);
3902                 }
3903             } else {
3904                 # assume other OS works like Linux
3905                 my ($ram, $vram) = ("134217728", "67108864");
3906                 if ($vga->{memory}) {
3907                     $ram = PVE::Tools::convert_size($qxlnum*4*$vga->{memory}, 'mb' => 'b');
3908                     $vram = PVE::Tools::convert_size($qxlnum*2*$vga->{memory}, 'mb' => 'b');
3909                 }
3910                 push @$cmd, '-global', "qxl-vga.ram_size=$ram";
3911                 push @$cmd, '-global', "qxl-vga.vram_size=$vram";
3912             }
3913         }
3914
3915         my $pciaddr = print_pci_addr("spice", $bridges, $arch, $machine_type);
3916
3917         my $nodename = PVE::INotify::nodename();
3918         my $pfamily = PVE::Tools::get_host_address_family($nodename);
3919         my @nodeaddrs = PVE::Tools::getaddrinfo_all('localhost', family => $pfamily);
3920         die "failed to get an ip address of type $pfamily for 'localhost'\n" if !@nodeaddrs;
3921
3922         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3923         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3924         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3925
3926         my $localhost = PVE::Network::addr_to_ip($nodeaddrs[0]->{addr});
3927         $spice_port = PVE::Tools::next_spice_port($pfamily, $localhost);
3928
3929         my $spice_enhancement = PVE::JSONSchema::parse_property_string($spice_enhancements_fmt, $conf->{spice_enhancements} // '');
3930         if ($spice_enhancement->{foldersharing}) {
3931             push @$devices, '-chardev', "spiceport,id=foldershare,name=org.spice-space.webdav.0";
3932             push @$devices, '-device', "virtserialport,chardev=foldershare,name=org.spice-space.webdav.0";
3933         }
3934
3935         my $spice_opts = "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";
3936         $spice_opts .= ",streaming-video=$spice_enhancement->{videostreaming}" if $spice_enhancement->{videostreaming};
3937         push @$devices, '-spice', "$spice_opts";
3938     }
3939
3940     # enable balloon by default, unless explicitly disabled
3941     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3942         $pciaddr = print_pci_addr("balloon0", $bridges, $arch, $machine_type);
3943         push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
3944     }
3945
3946     if ($conf->{watchdog}) {
3947         my $wdopts = parse_watchdog($conf->{watchdog});
3948         $pciaddr = print_pci_addr("watchdog", $bridges, $arch, $machine_type);
3949         my $watchdog = $wdopts->{model} || 'i6300esb';
3950         push @$devices, '-device', "$watchdog$pciaddr";
3951         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3952     }
3953
3954     my $vollist = [];
3955     my $scsicontroller = {};
3956     my $ahcicontroller = {};
3957     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3958
3959     # Add iscsi initiator name if available
3960     if (my $initiator = get_initiator_name()) {
3961         push @$devices, '-iscsi', "initiator-name=$initiator";
3962     }
3963
3964     foreach_drive($conf, sub {
3965         my ($ds, $drive) = @_;
3966
3967         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3968             push @$vollist, $drive->{file};
3969         }
3970
3971         # ignore efidisk here, already added in bios/fw handling code above
3972         return if $drive->{interface} eq 'efidisk';
3973
3974         $use_virtio = 1 if $ds =~ m/^virtio/;
3975
3976         if (drive_is_cdrom ($drive)) {
3977             if ($bootindex_hash->{d}) {
3978                 $drive->{bootindex} = $bootindex_hash->{d};
3979                 $bootindex_hash->{d} += 1;
3980             }
3981         } else {
3982             if ($bootindex_hash->{c}) {
3983                 $drive->{bootindex} = $bootindex_hash->{c} if $conf->{bootdisk} && ($conf->{bootdisk} eq $ds);
3984                 $bootindex_hash->{c} += 1;
3985             }
3986         }
3987
3988         if($drive->{interface} eq 'virtio'){
3989            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3990         }
3991
3992         if ($drive->{interface} eq 'scsi') {
3993
3994             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
3995
3996             $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges, $arch, $machine_type);
3997             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
3998
3999             my $iothread = '';
4000             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
4001                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
4002                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
4003             } elsif ($drive->{iothread}) {
4004                 warn "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n";
4005             }
4006
4007             my $queues = '';
4008             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
4009                 $queues = ",num_queues=$drive->{queues}";
4010             }
4011
4012             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues" if !$scsicontroller->{$controller};
4013             $scsicontroller->{$controller}=1;
4014         }
4015
4016         if ($drive->{interface} eq 'sata') {
4017            my $controller = int($drive->{index} / $MAX_SATA_DISKS);
4018            $pciaddr = print_pci_addr("ahci$controller", $bridges, $arch, $machine_type);
4019            push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr" if !$ahcicontroller->{$controller};
4020            $ahcicontroller->{$controller}=1;
4021         }
4022
4023         my $drive_cmd = print_drive_full($storecfg, $vmid, $drive);
4024         push @$devices, '-drive',$drive_cmd;
4025         push @$devices, '-device', print_drivedevice_full($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type);
4026     });
4027
4028     for (my $i = 0; $i < $MAX_NETS; $i++) {
4029          next if !$conf->{"net$i"};
4030          my $d = parse_net($conf->{"net$i"});
4031          next if !$d;
4032
4033          $use_virtio = 1 if $d->{model} eq 'virtio';
4034
4035          if ($bootindex_hash->{n}) {
4036             $d->{bootindex} = $bootindex_hash->{n};
4037             $bootindex_hash->{n} += 1;
4038          }
4039
4040          my $netdevfull = print_netdev_full($vmid, $conf, $arch, $d, "net$i");
4041          push @$devices, '-netdev', $netdevfull;
4042
4043          my $netdevicefull = print_netdevice_full($vmid, $conf, $d, "net$i", $bridges, $use_old_bios_files, $arch, $machine_type);
4044          push @$devices, '-device', $netdevicefull;
4045     }
4046
4047     if ($conf->{ivshmem}) {
4048         my $ivshmem = PVE::JSONSchema::parse_property_string($ivshmem_fmt, $conf->{ivshmem});
4049
4050         my $bus;
4051         if ($q35) {
4052             $bus = print_pcie_addr("ivshmem");
4053         } else {
4054             $bus = print_pci_addr("ivshmem", $bridges, $arch, $machine_type);
4055         }
4056
4057         my $ivshmem_name = $ivshmem->{name} // $vmid;
4058         my $path = '/dev/shm/pve-shm-' . $ivshmem_name;
4059
4060         push @$devices, '-device', "ivshmem-plain,memdev=ivshmem$bus,";
4061         push @$devices, '-object', "memory-backend-file,id=ivshmem,share=on,mem-path=$path,size=$ivshmem->{size}M";
4062     }
4063
4064     if (!$q35) {
4065         # add pci bridges
4066         if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
4067            $bridges->{1} = 1;
4068            $bridges->{2} = 1;
4069         }
4070
4071         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
4072
4073         for my $k (sort {$b cmp $a} keys %$bridges) {
4074             $pciaddr = print_pci_addr("pci.$k", undef, $arch, $machine_type);
4075             unshift @$devices, '-device', "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr" if $k > 0;
4076         }
4077     }
4078
4079     push @$cmd, @$devices;
4080     push @$cmd, '-rtc', join(',', @$rtcFlags)
4081         if scalar(@$rtcFlags);
4082     push @$cmd, '-machine', join(',', @$machineFlags)
4083         if scalar(@$machineFlags);
4084     push @$cmd, '-global', join(',', @$globalFlags)
4085         if scalar(@$globalFlags);
4086
4087     if (my $vmstate = $conf->{vmstate}) {
4088         my $statepath = PVE::Storage::path($storecfg, $vmstate);
4089         push @$vollist, $vmstate;
4090         push @$cmd, '-loadstate', $statepath;
4091     }
4092
4093     # add custom args
4094     if ($conf->{args}) {
4095         my $aa = PVE::Tools::split_args($conf->{args});
4096         push @$cmd, @$aa;
4097     }
4098
4099     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
4100 }
4101
4102 sub spice_port {
4103     my ($vmid) = @_;
4104
4105     my $res = vm_mon_cmd($vmid, 'query-spice');
4106
4107     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
4108 }
4109
4110 sub vm_devices_list {
4111     my ($vmid) = @_;
4112
4113     my $res = vm_mon_cmd($vmid, 'query-pci');
4114     my $devices_to_check = [];
4115     my $devices = {};
4116     foreach my $pcibus (@$res) {
4117         push @$devices_to_check, @{$pcibus->{devices}},
4118     }
4119
4120     while (@$devices_to_check) {
4121         my $to_check = [];
4122         for my $d (@$devices_to_check) {
4123             $devices->{$d->{'qdev_id'}} = 1 if $d->{'qdev_id'};
4124             next if !$d->{'pci_bridge'};
4125
4126             $devices->{$d->{'qdev_id'}} += scalar(@{$d->{'pci_bridge'}->{devices}});
4127             push @$to_check, @{$d->{'pci_bridge'}->{devices}};
4128         }
4129         $devices_to_check = $to_check;
4130     }
4131
4132     my $resblock = vm_mon_cmd($vmid, 'query-block');
4133     foreach my $block (@$resblock) {
4134         if($block->{device} =~ m/^drive-(\S+)/){
4135                 $devices->{$1} = 1;
4136         }
4137     }
4138
4139     my $resmice = vm_mon_cmd($vmid, 'query-mice');
4140     foreach my $mice (@$resmice) {
4141         if ($mice->{name} eq 'QEMU HID Tablet') {
4142             $devices->{tablet} = 1;
4143             last;
4144         }
4145     }
4146
4147     # for usb devices there is no query-usb
4148     # but we can iterate over the entries in
4149     # qom-list path=/machine/peripheral
4150     my $resperipheral = vm_mon_cmd($vmid, 'qom-list', path => '/machine/peripheral');
4151     foreach my $per (@$resperipheral) {
4152         if ($per->{name} =~ m/^usb\d+$/) {
4153             $devices->{$per->{name}} = 1;
4154         }
4155     }
4156
4157     return $devices;
4158 }
4159
4160 sub vm_deviceplug {
4161     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4162
4163     my $q35 = machine_type_is_q35($conf);
4164
4165     my $devices_list = vm_devices_list($vmid);
4166     return 1 if defined($devices_list->{$deviceid});
4167
4168     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid, $arch, $machine_type); # add PCI bridge if we need it for the device
4169
4170     if ($deviceid eq 'tablet') {
4171
4172         qemu_deviceadd($vmid, print_tabletdevice_full($conf, $arch));
4173
4174     } elsif ($deviceid eq 'keyboard') {
4175
4176         qemu_deviceadd($vmid, print_keyboarddevice_full($conf, $arch));
4177
4178     } elsif ($deviceid =~ m/^usb(\d+)$/) {
4179
4180         die "usb hotplug currently not reliable\n";
4181         # since we can't reliably hot unplug all added usb devices
4182         # and usb passthrough disables live migration
4183         # we disable usb hotplugging for now
4184         qemu_deviceadd($vmid, PVE::QemuServer::USB::print_usbdevice_full($conf, $deviceid, $device));
4185
4186     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4187
4188         qemu_iothread_add($vmid, $deviceid, $device);
4189
4190         qemu_driveadd($storecfg, $vmid, $device);
4191         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, $arch, $machine_type);
4192
4193         qemu_deviceadd($vmid, $devicefull);
4194         eval { qemu_deviceaddverify($vmid, $deviceid); };
4195         if (my $err = $@) {
4196             eval { qemu_drivedel($vmid, $deviceid); };
4197             warn $@ if $@;
4198             die $err;
4199         }
4200
4201     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4202
4203
4204         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
4205         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
4206         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
4207
4208         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
4209
4210         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
4211             qemu_iothread_add($vmid, $deviceid, $device);
4212             $devicefull .= ",iothread=iothread-$deviceid";
4213         }
4214
4215         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
4216             $devicefull .= ",num_queues=$device->{queues}";
4217         }
4218
4219         qemu_deviceadd($vmid, $devicefull);
4220         qemu_deviceaddverify($vmid, $deviceid);
4221
4222     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4223
4224         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device, $arch, $machine_type);
4225         qemu_driveadd($storecfg, $vmid, $device);
4226
4227         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, $arch, $machine_type);
4228         eval { qemu_deviceadd($vmid, $devicefull); };
4229         if (my $err = $@) {
4230             eval { qemu_drivedel($vmid, $deviceid); };
4231             warn $@ if $@;
4232             die $err;
4233         }
4234
4235     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4236
4237         return undef if !qemu_netdevadd($vmid, $conf, $arch, $device, $deviceid);
4238
4239         my $machine_type = PVE::QemuServer::qemu_machine_pxe($vmid, $conf);
4240         my $use_old_bios_files = undef;
4241         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
4242
4243         my $netdevicefull = print_netdevice_full($vmid, $conf, $device, $deviceid, undef, $use_old_bios_files, $arch, $machine_type);
4244         qemu_deviceadd($vmid, $netdevicefull);
4245         eval {
4246             qemu_deviceaddverify($vmid, $deviceid);
4247             qemu_set_link_status($vmid, $deviceid, !$device->{link_down});
4248         };
4249         if (my $err = $@) {
4250             eval { qemu_netdevdel($vmid, $deviceid); };
4251             warn $@ if $@;
4252             die $err;
4253         }
4254
4255     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
4256
4257         my $bridgeid = $2;
4258         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
4259         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
4260
4261         qemu_deviceadd($vmid, $devicefull);
4262         qemu_deviceaddverify($vmid, $deviceid);
4263
4264     } else {
4265         die "can't hotplug device '$deviceid'\n";
4266     }
4267
4268     return 1;
4269 }
4270
4271 # fixme: this should raise exceptions on error!
4272 sub vm_deviceunplug {
4273     my ($vmid, $conf, $deviceid) = @_;
4274
4275     my $devices_list = vm_devices_list($vmid);
4276     return 1 if !defined($devices_list->{$deviceid});
4277
4278     die "can't unplug bootdisk" if $conf->{bootdisk} && $conf->{bootdisk} eq $deviceid;
4279
4280     if ($deviceid eq 'tablet' || $deviceid eq 'keyboard') {
4281
4282         qemu_devicedel($vmid, $deviceid);
4283
4284     } elsif ($deviceid =~ m/^usb\d+$/) {
4285
4286         die "usb hotplug currently not reliable\n";
4287         # when unplugging usb devices this way,
4288         # there may be remaining usb controllers/hubs
4289         # so we disable it for now
4290         qemu_devicedel($vmid, $deviceid);
4291         qemu_devicedelverify($vmid, $deviceid);
4292
4293     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4294
4295         qemu_devicedel($vmid, $deviceid);
4296         qemu_devicedelverify($vmid, $deviceid);
4297         qemu_drivedel($vmid, $deviceid);
4298         qemu_iothread_del($conf, $vmid, $deviceid);
4299
4300     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4301
4302         qemu_devicedel($vmid, $deviceid);
4303         qemu_devicedelverify($vmid, $deviceid);
4304         qemu_iothread_del($conf, $vmid, $deviceid);
4305
4306     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4307
4308         qemu_devicedel($vmid, $deviceid);
4309         qemu_drivedel($vmid, $deviceid);
4310         qemu_deletescsihw($conf, $vmid, $deviceid);
4311
4312     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4313
4314         qemu_devicedel($vmid, $deviceid);
4315         qemu_devicedelverify($vmid, $deviceid);
4316         qemu_netdevdel($vmid, $deviceid);
4317
4318     } else {
4319         die "can't unplug device '$deviceid'\n";
4320     }
4321
4322     return 1;
4323 }
4324
4325 sub qemu_deviceadd {
4326     my ($vmid, $devicefull) = @_;
4327
4328     $devicefull = "driver=".$devicefull;
4329     my %options =  split(/[=,]/, $devicefull);
4330
4331     vm_mon_cmd($vmid, "device_add" , %options);
4332 }
4333
4334 sub qemu_devicedel {
4335     my ($vmid, $deviceid) = @_;
4336
4337     my $ret = vm_mon_cmd($vmid, "device_del", id => $deviceid);
4338 }
4339
4340 sub qemu_iothread_add {
4341     my($vmid, $deviceid, $device) = @_;
4342
4343     if ($device->{iothread}) {
4344         my $iothreads = vm_iothreads_list($vmid);
4345         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
4346     }
4347 }
4348
4349 sub qemu_iothread_del {
4350     my($conf, $vmid, $deviceid) = @_;
4351
4352     my $confid = $deviceid;
4353     if ($deviceid =~ m/^(?:virtioscsi|scsihw)(\d+)$/) {
4354         $confid = 'scsi' . $1;
4355     }
4356     my $device = parse_drive($confid, $conf->{$confid});
4357     if ($device->{iothread}) {
4358         my $iothreads = vm_iothreads_list($vmid);
4359         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
4360     }
4361 }
4362
4363 sub qemu_objectadd {
4364     my($vmid, $objectid, $qomtype) = @_;
4365
4366     vm_mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
4367
4368     return 1;
4369 }
4370
4371 sub qemu_objectdel {
4372     my($vmid, $objectid) = @_;
4373
4374     vm_mon_cmd($vmid, "object-del", id => $objectid);
4375
4376     return 1;
4377 }
4378
4379 sub qemu_driveadd {
4380     my ($storecfg, $vmid, $device) = @_;
4381
4382     my $drive = print_drive_full($storecfg, $vmid, $device);
4383     $drive =~ s/\\/\\\\/g;
4384     my $ret = vm_human_monitor_command($vmid, "drive_add auto \"$drive\"");
4385
4386     # If the command succeeds qemu prints: "OK"
4387     return 1 if $ret =~ m/OK/s;
4388
4389     die "adding drive failed: $ret\n";
4390 }
4391
4392 sub qemu_drivedel {
4393     my($vmid, $deviceid) = @_;
4394
4395     my $ret = vm_human_monitor_command($vmid, "drive_del drive-$deviceid");
4396     $ret =~ s/^\s+//;
4397
4398     return 1 if $ret eq "";
4399
4400     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
4401     return 1 if $ret =~ m/Device \'.*?\' not found/s;
4402
4403     die "deleting drive $deviceid failed : $ret\n";
4404 }
4405
4406 sub qemu_deviceaddverify {
4407     my ($vmid, $deviceid) = @_;
4408
4409     for (my $i = 0; $i <= 5; $i++) {
4410          my $devices_list = vm_devices_list($vmid);
4411          return 1 if defined($devices_list->{$deviceid});
4412          sleep 1;
4413     }
4414
4415     die "error on hotplug device '$deviceid'\n";
4416 }
4417
4418
4419 sub qemu_devicedelverify {
4420     my ($vmid, $deviceid) = @_;
4421
4422     # need to verify that the device is correctly removed as device_del
4423     # is async and empty return is not reliable
4424
4425     for (my $i = 0; $i <= 5; $i++) {
4426          my $devices_list = vm_devices_list($vmid);
4427          return 1 if !defined($devices_list->{$deviceid});
4428          sleep 1;
4429     }
4430
4431     die "error on hot-unplugging device '$deviceid'\n";
4432 }
4433
4434 sub qemu_findorcreatescsihw {
4435     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4436
4437     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4438
4439     my $scsihwid="$controller_prefix$controller";
4440     my $devices_list = vm_devices_list($vmid);
4441
4442     if(!defined($devices_list->{$scsihwid})) {
4443         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device, $arch, $machine_type);
4444     }
4445
4446     return 1;
4447 }
4448
4449 sub qemu_deletescsihw {
4450     my ($conf, $vmid, $opt) = @_;
4451
4452     my $device = parse_drive($opt, $conf->{$opt});
4453
4454     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
4455         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
4456         return 1;
4457     }
4458
4459     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4460
4461     my $devices_list = vm_devices_list($vmid);
4462     foreach my $opt (keys %{$devices_list}) {
4463         if (PVE::QemuServer::is_valid_drivename($opt)) {
4464             my $drive = PVE::QemuServer::parse_drive($opt, $conf->{$opt});
4465             if($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
4466                 return 1;
4467             }
4468         }
4469     }
4470
4471     my $scsihwid="scsihw$controller";
4472
4473     vm_deviceunplug($vmid, $conf, $scsihwid);
4474
4475     return 1;
4476 }
4477
4478 sub qemu_add_pci_bridge {
4479     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4480
4481     my $bridges = {};
4482
4483     my $bridgeid;
4484
4485     print_pci_addr($device, $bridges, $arch, $machine_type);
4486
4487     while (my ($k, $v) = each %$bridges) {
4488         $bridgeid = $k;
4489     }
4490     return 1 if !defined($bridgeid) || $bridgeid < 1;
4491
4492     my $bridge = "pci.$bridgeid";
4493     my $devices_list = vm_devices_list($vmid);
4494
4495     if (!defined($devices_list->{$bridge})) {
4496         vm_deviceplug($storecfg, $conf, $vmid, $bridge, $arch, $machine_type);
4497     }
4498
4499     return 1;
4500 }
4501
4502 sub qemu_set_link_status {
4503     my ($vmid, $device, $up) = @_;
4504
4505     vm_mon_cmd($vmid, "set_link", name => $device,
4506                up => $up ? JSON::true : JSON::false);
4507 }
4508
4509 sub qemu_netdevadd {
4510     my ($vmid, $conf, $arch, $device, $deviceid) = @_;
4511
4512     my $netdev = print_netdev_full($vmid, $conf, $arch, $device, $deviceid, 1);
4513     my %options =  split(/[=,]/, $netdev);
4514
4515     vm_mon_cmd($vmid, "netdev_add",  %options);
4516     return 1;
4517 }
4518
4519 sub qemu_netdevdel {
4520     my ($vmid, $deviceid) = @_;
4521
4522     vm_mon_cmd($vmid, "netdev_del", id => $deviceid);
4523 }
4524
4525 sub qemu_usb_hotplug {
4526     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4527
4528     return if !$device;
4529
4530     # remove the old one first
4531     vm_deviceunplug($vmid, $conf, $deviceid);
4532
4533     # check if xhci controller is necessary and available
4534     if ($device->{usb3}) {
4535
4536         my $devicelist = vm_devices_list($vmid);
4537
4538         if (!$devicelist->{xhci}) {
4539             my $pciaddr = print_pci_addr("xhci", undef, $arch, $machine_type);
4540             qemu_deviceadd($vmid, "nec-usb-xhci,id=xhci$pciaddr");
4541         }
4542     }
4543     my $d = parse_usb_device($device->{host});
4544     $d->{usb3} = $device->{usb3};
4545
4546     # add the new one
4547     vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $d, $arch, $machine_type);
4548 }
4549
4550 sub qemu_cpu_hotplug {
4551     my ($vmid, $conf, $vcpus) = @_;
4552
4553     my $machine_type = PVE::QemuServer::get_current_qemu_machine($vmid);
4554
4555     my $sockets = 1;
4556     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
4557     $sockets = $conf->{sockets} if  $conf->{sockets};
4558     my $cores = $conf->{cores} || 1;
4559     my $maxcpus = $sockets * $cores;
4560
4561     $vcpus = $maxcpus if !$vcpus;
4562
4563     die "you can't add more vcpus than maxcpus\n"
4564         if $vcpus > $maxcpus;
4565
4566     my $currentvcpus = $conf->{vcpus} || $maxcpus;
4567
4568     if ($vcpus < $currentvcpus) {
4569
4570         if (qemu_machine_feature_enabled ($machine_type, undef, 2, 7)) {
4571
4572             for (my $i = $currentvcpus; $i > $vcpus; $i--) {
4573                 qemu_devicedel($vmid, "cpu$i");
4574                 my $retry = 0;
4575                 my $currentrunningvcpus = undef;
4576                 while (1) {
4577                     $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
4578                     last if scalar(@{$currentrunningvcpus}) == $i-1;
4579                     raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
4580                     $retry++;
4581                     sleep 1;
4582                 }
4583                 #update conf after each succesfull cpu unplug
4584                 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4585                 PVE::QemuConfig->write_config($vmid, $conf);
4586             }
4587         } else {
4588             die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
4589         }
4590
4591         return;
4592     }
4593
4594     my $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
4595     die "vcpus in running vm does not match its configuration\n"
4596         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
4597
4598     if (qemu_machine_feature_enabled ($machine_type, undef, 2, 7)) {
4599
4600         for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
4601             my $cpustr = print_cpu_device($conf, $i);
4602             qemu_deviceadd($vmid, $cpustr);
4603
4604             my $retry = 0;
4605             my $currentrunningvcpus = undef;
4606             while (1) {
4607                 $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
4608                 last if scalar(@{$currentrunningvcpus}) == $i;
4609                 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
4610                 sleep 1;
4611                 $retry++;
4612             }
4613             #update conf after each succesfull cpu hotplug
4614             $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4615             PVE::QemuConfig->write_config($vmid, $conf);
4616         }
4617     } else {
4618
4619         for (my $i = $currentvcpus; $i < $vcpus; $i++) {
4620             vm_mon_cmd($vmid, "cpu-add", id => int($i));
4621         }
4622     }
4623 }
4624
4625 sub qemu_block_set_io_throttle {
4626     my ($vmid, $deviceid,
4627         $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
4628         $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max,
4629         $bps_max_length, $bps_rd_max_length, $bps_wr_max_length,
4630         $iops_max_length, $iops_rd_max_length, $iops_wr_max_length) = @_;
4631
4632     return if !check_running($vmid) ;
4633
4634     vm_mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
4635         bps => int($bps),
4636         bps_rd => int($bps_rd),
4637         bps_wr => int($bps_wr),
4638         iops => int($iops),
4639         iops_rd => int($iops_rd),
4640         iops_wr => int($iops_wr),
4641         bps_max => int($bps_max),
4642         bps_rd_max => int($bps_rd_max),
4643         bps_wr_max => int($bps_wr_max),
4644         iops_max => int($iops_max),
4645         iops_rd_max => int($iops_rd_max),
4646         iops_wr_max => int($iops_wr_max),
4647         bps_max_length => int($bps_max_length),
4648         bps_rd_max_length => int($bps_rd_max_length),
4649         bps_wr_max_length => int($bps_wr_max_length),
4650         iops_max_length => int($iops_max_length),
4651         iops_rd_max_length => int($iops_rd_max_length),
4652         iops_wr_max_length => int($iops_wr_max_length),
4653     );
4654
4655 }
4656
4657 # old code, only used to shutdown old VM after update
4658 sub __read_avail {
4659     my ($fh, $timeout) = @_;
4660
4661     my $sel = new IO::Select;
4662     $sel->add($fh);
4663
4664     my $res = '';
4665     my $buf;
4666
4667     my @ready;
4668     while (scalar (@ready = $sel->can_read($timeout))) {
4669         my $count;
4670         if ($count = $fh->sysread($buf, 8192)) {
4671             if ($buf =~ /^(.*)\(qemu\) $/s) {
4672                 $res .= $1;
4673                 last;
4674             } else {
4675                 $res .= $buf;
4676             }
4677         } else {
4678             if (!defined($count)) {
4679                 die "$!\n";
4680             }
4681             last;
4682         }
4683     }
4684
4685     die "monitor read timeout\n" if !scalar(@ready);
4686
4687     return $res;
4688 }
4689
4690 sub qemu_block_resize {
4691     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4692
4693     my $running = check_running($vmid);
4694
4695     $size = 0 if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4696
4697     return if !$running;
4698
4699     vm_mon_cmd($vmid, "block_resize", device => $deviceid, size => int($size));
4700
4701 }
4702
4703 sub qemu_volume_snapshot {
4704     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4705
4706     my $running = check_running($vmid);
4707
4708     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4709         vm_mon_cmd($vmid, 'blockdev-snapshot-internal-sync', device => $deviceid, name => $snap);
4710     } else {
4711         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4712     }
4713 }
4714
4715 sub qemu_volume_snapshot_delete {
4716     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4717
4718     my $running = check_running($vmid);
4719
4720     if($running) {
4721
4722         $running = undef;
4723         my $conf = PVE::QemuConfig->load_config($vmid);
4724         foreach_drive($conf, sub {
4725             my ($ds, $drive) = @_;
4726             $running = 1 if $drive->{file} eq $volid;
4727         });
4728     }
4729
4730     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4731         vm_mon_cmd($vmid, 'blockdev-snapshot-delete-internal-sync', device => $deviceid, name => $snap);
4732     } else {
4733         PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4734     }
4735 }
4736
4737 sub set_migration_caps {
4738     my ($vmid) = @_;
4739
4740     my $cap_ref = [];
4741
4742     my $enabled_cap = {
4743         "auto-converge" => 1,
4744         "xbzrle" => 1,
4745         "x-rdma-pin-all" => 0,
4746         "zero-blocks" => 0,
4747         "compress" => 0
4748     };
4749
4750     my $supported_capabilities = vm_mon_cmd_nocheck($vmid, "query-migrate-capabilities");
4751
4752     for my $supported_capability (@$supported_capabilities) {
4753         push @$cap_ref, {
4754             capability => $supported_capability->{capability},
4755             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4756         };
4757     }
4758
4759     vm_mon_cmd_nocheck($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4760 }
4761
4762 my $fast_plug_option = {
4763     'lock' => 1,
4764     'name' => 1,
4765     'onboot' => 1,
4766     'shares' => 1,
4767     'startup' => 1,
4768     'description' => 1,
4769     'protection' => 1,
4770     'vmstatestorage' => 1,
4771     'hookscript' => 1,
4772 };
4773
4774 # hotplug changes in [PENDING]
4775 # $selection hash can be used to only apply specified options, for
4776 # example: { cores => 1 } (only apply changed 'cores')
4777 # $errors ref is used to return error messages
4778 sub vmconfig_hotplug_pending {
4779     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4780
4781     my $defaults = load_defaults();
4782     my ($arch, $machine_type) = get_basic_machine_info($conf, undef);
4783
4784     # commit values which do not have any impact on running VM first
4785     # Note: those option cannot raise errors, we we do not care about
4786     # $selection and always apply them.
4787
4788     my $add_error = sub {
4789         my ($opt, $msg) = @_;
4790         $errors->{$opt} = "hotplug problem - $msg";
4791     };
4792
4793     my $changes = 0;
4794     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4795         if ($fast_plug_option->{$opt}) {
4796             $conf->{$opt} = $conf->{pending}->{$opt};
4797             delete $conf->{pending}->{$opt};
4798             $changes = 1;
4799         }
4800     }
4801
4802     if ($changes) {
4803         PVE::QemuConfig->write_config($vmid, $conf);
4804         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4805     }
4806
4807     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4808
4809     my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4810     foreach my $opt (sort keys %$pending_delete_hash) {
4811         next if $selection && !$selection->{$opt};
4812         my $force = $pending_delete_hash->{$opt}->{force};
4813         eval {
4814             if ($opt eq 'hotplug') {
4815                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
4816             } elsif ($opt eq 'tablet') {
4817                 die "skip\n" if !$hotplug_features->{usb};
4818                 if ($defaults->{tablet}) {
4819                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4820                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4821                         if $arch eq 'aarch64';
4822                 } else {
4823                     vm_deviceunplug($vmid, $conf, 'tablet');
4824                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4825                 }
4826             } elsif ($opt =~ m/^usb\d+/) {
4827                 die "skip\n";
4828                 # since we cannot reliably hot unplug usb devices
4829                 # we are disabling it
4830                 die "skip\n" if !$hotplug_features->{usb} || $conf->{$opt} =~ m/spice/i;
4831                 vm_deviceunplug($vmid, $conf, $opt);
4832             } elsif ($opt eq 'vcpus') {
4833                 die "skip\n" if !$hotplug_features->{cpu};
4834                 qemu_cpu_hotplug($vmid, $conf, undef);
4835             } elsif ($opt eq 'balloon') {
4836                 # enable balloon device is not hotpluggable
4837                 die "skip\n" if defined($conf->{balloon}) && $conf->{balloon} == 0;
4838                 # here we reset the ballooning value to memory
4839                 my $balloon = $conf->{memory} || $defaults->{memory};
4840                 vm_mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4841             } elsif ($fast_plug_option->{$opt}) {
4842                 # do nothing
4843             } elsif ($opt =~ m/^net(\d+)$/) {
4844                 die "skip\n" if !$hotplug_features->{network};
4845                 vm_deviceunplug($vmid, $conf, $opt);
4846             } elsif (is_valid_drivename($opt)) {
4847                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4848                 vm_deviceunplug($vmid, $conf, $opt);
4849                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4850             } elsif ($opt =~ m/^memory$/) {
4851                 die "skip\n" if !$hotplug_features->{memory};
4852                 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
4853             } elsif ($opt eq 'cpuunits') {
4854                 cgroups_write("cpu", $vmid, "cpu.shares", $defaults->{cpuunits});
4855             } elsif ($opt eq 'cpulimit') {
4856                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", -1);
4857             } else {
4858                 die "skip\n";
4859             }
4860         };
4861         if (my $err = $@) {
4862             &$add_error($opt, $err) if $err ne "skip\n";
4863         } else {
4864             # save new config if hotplug was successful
4865             delete $conf->{$opt};
4866             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4867             PVE::QemuConfig->write_config($vmid, $conf);
4868             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4869         }
4870     }
4871
4872     my ($apply_pending_cloudinit, $apply_pending_cloudinit_done);
4873     $apply_pending_cloudinit = sub {
4874         return if $apply_pending_cloudinit_done; # once is enough
4875         $apply_pending_cloudinit_done = 1; # once is enough
4876
4877         my ($key, $value) = @_;
4878
4879         my @cloudinit_opts = keys %$confdesc_cloudinit;
4880         foreach my $opt (keys %{$conf->{pending}}) {
4881             next if !grep { $_ eq $opt } @cloudinit_opts;
4882             $conf->{$opt} = delete $conf->{pending}->{$opt};
4883         }
4884
4885         my $new_conf = { %$conf };
4886         $new_conf->{$key} = $value;
4887         PVE::QemuServer::Cloudinit::generate_cloudinitconfig($new_conf, $vmid);
4888     };
4889
4890     foreach my $opt (keys %{$conf->{pending}}) {
4891         next if $selection && !$selection->{$opt};
4892         my $value = $conf->{pending}->{$opt};
4893         eval {
4894             if ($opt eq 'hotplug') {
4895                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
4896             } elsif ($opt eq 'tablet') {
4897                 die "skip\n" if !$hotplug_features->{usb};
4898                 if ($value == 1) {
4899                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4900                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4901                         if $arch eq 'aarch64';
4902                 } elsif ($value == 0) {
4903                     vm_deviceunplug($vmid, $conf, 'tablet');
4904                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4905                 }
4906             } elsif ($opt =~ m/^usb\d+$/) {
4907                 die "skip\n";
4908                 # since we cannot reliably hot unplug usb devices
4909                 # we are disabling it
4910                 die "skip\n" if !$hotplug_features->{usb} || $value =~ m/spice/i;
4911                 my $d = eval { PVE::JSONSchema::parse_property_string($usbdesc->{format}, $value) };
4912                 die "skip\n" if !$d;
4913                 qemu_usb_hotplug($storecfg, $conf, $vmid, $opt, $d, $arch, $machine_type);
4914             } elsif ($opt eq 'vcpus') {
4915                 die "skip\n" if !$hotplug_features->{cpu};
4916                 qemu_cpu_hotplug($vmid, $conf, $value);
4917             } elsif ($opt eq 'balloon') {
4918                 # enable/disable balloning device is not hotpluggable
4919                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
4920                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
4921                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
4922
4923                 # allow manual ballooning if shares is set to zero
4924                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
4925                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
4926                     vm_mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4927                 }
4928             } elsif ($opt =~ m/^net(\d+)$/) {
4929                 # some changes can be done without hotplug
4930                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
4931                                     $vmid, $opt, $value, $arch, $machine_type);
4932             } elsif (is_valid_drivename($opt)) {
4933                 # some changes can be done without hotplug
4934                 my $drive = parse_drive($opt, $value);
4935                 if (drive_is_cloudinit($drive)) {
4936                     &$apply_pending_cloudinit($opt, $value);
4937                 }
4938                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
4939                                      $vmid, $opt, $value, 1, $arch, $machine_type);
4940             } elsif ($opt =~ m/^memory$/) { #dimms
4941                 die "skip\n" if !$hotplug_features->{memory};
4942                 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
4943             } elsif ($opt eq 'cpuunits') {
4944                 cgroups_write("cpu", $vmid, "cpu.shares", $conf->{pending}->{$opt});
4945             } elsif ($opt eq 'cpulimit') {
4946                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
4947                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", $cpulimit);
4948             } else {
4949                 die "skip\n";  # skip non-hot-pluggable options
4950             }
4951         };
4952         if (my $err = $@) {
4953             &$add_error($opt, $err) if $err ne "skip\n";
4954         } else {
4955             # save new config if hotplug was successful
4956             $conf->{$opt} = $value;
4957             delete $conf->{pending}->{$opt};
4958             PVE::QemuConfig->write_config($vmid, $conf);
4959             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4960         }
4961     }
4962 }
4963
4964 sub try_deallocate_drive {
4965     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
4966
4967     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
4968         my $volid = $drive->{file};
4969         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
4970             my $sid = PVE::Storage::parse_volume_id($volid);
4971             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
4972
4973             # check if the disk is really unused
4974             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
4975                 if is_volume_in_use($storecfg, $conf, $key, $volid);
4976             PVE::Storage::vdisk_free($storecfg, $volid);
4977             return 1;
4978         } else {
4979             # If vm is not owner of this disk remove from config
4980             return 1;
4981         }
4982     }
4983
4984     return undef;
4985 }
4986
4987 sub vmconfig_delete_or_detach_drive {
4988     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
4989
4990     my $drive = parse_drive($opt, $conf->{$opt});
4991
4992     my $rpcenv = PVE::RPCEnvironment::get();
4993     my $authuser = $rpcenv->get_user();
4994
4995     if ($force) {
4996         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
4997         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
4998     } else {
4999         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
5000     }
5001 }
5002
5003
5004
5005 sub vmconfig_apply_pending {
5006     my ($vmid, $conf, $storecfg) = @_;
5007
5008     # cold plug
5009
5010     my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
5011     foreach my $opt (sort keys %$pending_delete_hash) {
5012         die "internal error" if $opt =~ m/^unused/;
5013         my $force = $pending_delete_hash->{$opt}->{force};
5014         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5015         if (!defined($conf->{$opt})) {
5016             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
5017             PVE::QemuConfig->write_config($vmid, $conf);
5018         } elsif (is_valid_drivename($opt)) {
5019             vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
5020             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
5021             delete $conf->{$opt};
5022             PVE::QemuConfig->write_config($vmid, $conf);
5023         } else {
5024             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
5025             delete $conf->{$opt};
5026             PVE::QemuConfig->write_config($vmid, $conf);
5027         }
5028     }
5029
5030     $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5031
5032     foreach my $opt (keys %{$conf->{pending}}) { # add/change
5033         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5034
5035         if (defined($conf->{$opt}) && ($conf->{$opt} eq $conf->{pending}->{$opt})) {
5036             # skip if nothing changed
5037         } elsif (is_valid_drivename($opt)) {
5038             vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
5039                 if defined($conf->{$opt});
5040             $conf->{$opt} = $conf->{pending}->{$opt};
5041         } else {
5042             $conf->{$opt} = $conf->{pending}->{$opt};
5043         }
5044
5045         delete $conf->{pending}->{$opt};
5046         PVE::QemuConfig->write_config($vmid, $conf);
5047     }
5048 }
5049
5050 my $safe_num_ne = sub {
5051     my ($a, $b) = @_;
5052
5053     return 0 if !defined($a) && !defined($b);
5054     return 1 if !defined($a);
5055     return 1 if !defined($b);
5056
5057     return $a != $b;
5058 };
5059
5060 my $safe_string_ne = sub {
5061     my ($a, $b) = @_;
5062
5063     return 0 if !defined($a) && !defined($b);
5064     return 1 if !defined($a);
5065     return 1 if !defined($b);
5066
5067     return $a ne $b;
5068 };
5069
5070 sub vmconfig_update_net {
5071     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
5072
5073     my $newnet = parse_net($value);
5074
5075     if ($conf->{$opt}) {
5076         my $oldnet = parse_net($conf->{$opt});
5077
5078         if (&$safe_string_ne($oldnet->{model}, $newnet->{model}) ||
5079             &$safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
5080             &$safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
5081             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
5082
5083             # for non online change, we try to hot-unplug
5084             die "skip\n" if !$hotplug;
5085             vm_deviceunplug($vmid, $conf, $opt);
5086         } else {
5087
5088             die "internal error" if $opt !~ m/net(\d+)/;
5089             my $iface = "tap${vmid}i$1";
5090
5091             if (&$safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
5092                 &$safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
5093                 &$safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
5094                 &$safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
5095                 PVE::Network::tap_unplug($iface);
5096                 PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
5097             } elsif (&$safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
5098                 # Rate can be applied on its own but any change above needs to
5099                 # include the rate in tap_plug since OVS resets everything.
5100                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
5101             }
5102
5103             if (&$safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
5104                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
5105             }
5106
5107             return 1;
5108         }
5109     }
5110
5111     if ($hotplug) {
5112         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet, $arch, $machine_type);
5113     } else {
5114         die "skip\n";
5115     }
5116 }
5117
5118 sub vmconfig_update_disk {
5119     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $force, $arch, $machine_type) = @_;
5120
5121     # fixme: do we need force?
5122
5123     my $drive = parse_drive($opt, $value);
5124
5125     if ($conf->{$opt}) {
5126
5127         if (my $old_drive = parse_drive($opt, $conf->{$opt}))  {
5128
5129             my $media = $drive->{media} || 'disk';
5130             my $oldmedia = $old_drive->{media} || 'disk';
5131             die "unable to change media type\n" if $media ne $oldmedia;
5132
5133             if (!drive_is_cdrom($old_drive)) {
5134
5135                 if ($drive->{file} ne $old_drive->{file}) {
5136
5137                     die "skip\n" if !$hotplug;
5138
5139                     # unplug and register as unused
5140                     vm_deviceunplug($vmid, $conf, $opt);
5141                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
5142
5143                 } else {
5144                     # update existing disk
5145
5146                     # skip non hotpluggable value
5147                     if (&$safe_string_ne($drive->{discard}, $old_drive->{discard}) ||
5148                         &$safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
5149                         &$safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
5150                         &$safe_string_ne($drive->{cache}, $old_drive->{cache})) {
5151                         die "skip\n";
5152                     }
5153
5154                     # apply throttle
5155                     if (&$safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
5156                         &$safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
5157                         &$safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
5158                         &$safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
5159                         &$safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
5160                         &$safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
5161                         &$safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
5162                         &$safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
5163                         &$safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
5164                         &$safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
5165                         &$safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
5166                         &$safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max}) ||
5167                         &$safe_num_ne($drive->{bps_max_length}, $old_drive->{bps_max_length}) ||
5168                         &$safe_num_ne($drive->{bps_rd_max_length}, $old_drive->{bps_rd_max_length}) ||
5169                         &$safe_num_ne($drive->{bps_wr_max_length}, $old_drive->{bps_wr_max_length}) ||
5170                         &$safe_num_ne($drive->{iops_max_length}, $old_drive->{iops_max_length}) ||
5171                         &$safe_num_ne($drive->{iops_rd_max_length}, $old_drive->{iops_rd_max_length}) ||
5172                         &$safe_num_ne($drive->{iops_wr_max_length}, $old_drive->{iops_wr_max_length})) {
5173
5174                         qemu_block_set_io_throttle($vmid,"drive-$opt",
5175                                                    ($drive->{mbps} || 0)*1024*1024,
5176                                                    ($drive->{mbps_rd} || 0)*1024*1024,
5177                                                    ($drive->{mbps_wr} || 0)*1024*1024,
5178                                                    $drive->{iops} || 0,
5179                                                    $drive->{iops_rd} || 0,
5180                                                    $drive->{iops_wr} || 0,
5181                                                    ($drive->{mbps_max} || 0)*1024*1024,
5182                                                    ($drive->{mbps_rd_max} || 0)*1024*1024,
5183                                                    ($drive->{mbps_wr_max} || 0)*1024*1024,
5184                                                    $drive->{iops_max} || 0,
5185                                                    $drive->{iops_rd_max} || 0,
5186                                                    $drive->{iops_wr_max} || 0,
5187                                                    $drive->{bps_max_length} || 1,
5188                                                    $drive->{bps_rd_max_length} || 1,
5189                                                    $drive->{bps_wr_max_length} || 1,
5190                                                    $drive->{iops_max_length} || 1,
5191                                                    $drive->{iops_rd_max_length} || 1,
5192                                                    $drive->{iops_wr_max_length} || 1);
5193
5194                     }
5195
5196                     return 1;
5197                 }
5198
5199             } else { # cdrom
5200
5201                 if ($drive->{file} eq 'none') {
5202                     vm_mon_cmd($vmid, "eject",force => JSON::true,device => "drive-$opt");
5203                     if (drive_is_cloudinit($old_drive)) {
5204                         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive);
5205                     }
5206                 } else {
5207                     my $path = get_iso_path($storecfg, $vmid, $drive->{file});
5208                     vm_mon_cmd($vmid, "eject", force => JSON::true,device => "drive-$opt"); # force eject if locked
5209                     vm_mon_cmd($vmid, "change", device => "drive-$opt",target => "$path") if $path;
5210                 }
5211
5212                 return 1;
5213             }
5214         }
5215     }
5216
5217     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
5218     # hotplug new disks
5219     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
5220     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive, $arch, $machine_type);
5221 }
5222
5223 sub vm_start {
5224     my ($storecfg, $vmid, $statefile, $skiplock, $migratedfrom, $paused,
5225         $forcemachine, $spice_ticket, $migration_network, $migration_type, $targetstorage) = @_;
5226
5227     PVE::QemuConfig->lock_config($vmid, sub {
5228         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
5229
5230         die "you can't start a vm if it's a template\n" if PVE::QemuConfig->is_template($conf);
5231
5232         my $is_suspended = PVE::QemuConfig->has_lock($conf, 'suspended');
5233
5234         PVE::QemuConfig->check_lock($conf)
5235             if !($skiplock || $is_suspended);
5236
5237         die "VM $vmid already running\n" if check_running($vmid, undef, $migratedfrom);
5238
5239         # clean up leftover reboot request files
5240         eval { clear_reboot_request($vmid); };
5241         warn $@ if $@;
5242
5243         if (!$statefile && scalar(keys %{$conf->{pending}})) {
5244             vmconfig_apply_pending($vmid, $conf, $storecfg);
5245             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5246         }
5247
5248         PVE::QemuServer::Cloudinit::generate_cloudinitconfig($conf, $vmid);
5249
5250         my $defaults = load_defaults();
5251
5252         # set environment variable useful inside network script
5253         $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
5254
5255         my $local_volumes = {};
5256
5257         if ($targetstorage) {
5258             foreach_drive($conf, sub {
5259                 my ($ds, $drive) = @_;
5260
5261                 return if drive_is_cdrom($drive);
5262
5263                 my $volid = $drive->{file};
5264
5265                 return if !$volid;
5266
5267                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
5268
5269                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5270                 return if $scfg->{shared};
5271                 $local_volumes->{$ds} = [$volid, $storeid, $volname];
5272             });
5273
5274             my $format = undef;
5275
5276             foreach my $opt (sort keys %$local_volumes) {
5277
5278                 my ($volid, $storeid, $volname) = @{$local_volumes->{$opt}};
5279                 my $drive = parse_drive($opt, $conf->{$opt});
5280
5281                 #if remote storage is specified, use default format
5282                 if ($targetstorage && $targetstorage ne "1") {
5283                     $storeid = $targetstorage;
5284                     my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
5285                     $format = $defFormat;
5286                 } else {
5287                     #else we use same format than original
5288                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5289                     $format = qemu_img_format($scfg, $volid);
5290                 }
5291
5292                 my $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $format, undef, ($drive->{size}/1024));
5293                 my $newdrive = $drive;
5294                 $newdrive->{format} = $format;
5295                 $newdrive->{file} = $newvolid;
5296                 my $drivestr = PVE::QemuServer::print_drive($vmid, $newdrive);
5297                 $local_volumes->{$opt} = $drivestr;
5298                 #pass drive to conf for command line
5299                 $conf->{$opt} = $drivestr;
5300             }
5301         }
5302
5303         PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-start', 1);
5304
5305         if ($is_suspended) {
5306             # enforce machine type on suspended vm to ensure HW compatibility
5307             $forcemachine = $conf->{runningmachine};
5308             print "Resuming suspended VM\n";
5309         }
5310
5311         my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine);
5312
5313         my $migration_ip;
5314         my $get_migration_ip = sub {
5315             my ($cidr, $nodename) = @_;
5316
5317             return $migration_ip if defined($migration_ip);
5318
5319             if (!defined($cidr)) {
5320                 my $dc_conf = PVE::Cluster::cfs_read_file('datacenter.cfg');
5321                 $cidr = $dc_conf->{migration}->{network};
5322             }
5323
5324             if (defined($cidr)) {
5325                 my $ips = PVE::Network::get_local_ip_from_cidr($cidr);
5326
5327                 die "could not get IP: no address configured on local " .
5328                     "node for network '$cidr'\n" if scalar(@$ips) == 0;
5329
5330                 die "could not get IP: multiple addresses configured on local " .
5331                     "node for network '$cidr'\n" if scalar(@$ips) > 1;
5332
5333                 $migration_ip = @$ips[0];
5334             }
5335
5336             $migration_ip = PVE::Cluster::remote_node_ip($nodename, 1)
5337                 if !defined($migration_ip);
5338
5339             return $migration_ip;
5340         };
5341
5342         my $migrate_uri;
5343         if ($statefile) {
5344             if ($statefile eq 'tcp') {
5345                 my $localip = "localhost";
5346                 my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
5347                 my $nodename = PVE::INotify::nodename();
5348
5349                 if (!defined($migration_type)) {
5350                     if (defined($datacenterconf->{migration}->{type})) {
5351                         $migration_type = $datacenterconf->{migration}->{type};
5352                     } else {
5353                         $migration_type = 'secure';
5354                     }
5355                 }
5356
5357                 if ($migration_type eq 'insecure') {
5358                     $localip = $get_migration_ip->($migration_network, $nodename);
5359                     $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5360                 }
5361
5362                 my $pfamily = PVE::Tools::get_host_address_family($nodename);
5363                 my $migrate_port = PVE::Tools::next_migrate_port($pfamily);
5364                 $migrate_uri = "tcp:${localip}:${migrate_port}";
5365                 push @$cmd, '-incoming', $migrate_uri;
5366                 push @$cmd, '-S';
5367
5368             } elsif ($statefile eq 'unix') {
5369                 # should be default for secure migrations as a ssh TCP forward
5370                 # tunnel is not deterministic reliable ready and fails regurarly
5371                 # to set up in time, so use UNIX socket forwards
5372                 my $socket_addr = "/run/qemu-server/$vmid.migrate";
5373                 unlink $socket_addr;
5374
5375                 $migrate_uri = "unix:$socket_addr";
5376
5377                 push @$cmd, '-incoming', $migrate_uri;
5378                 push @$cmd, '-S';
5379
5380             } elsif (-e $statefile) {
5381                 push @$cmd, '-loadstate', $statefile;
5382             } else {
5383                 my $statepath = PVE::Storage::path($storecfg, $statefile);
5384                 push @$vollist, $statefile;
5385                 push @$cmd, '-loadstate', $statepath;
5386             }
5387         } elsif ($paused) {
5388             push @$cmd, '-S';
5389         }
5390
5391         # host pci devices
5392         for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
5393           my $d = parse_hostpci($conf->{"hostpci$i"});
5394           next if !$d;
5395           my $pcidevices = $d->{pciid};
5396           foreach my $pcidevice (@$pcidevices) {
5397                 my $pciid = $pcidevice->{id};
5398
5399                 my $info = PVE::SysFSTools::pci_device_info("0000:$pciid");
5400                 die "IOMMU not present\n" if !PVE::SysFSTools::check_iommu_support();
5401                 die "no pci device info for device '$pciid'\n" if !$info;
5402
5403                 if ($d->{mdev}) {
5404                     my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $i);
5405                     PVE::SysFSTools::pci_create_mdev_device($pciid, $uuid, $d->{mdev});
5406                 } else {
5407                     die "can't unbind/bind pci group to vfio '$pciid'\n"
5408                         if !PVE::SysFSTools::pci_dev_group_bind_to_vfio($pciid);
5409                     die "can't reset pci device '$pciid'\n"
5410                         if $info->{has_fl_reset} and !PVE::SysFSTools::pci_dev_reset($info);
5411                 }
5412           }
5413         }
5414
5415         PVE::Storage::activate_volumes($storecfg, $vollist);
5416
5417         eval {
5418             run_command(['/bin/systemctl', 'stop', "$vmid.scope"],
5419                 outfunc => sub {}, errfunc => sub {});
5420         };
5421         # Issues with the above 'stop' not being fully completed are extremely rare, a very low
5422         # timeout should be more than enough here...
5423         PVE::Systemd::wait_for_unit_removed("$vmid.scope", 5);
5424
5425         my $cpuunits = defined($conf->{cpuunits}) ? $conf->{cpuunits}
5426                                                   : $defaults->{cpuunits};
5427
5428         my $start_timeout = ($conf->{hugepages} || $is_suspended) ? 300 : 30;
5429         my %run_params = (timeout => $statefile ? undef : $start_timeout, umask => 0077);
5430
5431         my %properties = (
5432             Slice => 'qemu.slice',
5433             KillMode => 'none',
5434             CPUShares => $cpuunits
5435         );
5436
5437         if (my $cpulimit = $conf->{cpulimit}) {
5438             $properties{CPUQuota} = int($cpulimit * 100);
5439         }
5440         $properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
5441
5442         my $run_qemu = sub {
5443             PVE::Tools::run_fork sub {
5444                 PVE::Systemd::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
5445                 run_command($cmd, %run_params);
5446             };
5447         };
5448
5449         if ($conf->{hugepages}) {
5450
5451             my $code = sub {
5452                 my $hugepages_topology = PVE::QemuServer::Memory::hugepages_topology($conf);
5453                 my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
5454
5455                 PVE::QemuServer::Memory::hugepages_mount();
5456                 PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
5457
5458                 eval { $run_qemu->() };
5459                 if (my $err = $@) {
5460                     PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology);
5461                     die $err;
5462                 }
5463
5464                 PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology);
5465             };
5466             eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
5467
5468         } else {
5469             eval { $run_qemu->() };
5470         }
5471
5472         if (my $err = $@) {
5473             # deactivate volumes if start fails
5474             eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
5475             die "start failed: $err";
5476         }
5477
5478         print "migration listens on $migrate_uri\n" if $migrate_uri;
5479
5480         if ($statefile && $statefile ne 'tcp' && $statefile ne 'unix')  {
5481             eval { vm_mon_cmd_nocheck($vmid, "cont"); };
5482             warn $@ if $@;
5483         }
5484
5485         #start nbd server for storage migration
5486         if ($targetstorage) {
5487             my $nodename = PVE::INotify::nodename();
5488             my $localip = $get_migration_ip->($migration_network, $nodename);
5489             my $pfamily = PVE::Tools::get_host_address_family($nodename);
5490             my $storage_migrate_port = PVE::Tools::next_migrate_port($pfamily);
5491
5492             vm_mon_cmd_nocheck($vmid, "nbd-server-start", addr => { type => 'inet', data => { host => "${localip}", port => "${storage_migrate_port}" } } );
5493
5494             $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5495
5496             foreach my $opt (sort keys %$local_volumes) {
5497                 my $volid = $local_volumes->{$opt};
5498                 vm_mon_cmd_nocheck($vmid, "nbd-server-add", device => "drive-$opt", writable => JSON::true );
5499                 my $migrate_storage_uri = "nbd:${localip}:${storage_migrate_port}:exportname=drive-$opt";
5500                 print "storage migration listens on $migrate_storage_uri volume:$volid\n";
5501             }
5502         }
5503
5504         if ($migratedfrom) {
5505             eval {
5506                 set_migration_caps($vmid);
5507             };
5508             warn $@ if $@;
5509
5510             if ($spice_port) {
5511                 print "spice listens on port $spice_port\n";
5512                 if ($spice_ticket) {
5513                     vm_mon_cmd_nocheck($vmid, "set_password", protocol => 'spice', password => $spice_ticket);
5514                     vm_mon_cmd_nocheck($vmid, "expire_password", protocol => 'spice', time => "+30");
5515                 }
5516             }
5517
5518         } else {
5519             vm_mon_cmd_nocheck($vmid, "balloon", value => $conf->{balloon}*1024*1024)
5520                 if !$statefile && $conf->{balloon};
5521
5522             foreach my $opt (keys %$conf) {
5523                 next if $opt !~  m/^net\d+$/;
5524                 my $nicconf = parse_net($conf->{$opt});
5525                 qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
5526             }
5527         }
5528
5529         vm_mon_cmd_nocheck($vmid, 'qom-set',
5530                     path => "machine/peripheral/balloon0",
5531                     property => "guest-stats-polling-interval",
5532                     value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
5533
5534         if ($is_suspended && (my $vmstate = $conf->{vmstate})) {
5535             print "Resumed VM, removing state\n";
5536             delete $conf->@{qw(lock vmstate runningmachine)};
5537             PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
5538             PVE::Storage::vdisk_free($storecfg, $vmstate);
5539             PVE::QemuConfig->write_config($vmid, $conf);
5540         }
5541
5542         PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'post-start');
5543     });
5544 }
5545
5546 sub vm_mon_cmd {
5547     my ($vmid, $execute, %params) = @_;
5548
5549     my $cmd = { execute => $execute, arguments => \%params };
5550     vm_qmp_command($vmid, $cmd);
5551 }
5552
5553 sub vm_mon_cmd_nocheck {
5554     my ($vmid, $execute, %params) = @_;
5555
5556     my $cmd = { execute => $execute, arguments => \%params };
5557     vm_qmp_command($vmid, $cmd, 1);
5558 }
5559
5560 sub vm_qmp_command {
5561     my ($vmid, $cmd, $nocheck) = @_;
5562
5563     my $res;
5564
5565     my $timeout;
5566     if ($cmd->{arguments}) {
5567         $timeout = delete $cmd->{arguments}->{timeout};
5568     }
5569
5570     eval {
5571         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
5572         my $sname = PVE::QemuServer::Helpers::qmp_socket($vmid);
5573         if (-e $sname) { # test if VM is reasonambe new and supports qmp/qga
5574             my $qmpclient = PVE::QMPClient->new();
5575
5576             $res = $qmpclient->cmd($vmid, $cmd, $timeout);
5577         } else {
5578             die "unable to open monitor socket\n";
5579         }
5580     };
5581     if (my $err = $@) {
5582         syslog("err", "VM $vmid qmp command failed - $err");
5583         die $err;
5584     }
5585
5586     return $res;
5587 }
5588
5589 sub vm_human_monitor_command {
5590     my ($vmid, $cmdline) = @_;
5591
5592     my $cmd = {
5593         execute => 'human-monitor-command',
5594         arguments => { 'command-line' => $cmdline},
5595     };
5596
5597     return vm_qmp_command($vmid, $cmd);
5598 }
5599
5600 sub vm_commandline {
5601     my ($storecfg, $vmid, $snapname) = @_;
5602
5603     my $conf = PVE::QemuConfig->load_config($vmid);
5604     my $forcemachine;
5605
5606     if ($snapname) {
5607         my $snapshot = $conf->{snapshots}->{$snapname};
5608         die "snapshot '$snapname' does not exist\n" if !defined($snapshot);
5609
5610         # check for a 'runningmachine' in snapshot
5611         $forcemachine = $snapshot->{runningmachine} if $snapshot->{runningmachine};
5612
5613         $snapshot->{digest} = $conf->{digest}; # keep file digest for API
5614
5615         $conf = $snapshot;
5616     }
5617
5618     my $defaults = load_defaults();
5619
5620     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine);
5621
5622     return PVE::Tools::cmd2string($cmd);
5623 }
5624
5625 sub vm_reset {
5626     my ($vmid, $skiplock) = @_;
5627
5628     PVE::QemuConfig->lock_config($vmid, sub {
5629
5630         my $conf = PVE::QemuConfig->load_config($vmid);
5631
5632         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5633
5634         vm_mon_cmd($vmid, "system_reset");
5635     });
5636 }
5637
5638 sub get_vm_volumes {
5639     my ($conf) = @_;
5640
5641     my $vollist = [];
5642     foreach_volid($conf, sub {
5643         my ($volid, $attr) = @_;
5644
5645         return if $volid =~ m|^/|;
5646
5647         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5648         return if !$sid;
5649
5650         push @$vollist, $volid;
5651     });
5652
5653     return $vollist;
5654 }
5655
5656 sub vm_stop_cleanup {
5657     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
5658
5659     eval {
5660
5661         if (!$keepActive) {
5662             my $vollist = get_vm_volumes($conf);
5663             PVE::Storage::deactivate_volumes($storecfg, $vollist);
5664         }
5665
5666         foreach my $ext (qw(mon qmp pid vnc qga)) {
5667             unlink "/var/run/qemu-server/${vmid}.$ext";
5668         }
5669
5670         if ($conf->{ivshmem}) {
5671             my $ivshmem = PVE::JSONSchema::parse_property_string($ivshmem_fmt, $conf->{ivshmem});
5672             # just delete it for now, VMs which have this already open do not
5673             # are affected, but new VMs will get a separated one. If this
5674             # becomes an issue we either add some sort of ref-counting or just
5675             # add a "don't delete on stop" flag to the ivshmem format.
5676             unlink '/dev/shm/pve-shm-' . ($ivshmem->{name} // $vmid);
5677         }
5678
5679         foreach my $key (keys %$conf) {
5680             next if $key !~ m/^hostpci(\d+)$/;
5681             my $hostpciindex = $1;
5682             my $d = parse_hostpci($conf->{$key});
5683             my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $hostpciindex);
5684
5685             foreach my $pci (@{$d->{pciid}}) {
5686                 my $pciid = $pci->{id};
5687                 PVE::SysFSTools::pci_cleanup_mdev_device($pciid, $uuid);
5688             }
5689         }
5690
5691         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
5692     };
5693     warn $@ if $@; # avoid errors - just warn
5694 }
5695
5696 # call only in locked context
5697 sub _do_vm_stop {
5698     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive) = @_;
5699
5700     my $pid = check_running($vmid, $nocheck);
5701     return if !$pid;
5702
5703     my $conf;
5704     if (!$nocheck) {
5705         $conf = PVE::QemuConfig->load_config($vmid);
5706         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5707         if (!defined($timeout) && $shutdown && $conf->{startup}) {
5708             my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
5709             $timeout = $opts->{down} if $opts->{down};
5710         }
5711         PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-stop');
5712     }
5713
5714     eval {
5715         if ($shutdown) {
5716             if (defined($conf) && parse_guest_agent($conf)->{enabled}) {
5717                 vm_qmp_command($vmid, {
5718                         execute => "guest-shutdown",
5719                         arguments => { timeout => $timeout }
5720                     }, $nocheck);
5721             } else {
5722                 vm_qmp_command($vmid, { execute => "system_powerdown" }, $nocheck);
5723             }
5724         } else {
5725             vm_qmp_command($vmid, { execute => "quit" }, $nocheck);
5726         }
5727     };
5728     my $err = $@;
5729
5730     if (!$err) {
5731         $timeout = 60 if !defined($timeout);
5732
5733         my $count = 0;
5734         while (($count < $timeout) && check_running($vmid, $nocheck)) {
5735             $count++;
5736             sleep 1;
5737         }
5738
5739         if ($count >= $timeout) {
5740             if ($force) {
5741                 warn "VM still running - terminating now with SIGTERM\n";
5742                 kill 15, $pid;
5743             } else {
5744                 die "VM quit/powerdown failed - got timeout\n";
5745             }
5746         } else {
5747             vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5748             return;
5749         }
5750     } else {
5751         if ($force) {
5752             warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
5753             kill 15, $pid;
5754         } else {
5755             die "VM quit/powerdown failed\n";
5756         }
5757     }
5758
5759     # wait again
5760     $timeout = 10;
5761
5762     my $count = 0;
5763     while (($count < $timeout) && check_running($vmid, $nocheck)) {
5764         $count++;
5765         sleep 1;
5766     }
5767
5768     if ($count >= $timeout) {
5769         warn "VM still running - terminating now with SIGKILL\n";
5770         kill 9, $pid;
5771         sleep 1;
5772     }
5773
5774     vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5775 }
5776
5777 # Note: use $nocheck to skip tests if VM configuration file exists.
5778 # We need that when migration VMs to other nodes (files already moved)
5779 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
5780 sub vm_stop {
5781     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
5782
5783     $force = 1 if !defined($force) && !$shutdown;
5784
5785     if ($migratedfrom){
5786         my $pid = check_running($vmid, $nocheck, $migratedfrom);
5787         kill 15, $pid if $pid;
5788         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
5789         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
5790         return;
5791     }
5792
5793     PVE::QemuConfig->lock_config($vmid, sub {
5794         _do_vm_stop($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive);
5795    });
5796 }
5797
5798 sub vm_reboot {
5799     my ($vmid, $timeout) = @_;
5800
5801     PVE::QemuConfig->lock_config($vmid, sub {
5802         eval {
5803
5804             # only reboot if running, as qmeventd starts it again on a stop event
5805             return if !check_running($vmid);
5806
5807             create_reboot_request($vmid);
5808
5809             my $storecfg = PVE::Storage::config();
5810             _do_vm_stop($storecfg, $vmid, undef, undef, $timeout, 1);
5811
5812         };
5813         if (my $err = $@) {
5814             # avoid that the next normal shutdown will be confused for a reboot
5815             clear_reboot_request($vmid);
5816             die $err;
5817         }
5818    });
5819 }
5820
5821 sub vm_suspend {
5822     my ($vmid, $skiplock, $includestate, $statestorage) = @_;
5823
5824     my $conf;
5825     my $path;
5826     my $storecfg;
5827     my $vmstate;
5828
5829     PVE::QemuConfig->lock_config($vmid, sub {
5830
5831         $conf = PVE::QemuConfig->load_config($vmid);
5832
5833         my $is_backing_up = PVE::QemuConfig->has_lock($conf, 'backup');
5834         PVE::QemuConfig->check_lock($conf)
5835             if !($skiplock || $is_backing_up);
5836
5837         die "cannot suspend to disk during backup\n"
5838             if $is_backing_up && $includestate;
5839
5840         if ($includestate) {
5841             $conf->{lock} = 'suspending';
5842             my $date = strftime("%Y-%m-%d", localtime(time()));
5843             $storecfg = PVE::Storage::config();
5844             $vmstate = PVE::QemuConfig->__snapshot_save_vmstate($vmid, $conf, "suspend-$date", $storecfg, $statestorage, 1);
5845             $path = PVE::Storage::path($storecfg, $vmstate);
5846             PVE::QemuConfig->write_config($vmid, $conf);
5847         } else {
5848             vm_mon_cmd($vmid, "stop");
5849         }
5850     });
5851
5852     if ($includestate) {
5853         # save vm state
5854         PVE::Storage::activate_volumes($storecfg, [$vmstate]);
5855
5856         eval {
5857             vm_mon_cmd($vmid, "savevm-start", statefile => $path);
5858             for(;;) {
5859                 my $state = vm_mon_cmd_nocheck($vmid, "query-savevm");
5860                 if (!$state->{status}) {
5861                     die "savevm not active\n";
5862                 } elsif ($state->{status} eq 'active') {
5863                     sleep(1);
5864                     next;
5865                 } elsif ($state->{status} eq 'completed') {
5866                     print "State saved, quitting\n";
5867                     last;
5868                 } elsif ($state->{status} eq 'failed' && $state->{error}) {
5869                     die "query-savevm failed with error '$state->{error}'\n"
5870                 } else {
5871                     die "query-savevm returned status '$state->{status}'\n";
5872                 }
5873             }
5874         };
5875         my $err = $@;
5876
5877         PVE::QemuConfig->lock_config($vmid, sub {
5878             $conf = PVE::QemuConfig->load_config($vmid);
5879             if ($err) {
5880                 # cleanup, but leave suspending lock, to indicate something went wrong
5881                 eval {
5882                     vm_mon_cmd($vmid, "savevm-end");
5883                     PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
5884                     PVE::Storage::vdisk_free($storecfg, $vmstate);
5885                     delete $conf->@{qw(vmstate runningmachine)};
5886                     PVE::QemuConfig->write_config($vmid, $conf);
5887                 };
5888                 warn $@ if $@;
5889                 die $err;
5890             }
5891
5892             die "lock changed unexpectedly\n"
5893                 if !PVE::QemuConfig->has_lock($conf, 'suspending');
5894
5895             vm_qmp_command($vmid, { execute => "quit" });
5896             $conf->{lock} = 'suspended';
5897             PVE::QemuConfig->write_config($vmid, $conf);
5898         });
5899     }
5900 }
5901
5902 sub vm_resume {
5903     my ($vmid, $skiplock, $nocheck) = @_;
5904
5905     PVE::QemuConfig->lock_config($vmid, sub {
5906         my $vm_mon_cmd = $nocheck ? \&vm_mon_cmd_nocheck : \&vm_mon_cmd;
5907         my $res = $vm_mon_cmd->($vmid, 'query-status');
5908         my $resume_cmd = 'cont';
5909
5910         if ($res->{status} && $res->{status} eq 'suspended') {
5911             $resume_cmd = 'system_wakeup';
5912         }
5913
5914         if (!$nocheck) {
5915
5916             my $conf = PVE::QemuConfig->load_config($vmid);
5917
5918             PVE::QemuConfig->check_lock($conf)
5919                 if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
5920         }
5921
5922         $vm_mon_cmd->($vmid, $resume_cmd);
5923     });
5924 }
5925
5926 sub vm_sendkey {
5927     my ($vmid, $skiplock, $key) = @_;
5928
5929     PVE::QemuConfig->lock_config($vmid, sub {
5930
5931         my $conf = PVE::QemuConfig->load_config($vmid);
5932
5933         # there is no qmp command, so we use the human monitor command
5934         my $res = vm_human_monitor_command($vmid, "sendkey $key");
5935         die $res if $res ne '';
5936     });
5937 }
5938
5939 # vzdump restore implementaion
5940
5941 sub tar_archive_read_firstfile {
5942     my $archive = shift;
5943
5944     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
5945
5946     # try to detect archive type first
5947     my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
5948         die "unable to open file '$archive'\n";
5949     my $firstfile = <$fh>;
5950     kill 15, $pid;
5951     close $fh;
5952
5953     die "ERROR: archive contaions no data\n" if !$firstfile;
5954     chomp $firstfile;
5955
5956     return $firstfile;
5957 }
5958
5959 sub tar_restore_cleanup {
5960     my ($storecfg, $statfile) = @_;
5961
5962     print STDERR "starting cleanup\n";
5963
5964     if (my $fd = IO::File->new($statfile, "r")) {
5965         while (defined(my $line = <$fd>)) {
5966             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5967                 my $volid = $2;
5968                 eval {
5969                     if ($volid =~ m|^/|) {
5970                         unlink $volid || die 'unlink failed\n';
5971                     } else {
5972                         PVE::Storage::vdisk_free($storecfg, $volid);
5973                     }
5974                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
5975                 };
5976                 print STDERR "unable to cleanup '$volid' - $@" if $@;
5977             } else {
5978                 print STDERR "unable to parse line in statfile - $line";
5979             }
5980         }
5981         $fd->close();
5982     }
5983 }
5984
5985 sub restore_archive {
5986     my ($archive, $vmid, $user, $opts) = @_;
5987
5988     my $format = $opts->{format};
5989     my $comp;
5990
5991     if ($archive =~ m/\.tgz$/ || $archive =~ m/\.tar\.gz$/) {
5992         $format = 'tar' if !$format;
5993         $comp = 'gzip';
5994     } elsif ($archive =~ m/\.tar$/) {
5995         $format = 'tar' if !$format;
5996     } elsif ($archive =~ m/.tar.lzo$/) {
5997         $format = 'tar' if !$format;
5998         $comp = 'lzop';
5999     } elsif ($archive =~ m/\.vma$/) {
6000         $format = 'vma' if !$format;
6001     } elsif ($archive =~ m/\.vma\.gz$/) {
6002         $format = 'vma' if !$format;
6003         $comp = 'gzip';
6004     } elsif ($archive =~ m/\.vma\.lzo$/) {
6005         $format = 'vma' if !$format;
6006         $comp = 'lzop';
6007     } else {
6008         $format = 'vma' if !$format; # default
6009     }
6010
6011     # try to detect archive format
6012     if ($format eq 'tar') {
6013         return restore_tar_archive($archive, $vmid, $user, $opts);
6014     } else {
6015         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
6016     }
6017 }
6018
6019 sub restore_update_config_line {
6020     my ($outfd, $cookie, $vmid, $map, $line, $unique) = @_;
6021
6022     return if $line =~ m/^\#qmdump\#/;
6023     return if $line =~ m/^\#vzdump\#/;
6024     return if $line =~ m/^lock:/;
6025     return if $line =~ m/^unused\d+:/;
6026     return if $line =~ m/^parent:/;
6027
6028     my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
6029     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
6030         # try to convert old 1.X settings
6031         my ($id, $ind, $ethcfg) = ($1, $2, $3);
6032         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
6033             my ($model, $macaddr) = split(/\=/, $devconfig);
6034             $macaddr = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if !$macaddr || $unique;
6035             my $net = {
6036                 model => $model,
6037                 bridge => "vmbr$ind",
6038                 macaddr => $macaddr,
6039             };
6040             my $netstr = print_net($net);
6041
6042             print $outfd "net$cookie->{netcount}: $netstr\n";
6043             $cookie->{netcount}++;
6044         }
6045     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
6046         my ($id, $netstr) = ($1, $2);
6047         my $net = parse_net($netstr);
6048         $net->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if $net->{macaddr};
6049         $netstr = print_net($net);
6050         print $outfd "$id: $netstr\n";
6051     } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk)\d+):\s*(\S+)\s*$/) {
6052         my $virtdev = $1;
6053         my $value = $3;
6054         my $di = parse_drive($virtdev, $value);
6055         if (defined($di->{backup}) && !$di->{backup}) {
6056             print $outfd "#$line";
6057         } elsif ($map->{$virtdev}) {
6058             delete $di->{format}; # format can change on restore
6059             $di->{file} = $map->{$virtdev};
6060             $value = print_drive($vmid, $di);
6061             print $outfd "$virtdev: $value\n";
6062         } else {
6063             print $outfd $line;
6064         }
6065     } elsif (($line =~ m/^vmgenid: (.*)/)) {
6066         my $vmgenid = $1;
6067         if ($vmgenid ne '0') {
6068             # always generate a new vmgenid if there was a valid one setup
6069             $vmgenid = generate_uuid();
6070         }
6071         print $outfd "vmgenid: $vmgenid\n";
6072     } elsif (($line =~ m/^(smbios1: )(.*)/) && $unique) {
6073         my ($uuid, $uuid_str);
6074         UUID::generate($uuid);
6075         UUID::unparse($uuid, $uuid_str);
6076         my $smbios1 = parse_smbios1($2);
6077         $smbios1->{uuid} = $uuid_str;
6078         print $outfd $1.print_smbios1($smbios1)."\n";
6079     } else {
6080         print $outfd $line;
6081     }
6082 }
6083
6084 sub scan_volids {
6085     my ($cfg, $vmid) = @_;
6086
6087     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid);
6088
6089     my $volid_hash = {};
6090     foreach my $storeid (keys %$info) {
6091         foreach my $item (@{$info->{$storeid}}) {
6092             next if !($item->{volid} && $item->{size});
6093             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
6094             $volid_hash->{$item->{volid}} = $item;
6095         }
6096     }
6097
6098     return $volid_hash;
6099 }
6100
6101 sub is_volume_in_use {
6102     my ($storecfg, $conf, $skip_drive, $volid) = @_;
6103
6104     my $path = PVE::Storage::path($storecfg, $volid);
6105
6106     my $scan_config = sub {
6107         my ($cref, $snapname) = @_;
6108
6109         foreach my $key (keys %$cref) {
6110             my $value = $cref->{$key};
6111             if (is_valid_drivename($key)) {
6112                 next if $skip_drive && $key eq $skip_drive;
6113                 my $drive = parse_drive($key, $value);
6114                 next if !$drive || !$drive->{file} || drive_is_cdrom($drive);
6115                 return 1 if $volid eq $drive->{file};
6116                 if ($drive->{file} =~ m!^/!) {
6117                     return 1 if $drive->{file} eq $path;
6118                 } else {
6119                     my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file}, 1);
6120                     next if !$storeid;
6121                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid, 1);
6122                     next if !$scfg;
6123                     return 1 if $path eq PVE::Storage::path($storecfg, $drive->{file}, $snapname);
6124                 }
6125             }
6126         }
6127
6128         return 0;
6129     };
6130
6131     return 1 if &$scan_config($conf);
6132
6133     undef $skip_drive;
6134
6135     foreach my $snapname (keys %{$conf->{snapshots}}) {
6136         return 1 if &$scan_config($conf->{snapshots}->{$snapname}, $snapname);
6137     }
6138
6139     return 0;
6140 }
6141
6142 sub update_disksize {
6143     my ($vmid, $conf, $volid_hash) = @_;
6144
6145     my $changes;
6146     my $prefix = "VM $vmid:";
6147
6148     # used and unused disks
6149     my $referenced = {};
6150
6151     # Note: it is allowed to define multiple storages with same path (alias), so
6152     # we need to check both 'volid' and real 'path' (two different volid can point
6153     # to the same path).
6154
6155     my $referencedpath = {};
6156
6157     # update size info
6158     foreach my $opt (keys %$conf) {
6159         if (is_valid_drivename($opt)) {
6160             my $drive = parse_drive($opt, $conf->{$opt});
6161             my $volid = $drive->{file};
6162             next if !$volid;
6163
6164             $referenced->{$volid} = 1;
6165             if ($volid_hash->{$volid} &&
6166                 (my $path = $volid_hash->{$volid}->{path})) {
6167                 $referencedpath->{$path} = 1;
6168             }
6169
6170             next if drive_is_cdrom($drive);
6171             next if !$volid_hash->{$volid};
6172
6173             $drive->{size} = $volid_hash->{$volid}->{size};
6174             my $new = print_drive($vmid, $drive);
6175             if ($new ne $conf->{$opt}) {
6176                 $changes = 1;
6177                 $conf->{$opt} = $new;
6178                 print "$prefix update disk '$opt' information.\n";
6179             }
6180         }
6181     }
6182
6183     # remove 'unusedX' entry if volume is used
6184     foreach my $opt (keys %$conf) {
6185         next if $opt !~ m/^unused\d+$/;
6186         my $volid = $conf->{$opt};
6187         my $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
6188         if ($referenced->{$volid} || ($path && $referencedpath->{$path})) {
6189             print "$prefix remove entry '$opt', its volume '$volid' is in use.\n";
6190             $changes = 1;
6191             delete $conf->{$opt};
6192         }
6193
6194         $referenced->{$volid} = 1;
6195         $referencedpath->{$path} = 1 if $path;
6196     }
6197
6198     foreach my $volid (sort keys %$volid_hash) {
6199         next if $volid =~ m/vm-$vmid-state-/;
6200         next if $referenced->{$volid};
6201         my $path = $volid_hash->{$volid}->{path};
6202         next if !$path; # just to be sure
6203         next if $referencedpath->{$path};
6204         $changes = 1;
6205         my $key = PVE::QemuConfig->add_unused_volume($conf, $volid);
6206         print "$prefix add unreferenced volume '$volid' as '$key' to config.\n";
6207         $referencedpath->{$path} = 1; # avoid to add more than once (aliases)
6208     }
6209
6210     return $changes;
6211 }
6212
6213 sub rescan {
6214     my ($vmid, $nolock, $dryrun) = @_;
6215
6216     my $cfg = PVE::Storage::config();
6217
6218     # FIXME: Remove once our RBD plugin can handle CT and VM on a single storage
6219     # see: https://pve.proxmox.com/pipermail/pve-devel/2018-July/032900.html
6220     foreach my $stor (keys %{$cfg->{ids}}) {
6221         delete($cfg->{ids}->{$stor}) if ! $cfg->{ids}->{$stor}->{content}->{images};
6222     }
6223
6224     print "rescan volumes...\n";
6225     my $volid_hash = scan_volids($cfg, $vmid);
6226
6227     my $updatefn =  sub {
6228         my ($vmid) = @_;
6229
6230         my $conf = PVE::QemuConfig->load_config($vmid);
6231
6232         PVE::QemuConfig->check_lock($conf);
6233
6234         my $vm_volids = {};
6235         foreach my $volid (keys %$volid_hash) {
6236             my $info = $volid_hash->{$volid};
6237             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
6238         }
6239
6240         my $changes = update_disksize($vmid, $conf, $vm_volids);
6241
6242         PVE::QemuConfig->write_config($vmid, $conf) if $changes && !$dryrun;
6243     };
6244
6245     if (defined($vmid)) {
6246         if ($nolock) {
6247             &$updatefn($vmid);
6248         } else {
6249             PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6250         }
6251     } else {
6252         my $vmlist = config_list();
6253         foreach my $vmid (keys %$vmlist) {
6254             if ($nolock) {
6255                 &$updatefn($vmid);
6256             } else {
6257                 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6258             }
6259         }
6260     }
6261 }
6262
6263 sub restore_vma_archive {
6264     my ($archive, $vmid, $user, $opts, $comp) = @_;
6265
6266     my $readfrom = $archive;
6267
6268     my $cfg = PVE::Storage::config();
6269     my $commands = [];
6270     my $bwlimit = $opts->{bwlimit};
6271
6272     my $dbg_cmdstring = '';
6273     my $add_pipe = sub {
6274         my ($cmd) = @_;
6275         push @$commands, $cmd;
6276         $dbg_cmdstring .= ' | ' if length($dbg_cmdstring);
6277         $dbg_cmdstring .= PVE::Tools::cmd2string($cmd);
6278         $readfrom = '-';
6279     };
6280
6281     my $input = undef;
6282     if ($archive eq '-') {
6283         $input = '<&STDIN';
6284     } else {
6285         # If we use a backup from a PVE defined storage we also consider that
6286         # storage's rate limit:
6287         my (undef, $volid) = PVE::Storage::path_to_volume_id($cfg, $archive);
6288         if (defined($volid)) {
6289             my ($sid, undef) = PVE::Storage::parse_volume_id($volid);
6290             my $readlimit = PVE::Storage::get_bandwidth_limit('restore', [$sid], $bwlimit);
6291             if ($readlimit) {
6292                 print STDERR "applying read rate limit: $readlimit\n";
6293                 my $cstream = ['cstream', '-t', $readlimit*1024, '--', $readfrom];
6294                 $add_pipe->($cstream);
6295             }
6296         }
6297     }
6298
6299     if ($comp) {
6300         my $cmd;
6301         if ($comp eq 'gzip') {
6302             $cmd = ['zcat', $readfrom];
6303         } elsif ($comp eq 'lzop') {
6304             $cmd = ['lzop', '-d', '-c', $readfrom];
6305         } else {
6306             die "unknown compression method '$comp'\n";
6307         }
6308         $add_pipe->($cmd);
6309     }
6310
6311     my $tmpdir = "/var/tmp/vzdumptmp$$";
6312     rmtree $tmpdir;
6313
6314     # disable interrupts (always do cleanups)
6315     local $SIG{INT} =
6316         local $SIG{TERM} =
6317         local $SIG{QUIT} =
6318         local $SIG{HUP} = sub { warn "got interrupt - ignored\n"; };
6319
6320     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
6321     POSIX::mkfifo($mapfifo, 0600);
6322     my $fifofh;
6323
6324     my $openfifo = sub {
6325         open($fifofh, '>', $mapfifo) || die $!;
6326     };
6327
6328     $add_pipe->(['vma', 'extract', '-v', '-r', $mapfifo, $readfrom, $tmpdir]);
6329
6330     my $oldtimeout;
6331     my $timeout = 5;
6332
6333     my $devinfo = {};
6334
6335     my $rpcenv = PVE::RPCEnvironment::get();
6336
6337     my $conffile = PVE::QemuConfig->config_file($vmid);
6338     my $tmpfn = "$conffile.$$.tmp";
6339
6340     # Note: $oldconf is undef if VM does not exists
6341     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
6342     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
6343
6344     my %storage_limits;
6345
6346     my $print_devmap = sub {
6347         my $virtdev_hash = {};
6348
6349         my $cfgfn = "$tmpdir/qemu-server.conf";
6350
6351         # we can read the config - that is already extracted
6352         my $fh = IO::File->new($cfgfn, "r") ||
6353             "unable to read qemu-server.conf - $!\n";
6354
6355         my $fwcfgfn = "$tmpdir/qemu-server.fw";
6356         if (-f $fwcfgfn) {
6357             my $pve_firewall_dir = '/etc/pve/firewall';
6358             mkdir $pve_firewall_dir; # make sure the dir exists
6359             PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
6360         }
6361
6362         while (defined(my $line = <$fh>)) {
6363             if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
6364                 my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
6365                 die "archive does not contain data for drive '$virtdev'\n"
6366                     if !$devinfo->{$devname};
6367                 if (defined($opts->{storage})) {
6368                     $storeid = $opts->{storage} || 'local';
6369                 } elsif (!$storeid) {
6370                     $storeid = 'local';
6371                 }
6372                 $format = 'raw' if !$format;
6373                 $devinfo->{$devname}->{devname} = $devname;
6374                 $devinfo->{$devname}->{virtdev} = $virtdev;
6375                 $devinfo->{$devname}->{format} = $format;
6376                 $devinfo->{$devname}->{storeid} = $storeid;
6377
6378                 # check permission on storage
6379                 my $pool = $opts->{pool}; # todo: do we need that?
6380                 if ($user ne 'root@pam') {
6381                     $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
6382                 }
6383
6384                 $storage_limits{$storeid} = $bwlimit;
6385
6386                 $virtdev_hash->{$virtdev} = $devinfo->{$devname};
6387             } elsif ($line =~ m/^((?:ide|sata|scsi)\d+):\s*(.*)\s*$/) {
6388                 my $virtdev = $1;
6389                 my $drive = parse_drive($virtdev, $2);
6390                 if (drive_is_cloudinit($drive)) {
6391                     my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6392                     my $scfg = PVE::Storage::storage_config($cfg, $storeid);
6393                     my $format = qemu_img_format($scfg, $volname); # has 'raw' fallback
6394
6395                     my $d = {
6396                         format => $format,
6397                         storeid => $opts->{storage} // $storeid,
6398                         size => PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE,
6399                         file => $drive->{file}, # to make drive_is_cloudinit check possible
6400                         name => "vm-$vmid-cloudinit",
6401                         is_cloudinit => 1,
6402                     };
6403                     $virtdev_hash->{$virtdev} = $d;
6404                 }
6405             }
6406         }
6407
6408         foreach my $key (keys %storage_limits) {
6409             my $limit = PVE::Storage::get_bandwidth_limit('restore', [$key], $bwlimit);
6410             next if !$limit;
6411             print STDERR "rate limit for storage $key: $limit KiB/s\n";
6412             $storage_limits{$key} = $limit * 1024;
6413         }
6414
6415         foreach my $devname (keys %$devinfo) {
6416             die "found no device mapping information for device '$devname'\n"
6417                 if !$devinfo->{$devname}->{virtdev};
6418         }
6419
6420         # create empty/temp config
6421         if ($oldconf) {
6422             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
6423             foreach_drive($oldconf, sub {
6424                 my ($ds, $drive) = @_;
6425
6426                 return if drive_is_cdrom($drive, 1);
6427
6428                 my $volid = $drive->{file};
6429                 return if !$volid || $volid =~ m|^/|;
6430
6431                 my ($path, $owner) = PVE::Storage::path($cfg, $volid);
6432                 return if !$path || !$owner || ($owner != $vmid);
6433
6434                 # Note: only delete disk we want to restore
6435                 # other volumes will become unused
6436                 if ($virtdev_hash->{$ds}) {
6437                     eval { PVE::Storage::vdisk_free($cfg, $volid); };
6438                     if (my $err = $@) {
6439                         warn $err;
6440                     }
6441                 }
6442             });
6443
6444             # delete vmstate files, after the restore we have no snapshots anymore
6445             foreach my $snapname (keys %{$oldconf->{snapshots}}) {
6446                 my $snap = $oldconf->{snapshots}->{$snapname};
6447                 if ($snap->{vmstate}) {
6448                     eval { PVE::Storage::vdisk_free($cfg, $snap->{vmstate}); };
6449                     if (my $err = $@) {
6450                         warn $err;
6451                     }
6452                 }
6453             }
6454         }
6455
6456         my $map = {};
6457         foreach my $virtdev (sort keys %$virtdev_hash) {
6458             my $d = $virtdev_hash->{$virtdev};
6459             my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
6460             my $storeid = $d->{storeid};
6461             my $scfg = PVE::Storage::storage_config($cfg, $storeid);
6462
6463             my $map_opts = '';
6464             if (my $limit = $storage_limits{$storeid}) {
6465                 $map_opts .= "throttling.bps=$limit:throttling.group=$storeid:";
6466             }
6467
6468             # test if requested format is supported
6469             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($cfg, $storeid);
6470             my $supported = grep { $_ eq $d->{format} } @$validFormats;
6471             $d->{format} = $defFormat if !$supported;
6472
6473             my $name;
6474             if ($d->{is_cloudinit}) {
6475                 $name = $d->{name};
6476                 $name .= ".$d->{format}" if $d->{format} ne 'raw';
6477             }
6478
6479             my $volid = PVE::Storage::vdisk_alloc($cfg, $storeid, $vmid, $d->{format}, $name, $alloc_size);
6480             print STDERR "new volume ID is '$volid'\n";
6481             $d->{volid} = $volid;
6482
6483             PVE::Storage::activate_volumes($cfg, [$volid]);
6484
6485             my $write_zeros = 1;
6486             if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
6487                 $write_zeros = 0;
6488             }
6489
6490             if (!$d->{is_cloudinit}) {
6491                 my $path = PVE::Storage::path($cfg, $volid);
6492
6493                 print $fifofh "${map_opts}format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
6494
6495                 print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
6496             }
6497             $map->{$virtdev} = $volid;
6498         }
6499
6500         $fh->seek(0, 0) || die "seek failed - $!\n";
6501
6502         my $outfd = new IO::File ($tmpfn, "w") ||
6503             die "unable to write config for VM $vmid\n";
6504
6505         my $cookie = { netcount => 0 };
6506         while (defined(my $line = <$fh>)) {
6507             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
6508         }
6509
6510         $fh->close();
6511         $outfd->close();
6512     };
6513
6514     eval {
6515         # enable interrupts
6516         local $SIG{INT} =
6517             local $SIG{TERM} =
6518             local $SIG{QUIT} =
6519             local $SIG{HUP} =
6520             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6521         local $SIG{ALRM} = sub { die "got timeout\n"; };
6522
6523         $oldtimeout = alarm($timeout);
6524
6525         my $parser = sub {
6526             my $line = shift;
6527
6528             print "$line\n";
6529
6530             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
6531                 my ($dev_id, $size, $devname) = ($1, $2, $3);
6532                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
6533             } elsif ($line =~ m/^CTIME: /) {
6534                 # we correctly received the vma config, so we can disable
6535                 # the timeout now for disk allocation (set to 10 minutes, so
6536                 # that we always timeout if something goes wrong)
6537                 alarm(600);
6538                 &$print_devmap();
6539                 print $fifofh "done\n";
6540                 my $tmp = $oldtimeout || 0;
6541                 $oldtimeout = undef;
6542                 alarm($tmp);
6543                 close($fifofh);
6544             }
6545         };
6546
6547         print "restore vma archive: $dbg_cmdstring\n";
6548         run_command($commands, input => $input, outfunc => $parser, afterfork => $openfifo);
6549     };
6550     my $err = $@;
6551
6552     alarm($oldtimeout) if $oldtimeout;
6553
6554     my $vollist = [];
6555     foreach my $devname (keys %$devinfo) {
6556         my $volid = $devinfo->{$devname}->{volid};
6557         push @$vollist, $volid if $volid;
6558     }
6559
6560     PVE::Storage::deactivate_volumes($cfg, $vollist);
6561
6562     unlink $mapfifo;
6563
6564     if ($err) {
6565         rmtree $tmpdir;
6566         unlink $tmpfn;
6567
6568         foreach my $devname (keys %$devinfo) {
6569             my $volid = $devinfo->{$devname}->{volid};
6570             next if !$volid;
6571             eval {
6572                 if ($volid =~ m|^/|) {
6573                     unlink $volid || die 'unlink failed\n';
6574                 } else {
6575                     PVE::Storage::vdisk_free($cfg, $volid);
6576                 }
6577                 print STDERR "temporary volume '$volid' sucessfuly removed\n";
6578             };
6579             print STDERR "unable to cleanup '$volid' - $@" if $@;
6580         }
6581         die $err;
6582     }
6583
6584     rmtree $tmpdir;
6585
6586     rename($tmpfn, $conffile) ||
6587         die "unable to commit configuration file '$conffile'\n";
6588
6589     PVE::Cluster::cfs_update(); # make sure we read new file
6590
6591     eval { rescan($vmid, 1); };
6592     warn $@ if $@;
6593 }
6594
6595 sub restore_tar_archive {
6596     my ($archive, $vmid, $user, $opts) = @_;
6597
6598     if ($archive ne '-') {
6599         my $firstfile = tar_archive_read_firstfile($archive);
6600         die "ERROR: file '$archive' dos not lock like a QemuServer vzdump backup\n"
6601             if $firstfile ne 'qemu-server.conf';
6602     }
6603
6604     my $storecfg = PVE::Storage::config();
6605
6606     # avoid zombie disks when restoring over an existing VM -> cleanup first
6607     # pass keep_empty_config=1 to keep the config (thus VMID) reserved for us
6608     # skiplock=1 because qmrestore has set the 'create' lock itself already
6609     my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
6610     destroy_vm($storecfg, $vmid, 1, { lock => 'restore' }) if -f $vmcfgfn;
6611
6612     my $tocmd = "/usr/lib/qemu-server/qmextract";
6613
6614     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
6615     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
6616     $tocmd .= ' --prealloc' if $opts->{prealloc};
6617     $tocmd .= ' --info' if $opts->{info};
6618
6619     # tar option "xf" does not autodetect compression when read from STDIN,
6620     # so we pipe to zcat
6621     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
6622         PVE::Tools::shellquote("--to-command=$tocmd");
6623
6624     my $tmpdir = "/var/tmp/vzdumptmp$$";
6625     mkpath $tmpdir;
6626
6627     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
6628     local $ENV{VZDUMP_VMID} = $vmid;
6629     local $ENV{VZDUMP_USER} = $user;
6630
6631     my $conffile = PVE::QemuConfig->config_file($vmid);
6632     my $tmpfn = "$conffile.$$.tmp";
6633
6634     # disable interrupts (always do cleanups)
6635     local $SIG{INT} =
6636         local $SIG{TERM} =
6637         local $SIG{QUIT} =
6638         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
6639
6640     eval {
6641         # enable interrupts
6642         local $SIG{INT} =
6643             local $SIG{TERM} =
6644             local $SIG{QUIT} =
6645             local $SIG{HUP} =
6646             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6647
6648         if ($archive eq '-') {
6649             print "extracting archive from STDIN\n";
6650             run_command($cmd, input => "<&STDIN");
6651         } else {
6652             print "extracting archive '$archive'\n";
6653             run_command($cmd);
6654         }
6655
6656         return if $opts->{info};
6657
6658         # read new mapping
6659         my $map = {};
6660         my $statfile = "$tmpdir/qmrestore.stat";
6661         if (my $fd = IO::File->new($statfile, "r")) {
6662             while (defined (my $line = <$fd>)) {
6663                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
6664                     $map->{$1} = $2 if $1;
6665                 } else {
6666                     print STDERR "unable to parse line in statfile - $line\n";
6667                 }
6668             }
6669             $fd->close();
6670         }
6671
6672         my $confsrc = "$tmpdir/qemu-server.conf";
6673
6674         my $srcfd = new IO::File($confsrc, "r") ||
6675             die "unable to open file '$confsrc'\n";
6676
6677         my $outfd = new IO::File ($tmpfn, "w") ||
6678             die "unable to write config for VM $vmid\n";
6679
6680         my $cookie = { netcount => 0 };
6681         while (defined (my $line = <$srcfd>)) {
6682             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
6683         }
6684
6685         $srcfd->close();
6686         $outfd->close();
6687     };
6688     if (my $err = $@) {
6689         unlink $tmpfn;
6690         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
6691         die $err;
6692     }
6693
6694     rmtree $tmpdir;
6695
6696     rename $tmpfn, $conffile ||
6697         die "unable to commit configuration file '$conffile'\n";
6698
6699     PVE::Cluster::cfs_update(); # make sure we read new file
6700
6701     eval { rescan($vmid, 1); };
6702     warn $@ if $@;
6703 };
6704
6705 sub foreach_storage_used_by_vm {
6706     my ($conf, $func) = @_;
6707
6708     my $sidhash = {};
6709
6710     foreach_drive($conf, sub {
6711         my ($ds, $drive) = @_;
6712         return if drive_is_cdrom($drive);
6713
6714         my $volid = $drive->{file};
6715
6716         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
6717         $sidhash->{$sid} = $sid if $sid;
6718     });
6719
6720     foreach my $sid (sort keys %$sidhash) {
6721         &$func($sid);
6722     }
6723 }
6724
6725 sub do_snapshots_with_qemu {
6726     my ($storecfg, $volid) = @_;
6727
6728     my $storage_name = PVE::Storage::parse_volume_id($volid);
6729     my $scfg = $storecfg->{ids}->{$storage_name};
6730
6731     if ($qemu_snap_storage->{$scfg->{type}} && !$scfg->{krbd}){
6732         return 1;
6733     }
6734
6735     if ($volid =~ m/\.(qcow2|qed)$/){
6736         return 1;
6737     }
6738
6739     return undef;
6740 }
6741
6742 sub qga_check_running {
6743     my ($vmid, $nowarn) = @_;
6744
6745     eval { vm_mon_cmd($vmid, "guest-ping", timeout => 3); };
6746     if ($@) {
6747         warn "Qemu Guest Agent is not running - $@" if !$nowarn;
6748         return 0;
6749     }
6750     return 1;
6751 }
6752
6753 sub template_create {
6754     my ($vmid, $conf, $disk) = @_;
6755
6756     my $storecfg = PVE::Storage::config();
6757
6758     foreach_drive($conf, sub {
6759         my ($ds, $drive) = @_;
6760
6761         return if drive_is_cdrom($drive);
6762         return if $disk && $ds ne $disk;
6763
6764         my $volid = $drive->{file};
6765         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
6766
6767         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
6768         $drive->{file} = $voliddst;
6769         $conf->{$ds} = print_drive($vmid, $drive);
6770         PVE::QemuConfig->write_config($vmid, $conf);
6771     });
6772 }
6773
6774 sub convert_iscsi_path {
6775     my ($path) = @_;
6776
6777     if ($path =~ m|^iscsi://([^/]+)/([^/]+)/(.+)$|) {
6778         my $portal = $1;
6779         my $target = $2;
6780         my $lun = $3;
6781
6782         my $initiator_name = get_initiator_name();
6783
6784         return "file.driver=iscsi,file.transport=tcp,file.initiator-name=$initiator_name,".
6785                "file.portal=$portal,file.target=$target,file.lun=$lun,driver=raw";
6786     }
6787
6788     die "cannot convert iscsi path '$path', unkown format\n";
6789 }
6790
6791 sub qemu_img_convert {
6792     my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized) = @_;
6793
6794     my $storecfg = PVE::Storage::config();
6795     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
6796     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
6797
6798     die "destination '$dst_volid' is not a valid volid form qemu-img convert\n" if !$dst_storeid;
6799
6800     my $cachemode;
6801     my $src_path;
6802     my $src_is_iscsi = 0;
6803     my $src_format = 'raw';
6804
6805     if ($src_storeid) {
6806         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
6807         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
6808         $src_format = qemu_img_format($src_scfg, $src_volname);
6809         $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
6810         $src_is_iscsi = ($src_path =~ m|^iscsi://|);
6811         $cachemode = 'none' if $src_scfg->{type} eq 'zfspool';
6812     } elsif (-f $src_volid) {
6813         $src_path = $src_volid;
6814         if ($src_path =~ m/\.($QEMU_FORMAT_RE)$/) {
6815             $src_format = $1;
6816         }
6817     }
6818
6819     die "source '$src_volid' is not a valid volid nor path for qemu-img convert\n" if !$src_path;
6820
6821     my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6822     my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
6823     my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6824     my $dst_is_iscsi = ($dst_path =~ m|^iscsi://|);
6825
6826     my $cmd = [];
6827     push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
6828     push @$cmd, '-l', "snapshot.name=$snapname" if($snapname && $src_format eq "qcow2");
6829     push @$cmd, '-t', 'none' if $dst_scfg->{type} eq 'zfspool';
6830     push @$cmd, '-T', $cachemode if defined($cachemode);
6831
6832     if ($src_is_iscsi) {
6833         push @$cmd, '--image-opts';
6834         $src_path = convert_iscsi_path($src_path);
6835     } else {
6836         push @$cmd, '-f', $src_format;
6837     }
6838
6839     if ($dst_is_iscsi) {
6840         push @$cmd, '--target-image-opts';
6841         $dst_path = convert_iscsi_path($dst_path);
6842     } else {
6843         push @$cmd, '-O', $dst_format;
6844     }
6845
6846     push @$cmd, $src_path;
6847
6848     if (!$dst_is_iscsi && $is_zero_initialized) {
6849         push @$cmd, "zeroinit:$dst_path";
6850     } else {
6851         push @$cmd, $dst_path;
6852     }
6853
6854     my $parser = sub {
6855         my $line = shift;
6856         if($line =~ m/\((\S+)\/100\%\)/){
6857             my $percent = $1;
6858             my $transferred = int($size * $percent / 100);
6859             my $remaining = $size - $transferred;
6860
6861             print "transferred: $transferred bytes remaining: $remaining bytes total: $size bytes progression: $percent %\n";
6862         }
6863
6864     };
6865
6866     eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
6867     my $err = $@;
6868     die "copy failed: $err" if $err;
6869 }
6870
6871 sub qemu_img_format {
6872     my ($scfg, $volname) = @_;
6873
6874     if ($scfg->{path} && $volname =~ m/\.($QEMU_FORMAT_RE)$/) {
6875         return $1;
6876     } else {
6877         return "raw";
6878     }
6879 }
6880
6881 sub qemu_drive_mirror {
6882     my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $skipcomplete, $qga, $bwlimit) = @_;
6883
6884     $jobs = {} if !$jobs;
6885
6886     my $qemu_target;
6887     my $format;
6888     $jobs->{"drive-$drive"} = {};
6889
6890     if ($dst_volid =~ /^nbd:/) {
6891         $qemu_target = $dst_volid;
6892         $format = "nbd";
6893     } else {
6894         my $storecfg = PVE::Storage::config();
6895         my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
6896
6897         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6898
6899         $format = qemu_img_format($dst_scfg, $dst_volname);
6900
6901         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6902
6903         $qemu_target = $is_zero_initialized ? "zeroinit:$dst_path" : $dst_path;
6904     }
6905
6906     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $qemu_target };
6907     $opts->{format} = $format if $format;
6908
6909     if (defined($bwlimit)) {
6910         $opts->{speed} = $bwlimit * 1024;
6911         print "drive mirror is starting for drive-$drive with bandwidth limit: ${bwlimit} KB/s\n";
6912     } else {
6913         print "drive mirror is starting for drive-$drive\n";
6914     }
6915
6916     # if a job already runs for this device we get an error, catch it for cleanup
6917     eval { vm_mon_cmd($vmid, "drive-mirror", %$opts); };
6918     if (my $err = $@) {
6919         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6920         warn "$@\n" if $@;
6921         die "mirroring error: $err\n";
6922     }
6923
6924     qemu_drive_mirror_monitor ($vmid, $vmiddst, $jobs, $skipcomplete, $qga);
6925 }
6926
6927 sub qemu_drive_mirror_monitor {
6928     my ($vmid, $vmiddst, $jobs, $skipcomplete, $qga) = @_;
6929
6930     eval {
6931         my $err_complete = 0;
6932
6933         while (1) {
6934             die "storage migration timed out\n" if $err_complete > 300;
6935
6936             my $stats = vm_mon_cmd($vmid, "query-block-jobs");
6937
6938             my $running_mirror_jobs = {};
6939             foreach my $stat (@$stats) {
6940                 next if $stat->{type} ne 'mirror';
6941                 $running_mirror_jobs->{$stat->{device}} = $stat;
6942             }
6943
6944             my $readycounter = 0;
6945
6946             foreach my $job (keys %$jobs) {
6947
6948                 if(defined($jobs->{$job}->{complete}) && !defined($running_mirror_jobs->{$job})) {
6949                     print "$job : finished\n";
6950                     delete $jobs->{$job};
6951                     next;
6952                 }
6953
6954                 die "$job: mirroring has been cancelled\n" if !defined($running_mirror_jobs->{$job});
6955
6956                 my $busy = $running_mirror_jobs->{$job}->{busy};
6957                 my $ready = $running_mirror_jobs->{$job}->{ready};
6958                 if (my $total = $running_mirror_jobs->{$job}->{len}) {
6959                     my $transferred = $running_mirror_jobs->{$job}->{offset} || 0;
6960                     my $remaining = $total - $transferred;
6961                     my $percent = sprintf "%.2f", ($transferred * 100 / $total);
6962
6963                     print "$job: transferred: $transferred bytes remaining: $remaining bytes total: $total bytes progression: $percent % busy: $busy ready: $ready \n";
6964                 }
6965
6966                 $readycounter++ if $running_mirror_jobs->{$job}->{ready};
6967             }
6968
6969             last if scalar(keys %$jobs) == 0;
6970
6971             if ($readycounter == scalar(keys %$jobs)) {
6972                 print "all mirroring jobs are ready \n";
6973                 last if $skipcomplete; #do the complete later
6974
6975                 if ($vmiddst && $vmiddst != $vmid) {
6976                     my $agent_running = $qga && qga_check_running($vmid);
6977                     if ($agent_running) {
6978                         print "freeze filesystem\n";
6979                         eval { PVE::QemuServer::vm_mon_cmd($vmid, "guest-fsfreeze-freeze"); };
6980                     } else {
6981                         print "suspend vm\n";
6982                         eval { PVE::QemuServer::vm_suspend($vmid, 1); };
6983                     }
6984
6985                     # if we clone a disk for a new target vm, we don't switch the disk
6986                     PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs);
6987
6988                     if ($agent_running) {
6989                         print "unfreeze filesystem\n";
6990                         eval { PVE::QemuServer::vm_mon_cmd($vmid, "guest-fsfreeze-thaw"); };
6991                     } else {
6992                         print "resume vm\n";
6993                         eval {  PVE::QemuServer::vm_resume($vmid, 1, 1); };
6994                     }
6995
6996                     last;
6997                 } else {
6998
6999                     foreach my $job (keys %$jobs) {
7000                         # try to switch the disk if source and destination are on the same guest
7001                         print "$job: Completing block job...\n";
7002
7003                         eval { vm_mon_cmd($vmid, "block-job-complete", device => $job) };
7004                         if ($@ =~ m/cannot be completed/) {
7005                             print "$job: Block job cannot be completed, try again.\n";
7006                             $err_complete++;
7007                         }else {
7008                             print "$job: Completed successfully.\n";
7009                             $jobs->{$job}->{complete} = 1;
7010                         }
7011                     }
7012                 }
7013             }
7014             sleep 1;
7015         }
7016     };
7017     my $err = $@;
7018
7019     if ($err) {
7020         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
7021         die "mirroring error: $err";
7022     }
7023
7024 }
7025
7026 sub qemu_blockjobs_cancel {
7027     my ($vmid, $jobs) = @_;
7028
7029     foreach my $job (keys %$jobs) {
7030         print "$job: Cancelling block job\n";
7031         eval { vm_mon_cmd($vmid, "block-job-cancel", device => $job); };
7032         $jobs->{$job}->{cancel} = 1;
7033     }
7034
7035     while (1) {
7036         my $stats = vm_mon_cmd($vmid, "query-block-jobs");
7037
7038         my $running_jobs = {};
7039         foreach my $stat (@$stats) {
7040             $running_jobs->{$stat->{device}} = $stat;
7041         }
7042
7043         foreach my $job (keys %$jobs) {
7044
7045             if (defined($jobs->{$job}->{cancel}) && !defined($running_jobs->{$job})) {
7046                 print "$job: Done.\n";
7047                 delete $jobs->{$job};
7048             }
7049         }
7050
7051         last if scalar(keys %$jobs) == 0;
7052
7053         sleep 1;
7054     }
7055 }
7056
7057 sub clone_disk {
7058     my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
7059         $newvmid, $storage, $format, $full, $newvollist, $jobs, $skipcomplete, $qga, $bwlimit) = @_;
7060
7061     my $newvolid;
7062
7063     if (!$full) {
7064         print "create linked clone of drive $drivename ($drive->{file})\n";
7065         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
7066         push @$newvollist, $newvolid;
7067     } else {
7068
7069         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
7070         $storeid = $storage if $storage;
7071
7072         my $dst_format = resolve_dst_disk_format($storecfg, $storeid, $volname, $format);
7073         my ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 3);
7074
7075         print "create full clone of drive $drivename ($drive->{file})\n";
7076         my $name = undef;
7077         if (drive_is_cloudinit($drive)) {
7078             $name = "vm-$newvmid-cloudinit";
7079             $name .= ".$dst_format" if $dst_format ne 'raw';
7080             $snapname = undef;
7081             $size = PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE;
7082         }
7083         $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $newvmid, $dst_format, $name, ($size/1024));
7084         push @$newvollist, $newvolid;
7085
7086         PVE::Storage::activate_volumes($storecfg, [$newvolid]);
7087
7088         if (drive_is_cloudinit($drive)) {
7089             goto no_data_clone;
7090         }
7091
7092         my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
7093         if (!$running || $snapname) {
7094             # TODO: handle bwlimits
7095             qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit);
7096         } else {
7097
7098             my $kvmver = get_running_qemu_version ($vmid);
7099             if (!qemu_machine_feature_enabled (undef, $kvmver, 2, 7)) {
7100                 die "drive-mirror with iothread requires qemu version 2.7 or higher\n"
7101                     if $drive->{iothread};
7102             }
7103
7104             qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid, $sparseinit, $jobs, $skipcomplete, $qga, $bwlimit);
7105         }
7106     }
7107
7108 no_data_clone:
7109     my ($size) = PVE::Storage::volume_size_info($storecfg, $newvolid, 3);
7110
7111     my $disk = $drive;
7112     $disk->{format} = undef;
7113     $disk->{file} = $newvolid;
7114     $disk->{size} = $size;
7115
7116     return $disk;
7117 }
7118
7119 # this only works if VM is running
7120 sub get_current_qemu_machine {
7121     my ($vmid) = @_;
7122
7123     my $cmd = { execute => 'query-machines', arguments => {} };
7124     my $res = vm_qmp_command($vmid, $cmd);
7125
7126     my ($current, $default);
7127     foreach my $e (@$res) {
7128         $default = $e->{name} if $e->{'is-default'};
7129         $current = $e->{name} if $e->{'is-current'};
7130     }
7131
7132     # fallback to the default machine if current is not supported by qemu
7133     return $current || $default || 'pc';
7134 }
7135
7136 sub get_running_qemu_version {
7137     my ($vmid) = @_;
7138     my $cmd = { execute => 'query-version', arguments => {} };
7139     my $res = vm_qmp_command($vmid, $cmd);
7140     return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
7141 }
7142
7143 sub qemu_machine_feature_enabled {
7144     my ($machine, $kvmver, $version_major, $version_minor) = @_;
7145
7146     my $current_major;
7147     my $current_minor;
7148
7149     if ($machine && $machine =~ m/^((?:pc(-i440fx|-q35)?|virt)-(\d+)\.(\d+))/) {
7150
7151         $current_major = $3;
7152         $current_minor = $4;
7153
7154     } elsif ($kvmver =~ m/^(\d+)\.(\d+)/) {
7155
7156         $current_major = $1;
7157         $current_minor = $2;
7158     }
7159
7160     return 1 if version_cmp($current_major, $version_major, $current_minor, $version_minor) >= 0;
7161 }
7162
7163 # gets in pairs the versions you want to compares, i.e.:
7164 # ($a-major, $b-major, $a-minor, $b-minor, $a-extra, $b-extra, ...)
7165 # returns 0 if same, -1 if $a is older than $b, +1 if $a is newer than $b
7166 sub version_cmp {
7167     my @versions = @_;
7168
7169     my $size = scalar(@versions);
7170
7171     return 0 if $size == 0;
7172     die "cannot compare odd count of versions" if $size & 1;
7173
7174     for (my $i = 0; $i < $size; $i += 2) {
7175         my ($a, $b) = splice(@versions, 0, 2);
7176         $a //= 0;
7177         $b //= 0;
7178
7179         return 1 if $a > $b;
7180         return -1 if $a < $b;
7181     }
7182     return 0;
7183 }
7184
7185 # dies if a) VM not running or not exisiting b) Version query failed
7186 # So, any defined return value is valid, any invalid state can be caught by eval
7187 sub runs_at_least_qemu_version {
7188     my ($vmid, $major, $minor, $extra) = @_;
7189
7190     my $v = vm_qmp_command($vmid, { execute => 'query-version' });
7191     die "could not query currently running version for VM $vmid\n" if !defined($v);
7192     $v = $v->{qemu};
7193
7194     return version_cmp($v->{major}, $major, $v->{minor}, $minor, $v->{micro}, $extra) >= 0;
7195 }
7196
7197 sub qemu_machine_pxe {
7198     my ($vmid, $conf) = @_;
7199
7200     my $machine =  PVE::QemuServer::get_current_qemu_machine($vmid);
7201
7202     if ($conf->{machine} && $conf->{machine} =~ m/\.pxe$/) {
7203         $machine .= '.pxe';
7204     }
7205
7206     return $machine;
7207 }
7208
7209 sub qemu_use_old_bios_files {
7210     my ($machine_type) = @_;
7211
7212     return if !$machine_type;
7213
7214     my $use_old_bios_files = undef;
7215
7216     if ($machine_type =~ m/^(\S+)\.pxe$/) {
7217         $machine_type = $1;
7218         $use_old_bios_files = 1;
7219     } else {
7220         my $kvmver = kvm_user_version();
7221         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
7222         # load new efi bios files on migration. So this hack is required to allow
7223         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
7224         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
7225         $use_old_bios_files = !qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 4);
7226     }
7227
7228     return ($use_old_bios_files, $machine_type);
7229 }
7230
7231 sub create_efidisk($$$$$) {
7232     my ($storecfg, $storeid, $vmid, $fmt, $arch) = @_;
7233
7234     my (undef, $ovmf_vars) = get_ovmf_files($arch);
7235     die "EFI vars default image not found\n" if ! -f $ovmf_vars;
7236
7237     my $vars_size_b = -s $ovmf_vars;
7238     my $vars_size = PVE::Tools::convert_size($vars_size_b, 'b' => 'kb');
7239     my $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $fmt, undef, $vars_size);
7240     PVE::Storage::activate_volumes($storecfg, [$volid]);
7241
7242     qemu_img_convert($ovmf_vars, $volid, $vars_size_b, undef, 0);
7243
7244     return ($volid, $vars_size);
7245 }
7246
7247 sub vm_iothreads_list {
7248     my ($vmid) = @_;
7249
7250     my $res = vm_mon_cmd($vmid, 'query-iothreads');
7251
7252     my $iothreads = {};
7253     foreach my $iothread (@$res) {
7254         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
7255     }
7256
7257     return $iothreads;
7258 }
7259
7260 sub scsihw_infos {
7261     my ($conf, $drive) = @_;
7262
7263     my $maxdev = 0;
7264
7265     if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)) {
7266         $maxdev = 7;
7267     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
7268         $maxdev = 1;
7269     } else {
7270         $maxdev = 256;
7271     }
7272
7273     my $controller = int($drive->{index} / $maxdev);
7274     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single') ? "virtioscsi" : "scsihw";
7275
7276     return ($maxdev, $controller, $controller_prefix);
7277 }
7278
7279 sub add_hyperv_enlightenments {
7280     my ($cpuFlags, $winversion, $machine_type, $kvmver, $bios, $gpu_passthrough, $hv_vendor_id) = @_;
7281
7282     return if $winversion < 6;
7283     return if $bios && $bios eq 'ovmf' && $winversion < 8;
7284
7285     if ($gpu_passthrough || defined($hv_vendor_id)) {
7286         $hv_vendor_id //= 'proxmox';
7287         push @$cpuFlags , "hv_vendor_id=$hv_vendor_id";
7288     }
7289
7290     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
7291         push @$cpuFlags , 'hv_spinlocks=0x1fff';
7292         push @$cpuFlags , 'hv_vapic';
7293         push @$cpuFlags , 'hv_time';
7294     } else {
7295         push @$cpuFlags , 'hv_spinlocks=0xffff';
7296     }
7297
7298     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 6)) {
7299         push @$cpuFlags , 'hv_reset';
7300         push @$cpuFlags , 'hv_vpindex';
7301         push @$cpuFlags , 'hv_runtime';
7302     }
7303
7304     if ($winversion >= 7) {
7305         push @$cpuFlags , 'hv_relaxed';
7306
7307         if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 12)) {
7308             push @$cpuFlags , 'hv_synic';
7309             push @$cpuFlags , 'hv_stimer';
7310         }
7311
7312         if (qemu_machine_feature_enabled ($machine_type, $kvmver, 3, 1)) {
7313             push @$cpuFlags , 'hv_ipi';
7314         }
7315     }
7316 }
7317
7318 sub windows_version {
7319     my ($ostype) = @_;
7320
7321     return 0 if !$ostype;
7322
7323     my $winversion = 0;
7324
7325     if($ostype eq 'wxp' || $ostype eq 'w2k3' || $ostype eq 'w2k') {
7326         $winversion = 5;
7327     } elsif($ostype eq 'w2k8' || $ostype eq 'wvista') {
7328         $winversion = 6;
7329     } elsif ($ostype =~ m/^win(\d+)$/) {
7330         $winversion = $1;
7331     }
7332
7333     return $winversion;
7334 }
7335
7336 sub resolve_dst_disk_format {
7337         my ($storecfg, $storeid, $src_volname, $format) = @_;
7338         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
7339
7340         if (!$format) {
7341             # if no target format is specified, use the source disk format as hint
7342             if ($src_volname) {
7343                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
7344                 $format = qemu_img_format($scfg, $src_volname);
7345             } else {
7346                 return $defFormat;
7347             }
7348         }
7349
7350         # test if requested format is supported - else use default
7351         my $supported = grep { $_ eq $format } @$validFormats;
7352         $format = $defFormat if !$supported;
7353         return $format;
7354 }
7355
7356 sub resolve_first_disk {
7357     my $conf = shift;
7358     my @disks = PVE::QemuServer::valid_drive_names();
7359     my $firstdisk;
7360     foreach my $ds (reverse @disks) {
7361         next if !$conf->{$ds};
7362         my $disk = PVE::QemuServer::parse_drive($ds, $conf->{$ds});
7363         next if PVE::QemuServer::drive_is_cdrom($disk);
7364         $firstdisk = $ds;
7365     }
7366     return $firstdisk;
7367 }
7368
7369 sub generate_uuid {
7370     my ($uuid, $uuid_str);
7371     UUID::generate($uuid);
7372     UUID::unparse($uuid, $uuid_str);
7373     return $uuid_str;
7374 }
7375
7376 sub generate_smbios1_uuid {
7377     return "uuid=".generate_uuid();
7378 }
7379
7380 sub nbd_stop {
7381     my ($vmid) = @_;
7382
7383     vm_mon_cmd($vmid, 'nbd-server-stop');
7384 }
7385
7386 sub create_reboot_request {
7387     my ($vmid) = @_;
7388     open(my $fh, '>', "/run/qemu-server/$vmid.reboot")
7389         or die "failed to create reboot trigger file: $!\n";
7390     close($fh);
7391 }
7392
7393 sub clear_reboot_request {
7394     my ($vmid) = @_;
7395     my $path = "/run/qemu-server/$vmid.reboot";
7396     my $res = 0;
7397
7398     $res = unlink($path);
7399     die "could not remove reboot request for $vmid: $!"
7400         if !$res && $! != POSIX::ENOENT;
7401
7402     return $res;
7403 }
7404
7405 # bash completion helper
7406
7407 sub complete_backup_archives {
7408     my ($cmdname, $pname, $cvalue) = @_;
7409
7410     my $cfg = PVE::Storage::config();
7411
7412     my $storeid;
7413
7414     if ($cvalue =~ m/^([^:]+):/) {
7415         $storeid = $1;
7416     }
7417
7418     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
7419
7420     my $res = [];
7421     foreach my $id (keys %$data) {
7422         foreach my $item (@{$data->{$id}}) {
7423             next if $item->{format} !~ m/^vma\.(gz|lzo)$/;
7424             push @$res, $item->{volid} if defined($item->{volid});
7425         }
7426     }
7427
7428     return $res;
7429 }
7430
7431 my $complete_vmid_full = sub {
7432     my ($running) = @_;
7433
7434     my $idlist = vmstatus();
7435
7436     my $res = [];
7437
7438     foreach my $id (keys %$idlist) {
7439         my $d = $idlist->{$id};
7440         if (defined($running)) {
7441             next if $d->{template};
7442             next if $running && $d->{status} ne 'running';
7443             next if !$running && $d->{status} eq 'running';
7444         }
7445         push @$res, $id;
7446
7447     }
7448     return $res;
7449 };
7450
7451 sub complete_vmid {
7452     return &$complete_vmid_full();
7453 }
7454
7455 sub complete_vmid_stopped {
7456     return &$complete_vmid_full(0);
7457 }
7458
7459 sub complete_vmid_running {
7460     return &$complete_vmid_full(1);
7461 }
7462
7463 sub complete_storage {
7464
7465     my $cfg = PVE::Storage::config();
7466     my $ids = $cfg->{ids};
7467
7468     my $res = [];
7469     foreach my $sid (keys %$ids) {
7470         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
7471         next if !$ids->{$sid}->{content}->{images};
7472         push @$res, $sid;
7473     }
7474
7475     return $res;
7476 }
7477
7478 sub complete_migration_storage {
7479     my ($cmd, $param, $current_value, $all_args) = @_;
7480
7481     my $targetnode = @$all_args[1];
7482
7483     my $cfg = PVE::Storage::config();
7484     my $ids = $cfg->{ids};
7485
7486     my $res = [];
7487     foreach my $sid (keys %$ids) {
7488         next if !PVE::Storage::storage_check_enabled($cfg, $sid, $targetnode, 1);
7489         next if !$ids->{$sid}->{content}->{images};
7490         push @$res, $sid;
7491     }
7492
7493     return $res;
7494 }
7495
7496 1;