PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5
   6 use Cwd 'abs_path';
   7 use Digest::SHA;
   8 use Fcntl ':flock';
   9 use Fcntl;
  10 use File::Basename;
  11 use File::Copy qw(copy);
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use IO::Dir;
  16 use IO::File;
  17 use IO::Handle;
  18 use IO::Select;
  19 use IO::Socket::UNIX;
  20 use IPC::Open3;
  21 use JSON;
  22 use MIME::Base64;
  23 use POSIX;
  24 use Storable qw(dclone);
  25 use Time::HiRes qw(gettimeofday);
  26 use URI::Escape;
  27 use UUID;
  28
  29 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file cfs_lock_file);
  30 use PVE::DataCenterConfig;
  31 use PVE::Exception qw(raise raise_param_exc);
  32 use PVE::GuestHelpers;
  33 use PVE::INotify;
  34 use PVE::JSONSchema qw(get_standard_option);
  35 use PVE::ProcFSTools;
  36 use PVE::RPCEnvironment;
  37 use PVE::Storage;
  38 use PVE::SysFSTools;
  39 use PVE::Systemd;
  40 use PVE::Tools qw(run_command lock_file lock_file_full file_read_firstline dir_glob_foreach get_host_arch $IPV6RE);
  41
  42 use PVE::QMPClient;
  43 use PVE::QemuConfig;
  44 use PVE::QemuServer::Helpers qw(min_version);
  45 use PVE::QemuServer::Cloudinit;
  46 use PVE::QemuServer::Machine;
  47 use PVE::QemuServer::Memory;
  48 use PVE::QemuServer::Monitor qw(mon_cmd);
  49 use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr print_pcie_root_port);
  50 use PVE::QemuServer::USB qw(parse_usb_device);
  51
  52 my $EDK2_FW_BASE = '/usr/share/pve-edk2-firmware/';
  53 my $OVMF = {
  54     x86_64 => [
  55         "$EDK2_FW_BASE/OVMF_CODE.fd",
  56         "$EDK2_FW_BASE/OVMF_VARS.fd"
  57     ],
  58     aarch64 => [
  59         "$EDK2_FW_BASE/AAVMF_CODE.fd",
  60         "$EDK2_FW_BASE/AAVMF_VARS.fd"
  61     ],
  62 };
  63
  64 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  65
  66 my $QEMU_FORMAT_RE = qr/raw|cow|qcow|qcow2|qed|vmdk|cloop/;
  67
  68 # Note about locking: we use flock on the config file protect
  69 # against concurent actions.
  70 # Aditionaly, we have a 'lock' setting in the config file. This
  71 # can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
  72 # allowed when such lock is set. But you can ignore this kind of
  73 # lock with the --skiplock flag.
  74
  75 cfs_register_file('/qemu-server/',
  76                   \&parse_vm_config,
  77                   \&write_vm_config);
  78
  79 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
  80     description => "Some command save/restore state from this location.",
  81     type => 'string',
  82     maxLength => 128,
  83     optional => 1,
  84 });
  85
  86 PVE::JSONSchema::register_standard_option('pve-qm-image-format', {
  87     type => 'string',
  88     enum => [qw(raw cow qcow qed qcow2 vmdk cloop)],
  89     description => "The drive's backing file's data format.",
  90     optional => 1,
  91 });
  92
  93 PVE::JSONSchema::register_standard_option('pve-qemu-machine', {
  94         description => "Specifies the Qemu machine type.",
  95         type => 'string',
  96         pattern => '(pc|pc(-i440fx)?-\d+(\.\d+)+(\.pxe)?|q35|pc-q35-\d+(\.\d+)+(\.pxe)?|virt(?:-\d+(\.\d+)+)?)',
  97         maxLength => 40,
  98         optional => 1,
  99 });
 100
 101 #no warnings 'redefine';
 102
 103 sub cgroups_write {
 104    my ($controller, $vmid, $option, $value) = @_;
 105
 106    my $path = "/sys/fs/cgroup/$controller/qemu.slice/$vmid.scope/$option";
 107    PVE::ProcFSTools::write_proc_entry($path, $value);
 108
 109 }
 110
 111 my $nodename = PVE::INotify::nodename();
 112
 113 my $cpu_vendor_list = {
 114     # Intel CPUs
 115     486 => 'GenuineIntel',
 116     pentium => 'GenuineIntel',
 117     pentium2  => 'GenuineIntel',
 118     pentium3  => 'GenuineIntel',
 119     coreduo => 'GenuineIntel',
 120     core2duo => 'GenuineIntel',
 121     Conroe  => 'GenuineIntel',
 122     Penryn  => 'GenuineIntel',
 123     Nehalem  => 'GenuineIntel',
 124     'Nehalem-IBRS'  => 'GenuineIntel',
 125     Westmere => 'GenuineIntel',
 126     'Westmere-IBRS' => 'GenuineIntel',
 127     SandyBridge => 'GenuineIntel',
 128     'SandyBridge-IBRS' => 'GenuineIntel',
 129     IvyBridge => 'GenuineIntel',
 130     'IvyBridge-IBRS' => 'GenuineIntel',
 131     Haswell => 'GenuineIntel',
 132     'Haswell-IBRS' => 'GenuineIntel',
 133     'Haswell-noTSX' => 'GenuineIntel',
 134     'Haswell-noTSX-IBRS' => 'GenuineIntel',
 135     Broadwell => 'GenuineIntel',
 136     'Broadwell-IBRS' => 'GenuineIntel',
 137     'Broadwell-noTSX' => 'GenuineIntel',
 138     'Broadwell-noTSX-IBRS' => 'GenuineIntel',
 139     'Skylake-Client' => 'GenuineIntel',
 140     'Skylake-Client-IBRS' => 'GenuineIntel',
 141     'Skylake-Server' => 'GenuineIntel',
 142     'Skylake-Server-IBRS' => 'GenuineIntel',
 143     'Cascadelake-Server' => 'GenuineIntel',
 144     KnightsMill => 'GenuineIntel',
 145
 146
 147     # AMD CPUs
 148     athlon => 'AuthenticAMD',
 149     phenom  => 'AuthenticAMD',
 150     Opteron_G1  => 'AuthenticAMD',
 151     Opteron_G2  => 'AuthenticAMD',
 152     Opteron_G3  => 'AuthenticAMD',
 153     Opteron_G4  => 'AuthenticAMD',
 154     Opteron_G5  => 'AuthenticAMD',
 155     EPYC => 'AuthenticAMD',
 156     'EPYC-IBPB' => 'AuthenticAMD',
 157
 158     # generic types, use vendor from host node
 159     host => 'default',
 160     kvm32 => 'default',
 161     kvm64 => 'default',
 162     qemu32 => 'default',
 163     qemu64 => 'default',
 164     max => 'default',
 165 };
 166
 167 my @supported_cpu_flags = (
 168     'pcid',
 169     'spec-ctrl',
 170     'ibpb',
 171     'ssbd',
 172     'virt-ssbd',
 173     'amd-ssbd',
 174     'amd-no-ssb',
 175     'pdpe1gb',
 176     'md-clear',
 177     'hv-tlbflush',
 178     'hv-evmcs',
 179     'aes'
 180 );
 181 my $cpu_flag = qr/[+-](@{[join('|', @supported_cpu_flags)]})/;
 182
 183 my $cpu_fmt = {
 184     cputype => {
 185         description => "Emulated CPU type.",
 186         type => 'string',
 187         enum => [ sort { "\L$a" cmp "\L$b" } keys %$cpu_vendor_list ],
 188         default => 'kvm64',
 189         default_key => 1,
 190     },
 191     hidden => {
 192         description => "Do not identify as a KVM virtual machine.",
 193         type => 'boolean',
 194         optional => 1,
 195         default => 0
 196     },
 197     'hv-vendor-id' => {
 198         type => 'string',
 199         pattern => qr/[a-zA-Z0-9]{1,12}/,
 200         format_description => 'vendor-id',
 201         description => 'The Hyper-V vendor ID. Some drivers or programs inside Windows guests need a specific ID.',
 202         optional => 1,
 203     },
 204     flags => {
 205         description => "List of additional CPU flags separated by ';'."
 206                      . " Use '+FLAG' to enable, '-FLAG' to disable a flag."
 207                      . " Currently supported flags: @{[join(', ', @supported_cpu_flags)]}.",
 208         format_description => '+FLAG[;-FLAG...]',
 209         type => 'string',
 210         pattern => qr/$cpu_flag(;$cpu_flag)*/,
 211         optional => 1,
 212     },
 213 };
 214
 215 my $watchdog_fmt = {
 216     model => {
 217         default_key => 1,
 218         type => 'string',
 219         enum => [qw(i6300esb ib700)],
 220         description => "Watchdog type to emulate.",
 221         default => 'i6300esb',
 222         optional => 1,
 223     },
 224     action => {
 225         type => 'string',
 226         enum => [qw(reset shutdown poweroff pause debug none)],
 227         description => "The action to perform if after activation the guest fails to poll the watchdog in time.",
 228         optional => 1,
 229     },
 230 };
 231 PVE::JSONSchema::register_format('pve-qm-watchdog', $watchdog_fmt);
 232
 233 my $agent_fmt = {
 234     enabled => {
 235         description => "Enable/disable Qemu GuestAgent.",
 236         type => 'boolean',
 237         default => 0,
 238         default_key => 1,
 239     },
 240     fstrim_cloned_disks => {
 241         description => "Run fstrim after cloning/moving a disk.",
 242         type => 'boolean',
 243         optional => 1,
 244         default => 0
 245     },
 246     type => {
 247         description => "Select the agent type",
 248         type => 'string',
 249         default => 'virtio',
 250         optional => 1,
 251         enum => [qw(virtio isa)],
 252     },
 253 };
 254
 255 my $vga_fmt = {
 256     type => {
 257         description => "Select the VGA type.",
 258         type => 'string',
 259         default => 'std',
 260         optional => 1,
 261         default_key => 1,
 262         enum => [qw(cirrus qxl qxl2 qxl3 qxl4 none serial0 serial1 serial2 serial3 std virtio vmware)],
 263     },
 264     memory => {
 265         description => "Sets the VGA memory (in MiB). Has no effect with serial display.",
 266         type => 'integer',
 267         optional => 1,
 268         minimum => 4,
 269         maximum => 512,
 270     },
 271 };
 272
 273 my $ivshmem_fmt = {
 274     size => {
 275         type => 'integer',
 276         minimum => 1,
 277         description => "The size of the file in MB.",
 278     },
 279     name => {
 280         type => 'string',
 281         pattern => '[a-zA-Z0-9\-]+',
 282         optional => 1,
 283         format_description => 'string',
 284         description => "The name of the file. Will be prefixed with 'pve-shm-'. Default is the VMID. Will be deleted when the VM is stopped.",
 285     },
 286 };
 287
 288 my $audio_fmt = {
 289     device => {
 290         type => 'string',
 291         enum => [qw(ich9-intel-hda intel-hda AC97)],
 292         description =>  "Configure an audio device."
 293     },
 294     driver =>  {
 295         type => 'string',
 296         enum => ['spice'],
 297         default => 'spice',
 298         optional => 1,
 299         description => "Driver backend for the audio device."
 300     },
 301 };
 302
 303 my $spice_enhancements_fmt = {
 304     foldersharing => {
 305         type => 'boolean',
 306         optional => 1,
 307         default => '0',
 308         description =>  "Enable folder sharing via SPICE. Needs Spice-WebDAV daemon installed in the VM."
 309     },
 310     videostreaming =>  {
 311         type => 'string',
 312         enum => ['off', 'all', 'filter'],
 313         default => 'off',
 314         optional => 1,
 315         description => "Enable video streaming. Uses compression for detected video streams."
 316     },
 317 };
 318
 319 my $confdesc = {
 320     onboot => {
 321         optional => 1,
 322         type => 'boolean',
 323         description => "Specifies whether a VM will be started during system bootup.",
 324         default => 0,
 325     },
 326     autostart => {
 327         optional => 1,
 328         type => 'boolean',
 329         description => "Automatic restart after crash (currently ignored).",
 330         default => 0,
 331     },
 332     hotplug => {
 333         optional => 1,
 334         type => 'string', format => 'pve-hotplug-features',
 335         description => "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
 336         default => 'network,disk,usb',
 337     },
 338     reboot => {
 339         optional => 1,
 340         type => 'boolean',
 341         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 342         default => 1,
 343     },
 344     lock => {
 345         optional => 1,
 346         type => 'string',
 347         description => "Lock/unlock the VM.",
 348         enum => [qw(backup clone create migrate rollback snapshot snapshot-delete suspending suspended)],
 349     },
 350     cpulimit => {
 351         optional => 1,
 352         type => 'number',
 353         description => "Limit of CPU usage.",
 354         verbose_description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
 355         minimum => 0,
 356         maximum => 128,
 357         default => 0,
 358     },
 359     cpuunits => {
 360         optional => 1,
 361         type => 'integer',
 362         description => "CPU weight for a VM.",
 363         verbose_description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.",
 364         minimum => 2,
 365         maximum => 262144,
 366         default => 1024,
 367     },
 368     memory => {
 369         optional => 1,
 370         type => 'integer',
 371         description => "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
 372         minimum => 16,
 373         default => 512,
 374     },
 375     balloon => {
 376         optional => 1,
 377         type => 'integer',
 378         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 379         minimum => 0,
 380     },
 381     shares => {
 382         optional => 1,
 383         type => 'integer',
 384         description => "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning. Auto-ballooning is done by pvestatd.",
 385         minimum => 0,
 386         maximum => 50000,
 387         default => 1000,
 388     },
 389     keyboard => {
 390         optional => 1,
 391         type => 'string',
 392         description => "Keybord layout for vnc server. Default is read from the '/etc/pve/datacenter.cfg' configuration file.".
 393                        "It should not be necessary to set it.",
 394         enum => PVE::Tools::kvmkeymaplist(),
 395         default => undef,
 396     },
 397     name => {
 398         optional => 1,
 399         type => 'string', format => 'dns-name',
 400         description => "Set a name for the VM. Only used on the configuration web interface.",
 401     },
 402     scsihw => {
 403         optional => 1,
 404         type => 'string',
 405         description => "SCSI controller model",
 406         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 407         default => 'lsi',
 408     },
 409     description => {
 410         optional => 1,
 411         type => 'string',
 412         description => "Description for the VM. Only used on the configuration web interface. This is saved as comment inside the configuration file.",
 413     },
 414     ostype => {
 415         optional => 1,
 416         type => 'string',
 417         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 l24 l26 solaris)],
 418         description => "Specify guest operating system.",
 419         verbose_description => <<EODESC,
 420 Specify guest operating system. This is used to enable special
 421 optimization/features for specific operating systems:
 422
 423 [horizontal]
 424 other;; unspecified OS
 425 wxp;; Microsoft Windows XP
 426 w2k;; Microsoft Windows 2000
 427 w2k3;; Microsoft Windows 2003
 428 w2k8;; Microsoft Windows 2008
 429 wvista;; Microsoft Windows Vista
 430 win7;; Microsoft Windows 7
 431 win8;; Microsoft Windows 8/2012/2012r2
 432 win10;; Microsoft Windows 10/2016
 433 l24;; Linux 2.4 Kernel
 434 l26;; Linux 2.6 - 5.X Kernel
 435 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
 436 EODESC
 437     },
 438     boot => {
 439         optional => 1,
 440         type => 'string',
 441         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n).",
 442         pattern => '[acdn]{1,4}',
 443         default => 'cdn',
 444     },
 445     bootdisk => {
 446         optional => 1,
 447         type => 'string', format => 'pve-qm-bootdisk',
 448         description => "Enable booting from specified disk.",
 449         pattern => '(ide|sata|scsi|virtio)\d+',
 450     },
 451     smp => {
 452         optional => 1,
 453         type => 'integer',
 454         description => "The number of CPUs. Please use option -sockets instead.",
 455         minimum => 1,
 456         default => 1,
 457     },
 458     sockets => {
 459         optional => 1,
 460         type => 'integer',
 461         description => "The number of CPU sockets.",
 462         minimum => 1,
 463         default => 1,
 464     },
 465     cores => {
 466         optional => 1,
 467         type => 'integer',
 468         description => "The number of cores per socket.",
 469         minimum => 1,
 470         default => 1,
 471     },
 472     numa => {
 473         optional => 1,
 474         type => 'boolean',
 475         description => "Enable/disable NUMA.",
 476         default => 0,
 477     },
 478     hugepages => {
 479         optional => 1,
 480         type => 'string',
 481         description => "Enable/disable hugepages memory.",
 482         enum => [qw(any 2 1024)],
 483     },
 484     vcpus => {
 485         optional => 1,
 486         type => 'integer',
 487         description => "Number of hotplugged vcpus.",
 488         minimum => 1,
 489         default => 0,
 490     },
 491     acpi => {
 492         optional => 1,
 493         type => 'boolean',
 494         description => "Enable/disable ACPI.",
 495         default => 1,
 496     },
 497     agent => {
 498         optional => 1,
 499         description => "Enable/disable Qemu GuestAgent and its properties.",
 500         type => 'string',
 501         format => $agent_fmt,
 502     },
 503     kvm => {
 504         optional => 1,
 505         type => 'boolean',
 506         description => "Enable/disable KVM hardware virtualization.",
 507         default => 1,
 508     },
 509     tdf => {
 510         optional => 1,
 511         type => 'boolean',
 512         description => "Enable/disable time drift fix.",
 513         default => 0,
 514     },
 515     localtime => {
 516         optional => 1,
 517         type => 'boolean',
 518         description => "Set the real time clock to local time. This is enabled by default if ostype indicates a Microsoft OS.",
 519     },
 520     freeze => {
 521         optional => 1,
 522         type => 'boolean',
 523         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 524     },
 525     vga => {
 526         optional => 1,
 527         type => 'string', format => $vga_fmt,
 528         description => "Configure the VGA hardware.",
 529         verbose_description => "Configure the VGA Hardware. If you want to use ".
 530             "high resolution modes (>= 1280x1024x16) you may need to increase " .
 531             "the vga memory option. Since QEMU 2.9 the default VGA display type " .
 532             "is 'std' for all OS types besides some Windows versions (XP and " .
 533             "older) which use 'cirrus'. The 'qxl' option enables the SPICE " .
 534             "display server. For win* OS you can select how many independent " .
 535             "displays you want, Linux guests can add displays them self.\n".
 536             "You can also run without any graphic card, using a serial device as terminal.",
 537     },
 538     watchdog => {
 539         optional => 1,
 540         type => 'string', format => 'pve-qm-watchdog',
 541         description => "Create a virtual hardware watchdog device.",
 542         verbose_description => "Create a virtual hardware watchdog device. Once enabled" .
 543             " (by a guest action), the watchdog must be periodically polled " .
 544             "by an agent inside the guest or else the watchdog will reset " .
 545             "the guest (or execute the respective action specified)",
 546     },
 547     startdate => {
 548         optional => 1,
 549         type => 'string',
 550         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 551         description => "Set the initial date of the real time clock. Valid format for date are: 'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 552         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 553         default => 'now',
 554     },
 555     startup =>  get_standard_option('pve-startup-order'),
 556     template => {
 557         optional => 1,
 558         type => 'boolean',
 559         description => "Enable/disable Template.",
 560         default => 0,
 561     },
 562     args => {
 563         optional => 1,
 564         type => 'string',
 565         description => "Arbitrary arguments passed to kvm.",
 566         verbose_description => <<EODESCR,
 567 Arbitrary arguments passed to kvm, for example:
 568
 569 args: -no-reboot -no-hpet
 570
 571 NOTE: this option is for experts only.
 572 EODESCR
 573     },
 574     tablet => {
 575         optional => 1,
 576         type => 'boolean',
 577         default => 1,
 578         description => "Enable/disable the USB tablet device.",
 579         verbose_description => "Enable/disable the USB tablet device. This device is " .
 580             "usually needed to allow absolute mouse positioning with VNC. " .
 581             "Else the mouse runs out of sync with normal VNC clients. " .
 582             "If you're running lots of console-only guests on one host, " .
 583             "you may consider disabling this to save some context switches. " .
 584             "This is turned off by default if you use spice (-vga=qxl).",
 585     },
 586     migrate_speed => {
 587         optional => 1,
 588         type => 'integer',
 589         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 590         minimum => 0,
 591         default => 0,
 592     },
 593     migrate_downtime => {
 594         optional => 1,
 595         type => 'number',
 596         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 597         minimum => 0,
 598         default => 0.1,
 599     },
 600     cdrom => {
 601         optional => 1,
 602         type => 'string', format => 'pve-qm-ide',
 603         typetext => '<volume>',
 604         description => "This is an alias for option -ide2",
 605     },
 606     cpu => {
 607         optional => 1,
 608         description => "Emulated CPU type.",
 609         type => 'string',
 610         format => $cpu_fmt,
 611     },
 612     parent => get_standard_option('pve-snapshot-name', {
 613         optional => 1,
 614         description => "Parent snapshot name. This is used internally, and should not be modified.",
 615     }),
 616     snaptime => {
 617         optional => 1,
 618         description => "Timestamp for snapshots.",
 619         type => 'integer',
 620         minimum => 0,
 621     },
 622     vmstate => {
 623         optional => 1,
 624         type => 'string', format => 'pve-volume-id',
 625         description => "Reference to a volume which stores the VM state. This is used internally for snapshots.",
 626     },
 627     vmstatestorage => get_standard_option('pve-storage-id', {
 628         description => "Default storage for VM state volumes/files.",
 629         optional => 1,
 630     }),
 631     runningmachine => get_standard_option('pve-qemu-machine', {
 632         description => "Specifies the Qemu machine type of the running vm. This is used internally for snapshots.",
 633     }),
 634     machine => get_standard_option('pve-qemu-machine'),
 635     arch => {
 636         description => "Virtual processor architecture. Defaults to the host.",
 637         optional => 1,
 638         type => 'string',
 639         enum => [qw(x86_64 aarch64)],
 640     },
 641     smbios1 => {
 642         description => "Specify SMBIOS type 1 fields.",
 643         type => 'string', format => 'pve-qm-smbios1',
 644         maxLength => 512,
 645         optional => 1,
 646     },
 647     protection => {
 648         optional => 1,
 649         type => 'boolean',
 650         description => "Sets the protection flag of the VM. This will disable the remove VM and remove disk operations.",
 651         default => 0,
 652     },
 653     bios => {
 654         optional => 1,
 655         type => 'string',
 656         enum => [ qw(seabios ovmf) ],
 657         description => "Select BIOS implementation.",
 658         default => 'seabios',
 659     },
 660     vmgenid => {
 661         type => 'string',
 662         pattern => '(?:[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}|[01])',
 663         format_description => 'UUID',
 664         description => "Set VM Generation ID. Use '1' to autogenerate on create or update, pass '0' to disable explicitly.",
 665         verbose_description => "The VM generation ID (vmgenid) device exposes a".
 666             " 128-bit integer value identifier to the guest OS. This allows to".
 667             " notify the guest operating system when the virtual machine is".
 668             " executed with a different configuration (e.g. snapshot execution".
 669             " or creation from a template). The guest operating system notices".
 670             " the change, and is then able to react as appropriate by marking".
 671             " its copies of distributed databases as dirty, re-initializing its".
 672             " random number generator, etc.\n".
 673             "Note that auto-creation only works when done throug API/CLI create".
 674             " or update methods, but not when manually editing the config file.",
 675         default => "1 (autogenerated)",
 676         optional => 1,
 677     },
 678     hookscript => {
 679         type => 'string',
 680         format => 'pve-volume-id',
 681         optional => 1,
 682         description => "Script that will be executed during various steps in the vms lifetime.",
 683     },
 684     ivshmem => {
 685         type => 'string',
 686         format => $ivshmem_fmt,
 687         description => "Inter-VM shared memory. Useful for direct communication between VMs, or to the host.",
 688         optional => 1,
 689     },
 690     audio0 => {
 691         type => 'string',
 692         format => $audio_fmt,
 693         description => "Configure a audio device, useful in combination with QXL/Spice.",
 694         optional => 1
 695     },
 696     spice_enhancements => {
 697         type => 'string',
 698         format => $spice_enhancements_fmt,
 699         description => "Configure additional enhancements for SPICE.",
 700         optional => 1
 701     },
 702 };
 703
 704 my $cicustom_fmt = {
 705     meta => {
 706         type => 'string',
 707         optional => 1,
 708         description => 'Specify a custom file containing all meta data passed to the VM via cloud-init. This is provider specific meaning configdrive2 and nocloud differ.',
 709         format => 'pve-volume-id',
 710         format_description => 'volume',
 711     },
 712     network => {
 713         type => 'string',
 714         optional => 1,
 715         description => 'Specify a custom file containing all network data passed to the VM via cloud-init.',
 716         format => 'pve-volume-id',
 717         format_description => 'volume',
 718     },
 719     user => {
 720         type => 'string',
 721         optional => 1,
 722         description => 'Specify a custom file containing all user data passed to the VM via cloud-init.',
 723         format => 'pve-volume-id',
 724         format_description => 'volume',
 725     },
 726 };
 727 PVE::JSONSchema::register_format('pve-qm-cicustom', $cicustom_fmt);
 728
 729 my $confdesc_cloudinit = {
 730     citype => {
 731         optional => 1,
 732         type => 'string',
 733         description => 'Specifies the cloud-init configuration format. The default depends on the configured operating system type (`ostype`. We use the `nocloud` format for Linux, and `configdrive2` for windows.',
 734         enum => ['configdrive2', 'nocloud'],
 735     },
 736     ciuser => {
 737         optional => 1,
 738         type => 'string',
 739         description => "cloud-init: User name to change ssh keys and password for instead of the image's configured default user.",
 740     },
 741     cipassword => {
 742         optional => 1,
 743         type => 'string',
 744         description => 'cloud-init: Password to assign the user. Using this is generally not recommended. Use ssh keys instead. Also note that older cloud-init versions do not support hashed passwords.',
 745     },
 746     cicustom => {
 747         optional => 1,
 748         type => 'string',
 749         description => 'cloud-init: Specify custom files to replace the automatically generated ones at start.',
 750         format => 'pve-qm-cicustom',
 751     },
 752     searchdomain => {
 753         optional => 1,
 754         type => 'string',
 755         description => "cloud-init: Sets DNS search domains for a container. Create will automatically use the setting from the host if neither searchdomain nor nameserver are set.",
 756     },
 757     nameserver => {
 758         optional => 1,
 759         type => 'string', format => 'address-list',
 760         description => "cloud-init: Sets DNS server IP address for a container. Create will automatically use the setting from the host if neither searchdomain nor nameserver are set.",
 761     },
 762     sshkeys => {
 763         optional => 1,
 764         type => 'string',
 765         format => 'urlencoded',
 766         description => "cloud-init: Setup public SSH keys (one key per line, OpenSSH format).",
 767     },
 768 };
 769
 770 # what about other qemu settings ?
 771 #cpu => 'string',
 772 #machine => 'string',
 773 #fda => 'file',
 774 #fdb => 'file',
 775 #mtdblock => 'file',
 776 #sd => 'file',
 777 #pflash => 'file',
 778 #snapshot => 'bool',
 779 #bootp => 'file',
 780 ##tftp => 'dir',
 781 ##smb => 'dir',
 782 #kernel => 'file',
 783 #append => 'string',
 784 #initrd => 'file',
 785 ##soundhw => 'string',
 786
 787 while (my ($k, $v) = each %$confdesc) {
 788     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 789 }
 790
 791 my $MAX_IDE_DISKS = 4;
 792 my $MAX_SCSI_DISKS = 14;
 793 my $MAX_VIRTIO_DISKS = 16;
 794 my $MAX_SATA_DISKS = 6;
 795 my $MAX_USB_DEVICES = 5;
 796 my $MAX_NETS = 32;
 797 my $MAX_UNUSED_DISKS = 256;
 798 my $MAX_HOSTPCI_DEVICES = 16;
 799 my $MAX_SERIAL_PORTS = 4;
 800 my $MAX_PARALLEL_PORTS = 3;
 801 my $MAX_NUMA = 8;
 802
 803 my $numa_fmt = {
 804     cpus => {
 805         type => "string",
 806         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 807         description => "CPUs accessing this NUMA node.",
 808         format_description => "id[-id];...",
 809     },
 810     memory => {
 811         type => "number",
 812         description => "Amount of memory this NUMA node provides.",
 813         optional => 1,
 814     },
 815     hostnodes => {
 816         type => "string",
 817         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 818         description => "Host NUMA nodes to use.",
 819         format_description => "id[-id];...",
 820         optional => 1,
 821     },
 822     policy => {
 823         type => 'string',
 824         enum => [qw(preferred bind interleave)],
 825         description => "NUMA allocation policy.",
 826         optional => 1,
 827     },
 828 };
 829 PVE::JSONSchema::register_format('pve-qm-numanode', $numa_fmt);
 830 my $numadesc = {
 831     optional => 1,
 832     type => 'string', format => $numa_fmt,
 833     description => "NUMA topology.",
 834 };
 835 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 836
 837 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 838     $confdesc->{"numa$i"} = $numadesc;
 839 }
 840
 841 my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000',  'pcnet',  'virtio',
 842                       'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3',
 843                       'e1000-82540em', 'e1000-82544gc', 'e1000-82545em'];
 844 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 845
 846 my $net_fmt_bridge_descr = <<__EOD__;
 847 Bridge to attach the network device to. The Proxmox VE standard bridge
 848 is called 'vmbr0'.
 849
 850 If you do not specify a bridge, we create a kvm user (NATed) network
 851 device, which provides DHCP and DNS services. The following addresses
 852 are used:
 853
 854  10.0.2.2   Gateway
 855  10.0.2.3   DNS Server
 856  10.0.2.4   SMB Server
 857
 858 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 859 __EOD__
 860
 861 my $net_fmt = {
 862     macaddr  => get_standard_option('mac-addr', {
 863         description => "MAC address. That address must be unique withing your network. This is automatically generated if not specified.",
 864     }),
 865     model => {
 866         type => 'string',
 867         description => "Network Card Model. The 'virtio' model provides the best performance with very low CPU overhead. If your guest does not support this driver, it is usually best to use 'e1000'.",
 868         enum => $nic_model_list,
 869         default_key => 1,
 870     },
 871     (map { $_ => { keyAlias => 'model', alias => 'macaddr' }} @$nic_model_list),
 872     bridge => {
 873         type => 'string',
 874         description => $net_fmt_bridge_descr,
 875         format_description => 'bridge',
 876         optional => 1,
 877     },
 878     queues => {
 879         type => 'integer',
 880         minimum => 0, maximum => 16,
 881         description => 'Number of packet queues to be used on the device.',
 882         optional => 1,
 883     },
 884     rate => {
 885         type => 'number',
 886         minimum => 0,
 887         description => "Rate limit in mbps (megabytes per second) as floating point number.",
 888         optional => 1,
 889     },
 890     tag => {
 891         type => 'integer',
 892         minimum => 1, maximum => 4094,
 893         description => 'VLAN tag to apply to packets on this interface.',
 894         optional => 1,
 895     },
 896     trunks => {
 897         type => 'string',
 898         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 899         description => 'VLAN trunks to pass through this interface.',
 900         format_description => 'vlanid[;vlanid...]',
 901         optional => 1,
 902     },
 903     firewall => {
 904         type => 'boolean',
 905         description => 'Whether this interface should be protected by the firewall.',
 906         optional => 1,
 907     },
 908     link_down => {
 909         type => 'boolean',
 910         description => 'Whether this interface should be disconnected (like pulling the plug).',
 911         optional => 1,
 912     },
 913 };
 914
 915 my $netdesc = {
 916     optional => 1,
 917     type => 'string', format => $net_fmt,
 918     description => "Specify network devices.",
 919 };
 920
 921 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 922
 923 my $ipconfig_fmt = {
 924     ip => {
 925         type => 'string',
 926         format => 'pve-ipv4-config',
 927         format_description => 'IPv4Format/CIDR',
 928         description => 'IPv4 address in CIDR format.',
 929         optional => 1,
 930         default => 'dhcp',
 931     },
 932     gw => {
 933         type => 'string',
 934         format => 'ipv4',
 935         format_description => 'GatewayIPv4',
 936         description => 'Default gateway for IPv4 traffic.',
 937         optional => 1,
 938         requires => 'ip',
 939     },
 940     ip6 => {
 941         type => 'string',
 942         format => 'pve-ipv6-config',
 943         format_description => 'IPv6Format/CIDR',
 944         description => 'IPv6 address in CIDR format.',
 945         optional => 1,
 946         default => 'dhcp',
 947     },
 948     gw6 => {
 949         type => 'string',
 950         format => 'ipv6',
 951         format_description => 'GatewayIPv6',
 952         description => 'Default gateway for IPv6 traffic.',
 953         optional => 1,
 954         requires => 'ip6',
 955     },
 956 };
 957 PVE::JSONSchema::register_format('pve-qm-ipconfig', $ipconfig_fmt);
 958 my $ipconfigdesc = {
 959     optional => 1,
 960     type => 'string', format => 'pve-qm-ipconfig',
 961     description => <<'EODESCR',
 962 cloud-init: Specify IP addresses and gateways for the corresponding interface.
 963
 964 IP addresses use CIDR notation, gateways are optional but need an IP of the same type specified.
 965
 966 The special string 'dhcp' can be used for IP addresses to use DHCP, in which case no explicit gateway should be provided.
 967 For IPv6 the special string 'auto' can be used to use stateless autoconfiguration.
 968
 969 If cloud-init is enabled and neither an IPv4 nor an IPv6 address is specified, it defaults to using dhcp on IPv4.
 970 EODESCR
 971 };
 972 PVE::JSONSchema::register_standard_option("pve-qm-ipconfig", $netdesc);
 973
 974 for (my $i = 0; $i < $MAX_NETS; $i++)  {
 975     $confdesc->{"net$i"} = $netdesc;
 976     $confdesc_cloudinit->{"ipconfig$i"} = $ipconfigdesc;
 977 }
 978
 979 foreach my $key (keys %$confdesc_cloudinit) {
 980     $confdesc->{$key} = $confdesc_cloudinit->{$key};
 981 }
 982
 983 PVE::JSONSchema::register_format('pve-volume-id-or-qm-path', \&verify_volume_id_or_qm_path);
 984 sub verify_volume_id_or_qm_path {
 985     my ($volid, $noerr) = @_;
 986
 987     if ($volid eq 'none' || $volid eq 'cdrom' || $volid =~ m|^/|) {
 988         return $volid;
 989     }
 990
 991     # if its neither 'none' nor 'cdrom' nor a path, check if its a volume-id
 992     $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') };
 993     if ($@) {
 994         return undef if $noerr;
 995         die $@;
 996     }
 997     return $volid;
 998 }
 999
1000 my $drivename_hash;
1001
1002 my %drivedesc_base = (
1003     volume => { alias => 'file' },
1004     file => {
1005         type => 'string',
1006         format => 'pve-volume-id-or-qm-path',
1007         default_key => 1,
1008         format_description => 'volume',
1009         description => "The drive's backing volume.",
1010     },
1011     media => {
1012         type => 'string',
1013         enum => [qw(cdrom disk)],
1014         description => "The drive's media type.",
1015         default => 'disk',
1016         optional => 1
1017     },
1018     cyls => {
1019         type => 'integer',
1020         description => "Force the drive's physical geometry to have a specific cylinder count.",
1021         optional => 1
1022     },
1023     heads => {
1024         type => 'integer',
1025         description => "Force the drive's physical geometry to have a specific head count.",
1026         optional => 1
1027     },
1028     secs => {
1029         type => 'integer',
1030         description => "Force the drive's physical geometry to have a specific sector count.",
1031         optional => 1
1032     },
1033     trans => {
1034         type => 'string',
1035         enum => [qw(none lba auto)],
1036         description => "Force disk geometry bios translation mode.",
1037         optional => 1,
1038     },
1039     snapshot => {
1040         type => 'boolean',
1041         description => "Controls qemu's snapshot mode feature."
1042             . " If activated, changes made to the disk are temporary and will"
1043             . " be discarded when the VM is shutdown.",
1044         optional => 1,
1045     },
1046     cache => {
1047         type => 'string',
1048         enum => [qw(none writethrough writeback unsafe directsync)],
1049         description => "The drive's cache mode",
1050         optional => 1,
1051     },
1052     format => get_standard_option('pve-qm-image-format'),
1053     size => {
1054         type => 'string',
1055         format => 'disk-size',
1056         format_description => 'DiskSize',
1057         description => "Disk size. This is purely informational and has no effect.",
1058         optional => 1,
1059     },
1060     backup => {
1061         type => 'boolean',
1062         description => "Whether the drive should be included when making backups.",
1063         optional => 1,
1064     },
1065     replicate => {
1066         type => 'boolean',
1067         description => 'Whether the drive should considered for replication jobs.',
1068         optional => 1,
1069         default => 1,
1070     },
1071     rerror => {
1072         type => 'string',
1073         enum => [qw(ignore report stop)],
1074         description => 'Read error action.',
1075         optional => 1,
1076     },
1077     werror => {
1078         type => 'string',
1079         enum => [qw(enospc ignore report stop)],
1080         description => 'Write error action.',
1081         optional => 1,
1082     },
1083     aio => {
1084         type => 'string',
1085         enum => [qw(native threads)],
1086         description => 'AIO type to use.',
1087         optional => 1,
1088     },
1089     discard => {
1090         type => 'string',
1091         enum => [qw(ignore on)],
1092         description => 'Controls whether to pass discard/trim requests to the underlying storage.',
1093         optional => 1,
1094     },
1095     detect_zeroes => {
1096         type => 'boolean',
1097         description => 'Controls whether to detect and try to optimize writes of zeroes.',
1098         optional => 1,
1099     },
1100     serial => {
1101         type => 'string',
1102         format => 'urlencoded',
1103         format_description => 'serial',
1104         maxLength => 20*3, # *3 since it's %xx url enoded
1105         description => "The drive's reported serial number, url-encoded, up to 20 bytes long.",
1106         optional => 1,
1107     },
1108     shared => {
1109         type => 'boolean',
1110         description => 'Mark this locally-managed volume as available on all nodes',
1111         verbose_description => "Mark this locally-managed volume as available on all nodes.\n\nWARNING: This option does not share the volume automatically, it assumes it is shared already!",
1112         optional => 1,
1113         default => 0,
1114     }
1115 );
1116
1117 my %iothread_fmt = ( iothread => {
1118         type => 'boolean',
1119         description => "Whether to use iothreads for this drive",
1120         optional => 1,
1121 });
1122
1123 my %model_fmt = (
1124     model => {
1125         type => 'string',
1126         format => 'urlencoded',
1127         format_description => 'model',
1128         maxLength => 40*3, # *3 since it's %xx url enoded
1129         description => "The drive's reported model name, url-encoded, up to 40 bytes long.",
1130         optional => 1,
1131     },
1132 );
1133
1134 my %queues_fmt = (
1135     queues => {
1136         type => 'integer',
1137         description => "Number of queues.",
1138         minimum => 2,
1139         optional => 1
1140     }
1141 );
1142
1143 my %scsiblock_fmt = (
1144     scsiblock => {
1145         type => 'boolean',
1146         description => "whether to use scsi-block for full passthrough of host block device\n\nWARNING: can lead to I/O errors in combination with low memory or high memory fragmentation on host",
1147         optional => 1,
1148         default => 0,
1149     },
1150 );
1151
1152 my %ssd_fmt = (
1153     ssd => {
1154         type => 'boolean',
1155         description => "Whether to expose this drive as an SSD, rather than a rotational hard disk.",
1156         optional => 1,
1157     },
1158 );
1159
1160 my %wwn_fmt = (
1161     wwn => {
1162         type => 'string',
1163         pattern => qr/^(0x)[0-9a-fA-F]{16}/,
1164         format_description => 'wwn',
1165         description => "The drive's worldwide name, encoded as 16 bytes hex string, prefixed by '0x'.",
1166         optional => 1,
1167     },
1168 );
1169
1170 my $add_throttle_desc = sub {
1171     my ($key, $type, $what, $unit, $longunit, $minimum) = @_;
1172     my $d = {
1173         type => $type,
1174         format_description => $unit,
1175         description => "Maximum $what in $longunit.",
1176         optional => 1,
1177     };
1178     $d->{minimum} = $minimum if defined($minimum);
1179     $drivedesc_base{$key} = $d;
1180 };
1181 # throughput: (leaky bucket)
1182 $add_throttle_desc->('bps',     'integer', 'r/w speed',   'bps',  'bytes per second');
1183 $add_throttle_desc->('bps_rd',  'integer', 'read speed',  'bps',  'bytes per second');
1184 $add_throttle_desc->('bps_wr',  'integer', 'write speed', 'bps',  'bytes per second');
1185 $add_throttle_desc->('mbps',    'number',  'r/w speed',   'mbps', 'megabytes per second');
1186 $add_throttle_desc->('mbps_rd', 'number',  'read speed',  'mbps', 'megabytes per second');
1187 $add_throttle_desc->('mbps_wr', 'number',  'write speed', 'mbps', 'megabytes per second');
1188 $add_throttle_desc->('iops',    'integer', 'r/w I/O',     'iops', 'operations per second');
1189 $add_throttle_desc->('iops_rd', 'integer', 'read I/O',    'iops', 'operations per second');
1190 $add_throttle_desc->('iops_wr', 'integer', 'write I/O',   'iops', 'operations per second');
1191
1192 # pools: (pool of IO before throttling starts taking effect)
1193 $add_throttle_desc->('mbps_max',    'number',  'unthrottled r/w pool',       'mbps', 'megabytes per second');
1194 $add_throttle_desc->('mbps_rd_max', 'number',  'unthrottled read pool',      'mbps', 'megabytes per second');
1195 $add_throttle_desc->('mbps_wr_max', 'number',  'unthrottled write pool',     'mbps', 'megabytes per second');
1196 $add_throttle_desc->('iops_max',    'integer', 'unthrottled r/w I/O pool',   'iops', 'operations per second');
1197 $add_throttle_desc->('iops_rd_max', 'integer', 'unthrottled read I/O pool',  'iops', 'operations per second');
1198 $add_throttle_desc->('iops_wr_max', 'integer', 'unthrottled write I/O pool', 'iops', 'operations per second');
1199
1200 # burst lengths
1201 $add_throttle_desc->('bps_max_length',     'integer', 'length of I/O bursts',       'seconds', 'seconds', 1);
1202 $add_throttle_desc->('bps_rd_max_length',  'integer', 'length of read I/O bursts',  'seconds', 'seconds', 1);
1203 $add_throttle_desc->('bps_wr_max_length',  'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
1204 $add_throttle_desc->('iops_max_length',    'integer', 'length of I/O bursts',       'seconds', 'seconds', 1);
1205 $add_throttle_desc->('iops_rd_max_length', 'integer', 'length of read I/O bursts',  'seconds', 'seconds', 1);
1206 $add_throttle_desc->('iops_wr_max_length', 'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
1207
1208 # legacy support
1209 $drivedesc_base{'bps_rd_length'} = { alias => 'bps_rd_max_length' };
1210 $drivedesc_base{'bps_wr_length'} = { alias => 'bps_wr_max_length' };
1211 $drivedesc_base{'iops_rd_length'} = { alias => 'iops_rd_max_length' };
1212 $drivedesc_base{'iops_wr_length'} = { alias => 'iops_wr_max_length' };
1213
1214 my $ide_fmt = {
1215     %drivedesc_base,
1216     %model_fmt,
1217     %ssd_fmt,
1218     %wwn_fmt,
1219 };
1220 PVE::JSONSchema::register_format("pve-qm-ide", $ide_fmt);
1221
1222 my $idedesc = {
1223     optional => 1,
1224     type => 'string', format => $ide_fmt,
1225     description => "Use volume as IDE hard disk or CD-ROM (n is 0 to " .($MAX_IDE_DISKS -1) . ").",
1226 };
1227 PVE::JSONSchema::register_standard_option("pve-qm-ide", $idedesc);
1228
1229 my $scsi_fmt = {
1230     %drivedesc_base,
1231     %iothread_fmt,
1232     %queues_fmt,
1233     %scsiblock_fmt,
1234     %ssd_fmt,
1235     %wwn_fmt,
1236 };
1237 my $scsidesc = {
1238     optional => 1,
1239     type => 'string', format => $scsi_fmt,
1240     description => "Use volume as SCSI hard disk or CD-ROM (n is 0 to " . ($MAX_SCSI_DISKS - 1) . ").",
1241 };
1242 PVE::JSONSchema::register_standard_option("pve-qm-scsi", $scsidesc);
1243
1244 my $sata_fmt = {
1245     %drivedesc_base,
1246     %ssd_fmt,
1247     %wwn_fmt,
1248 };
1249 my $satadesc = {
1250     optional => 1,
1251     type => 'string', format => $sata_fmt,
1252     description => "Use volume as SATA hard disk or CD-ROM (n is 0 to " . ($MAX_SATA_DISKS - 1). ").",
1253 };
1254 PVE::JSONSchema::register_standard_option("pve-qm-sata", $satadesc);
1255
1256 my $virtio_fmt = {
1257     %drivedesc_base,
1258     %iothread_fmt,
1259 };
1260 my $virtiodesc = {
1261     optional => 1,
1262     type => 'string', format => $virtio_fmt,
1263     description => "Use volume as VIRTIO hard disk (n is 0 to " . ($MAX_VIRTIO_DISKS - 1) . ").",
1264 };
1265 PVE::JSONSchema::register_standard_option("pve-qm-virtio", $virtiodesc);
1266
1267 my $alldrive_fmt = {
1268     %drivedesc_base,
1269     %iothread_fmt,
1270     %model_fmt,
1271     %queues_fmt,
1272     %scsiblock_fmt,
1273     %ssd_fmt,
1274     %wwn_fmt,
1275 };
1276
1277 my $efidisk_fmt = {
1278     volume => { alias => 'file' },
1279     file => {
1280         type => 'string',
1281         format => 'pve-volume-id-or-qm-path',
1282         default_key => 1,
1283         format_description => 'volume',
1284         description => "The drive's backing volume.",
1285     },
1286     format => get_standard_option('pve-qm-image-format'),
1287     size => {
1288         type => 'string',
1289         format => 'disk-size',
1290         format_description => 'DiskSize',
1291         description => "Disk size. This is purely informational and has no effect.",
1292         optional => 1,
1293     },
1294 };
1295
1296 my $efidisk_desc = {
1297     optional => 1,
1298     type => 'string', format => $efidisk_fmt,
1299     description => "Configure a Disk for storing EFI vars",
1300 };
1301
1302 PVE::JSONSchema::register_standard_option("pve-qm-efidisk", $efidisk_desc);
1303
1304 my $usb_fmt = {
1305     host => {
1306         default_key => 1,
1307         type => 'string', format => 'pve-qm-usb-device',
1308         format_description => 'HOSTUSBDEVICE|spice',
1309         description => <<EODESCR,
1310 The Host USB device or port or the value 'spice'. HOSTUSBDEVICE syntax is:
1311
1312  'bus-port(.port)*' (decimal numbers) or
1313  'vendor_id:product_id' (hexadeciaml numbers) or
1314  'spice'
1315
1316 You can use the 'lsusb -t' command to list existing usb devices.
1317
1318 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1319
1320 The value 'spice' can be used to add a usb redirection devices for spice.
1321 EODESCR
1322     },
1323     usb3 => {
1324         optional => 1,
1325         type => 'boolean',
1326         description => "Specifies whether if given host option is a USB3 device or port.",
1327         default => 0,
1328     },
1329 };
1330
1331 my $usbdesc = {
1332     optional => 1,
1333     type => 'string', format => $usb_fmt,
1334     description => "Configure an USB device (n is 0 to 4).",
1335 };
1336 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
1337
1338 my $PCIRE = qr/[a-f0-9]{2}:[a-f0-9]{2}(?:\.[a-f0-9])?/;
1339 my $hostpci_fmt = {
1340     host => {
1341         default_key => 1,
1342         type => 'string',
1343         pattern => qr/$PCIRE(;$PCIRE)*/,
1344         format_description => 'HOSTPCIID[;HOSTPCIID2...]',
1345         description => <<EODESCR,
1346 Host PCI device pass through. The PCI ID of a host's PCI device or a list
1347 of PCI virtual functions of the host. HOSTPCIID syntax is:
1348
1349 'bus:dev.func' (hexadecimal numbers)
1350
1351 You can us the 'lspci' command to list existing PCI devices.
1352 EODESCR
1353     },
1354     rombar => {
1355         type => 'boolean',
1356         description =>  "Specify whether or not the device's ROM will be visible in the guest's memory map.",
1357         optional => 1,
1358         default => 1,
1359     },
1360     romfile => {
1361         type => 'string',
1362         pattern => '[^,;]+',
1363         format_description => 'string',
1364         description => "Custom pci device rom filename (must be located in /usr/share/kvm/).",
1365         optional => 1,
1366     },
1367     pcie => {
1368         type => 'boolean',
1369         description =>  "Choose the PCI-express bus (needs the 'q35' machine model).",
1370         optional => 1,
1371         default => 0,
1372     },
1373     'x-vga' => {
1374         type => 'boolean',
1375         description =>  "Enable vfio-vga device support.",
1376         optional => 1,
1377         default => 0,
1378     },
1379     'mdev' => {
1380         type => 'string',
1381         format_description => 'string',
1382         pattern => '[^/\.:]+',
1383         optional => 1,
1384         description => <<EODESCR
1385 The type of mediated device to use.
1386 An instance of this type will be created on startup of the VM and
1387 will be cleaned up when the VM stops.
1388 EODESCR
1389     }
1390 };
1391 PVE::JSONSchema::register_format('pve-qm-hostpci', $hostpci_fmt);
1392
1393 my $hostpcidesc = {
1394         optional => 1,
1395         type => 'string', format => 'pve-qm-hostpci',
1396         description => "Map host PCI devices into guest.",
1397         verbose_description =>  <<EODESCR,
1398 Map host PCI devices into guest.
1399
1400 NOTE: This option allows direct access to host hardware. So it is no longer
1401 possible to migrate such machines - use with special care.
1402
1403 CAUTION: Experimental! User reported problems with this option.
1404 EODESCR
1405 };
1406 PVE::JSONSchema::register_standard_option("pve-qm-hostpci", $hostpcidesc);
1407
1408 my $serialdesc = {
1409         optional => 1,
1410         type => 'string',
1411         pattern => '(/dev/.+|socket)',
1412         description =>  "Create a serial device inside the VM (n is 0 to 3)",
1413         verbose_description =>  <<EODESCR,
1414 Create a serial device inside the VM (n is 0 to 3), and pass through a
1415 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1416 host side (use 'qm terminal' to open a terminal connection).
1417
1418 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines - use with special care.
1419
1420 CAUTION: Experimental! User reported problems with this option.
1421 EODESCR
1422 };
1423
1424 my $paralleldesc= {
1425         optional => 1,
1426         type => 'string',
1427         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
1428         description =>  "Map host parallel devices (n is 0 to 2).",
1429         verbose_description =>  <<EODESCR,
1430 Map host parallel devices (n is 0 to 2).
1431
1432 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1433
1434 CAUTION: Experimental! User reported problems with this option.
1435 EODESCR
1436 };
1437
1438 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
1439     $confdesc->{"parallel$i"} = $paralleldesc;
1440 }
1441
1442 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
1443     $confdesc->{"serial$i"} = $serialdesc;
1444 }
1445
1446 for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
1447     $confdesc->{"hostpci$i"} = $hostpcidesc;
1448 }
1449
1450 for (my $i = 0; $i < $MAX_IDE_DISKS; $i++)  {
1451     $drivename_hash->{"ide$i"} = 1;
1452     $confdesc->{"ide$i"} = $idedesc;
1453 }
1454
1455 for (my $i = 0; $i < $MAX_SATA_DISKS; $i++)  {
1456     $drivename_hash->{"sata$i"} = 1;
1457     $confdesc->{"sata$i"} = $satadesc;
1458 }
1459
1460 for (my $i = 0; $i < $MAX_SCSI_DISKS; $i++)  {
1461     $drivename_hash->{"scsi$i"} = 1;
1462     $confdesc->{"scsi$i"} = $scsidesc ;
1463 }
1464
1465 for (my $i = 0; $i < $MAX_VIRTIO_DISKS; $i++)  {
1466     $drivename_hash->{"virtio$i"} = 1;
1467     $confdesc->{"virtio$i"} = $virtiodesc;
1468 }
1469
1470 $drivename_hash->{efidisk0} = 1;
1471 $confdesc->{efidisk0} = $efidisk_desc;
1472
1473 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
1474     $confdesc->{"usb$i"} = $usbdesc;
1475 }
1476
1477 my $unuseddesc = {
1478     optional => 1,
1479     type => 'string', format => 'pve-volume-id',
1480     description => "Reference to unused volumes. This is used internally, and should not be modified manually.",
1481 };
1482
1483 for (my $i = 0; $i < $MAX_UNUSED_DISKS; $i++)  {
1484     $confdesc->{"unused$i"} = $unuseddesc;
1485 }
1486
1487 my $kvm_api_version = 0;
1488
1489 sub kvm_version {
1490     return $kvm_api_version if $kvm_api_version;
1491
1492     open my $fh, '<', '/dev/kvm'
1493         or return undef;
1494
1495     # 0xae00 => KVM_GET_API_VERSION
1496     $kvm_api_version = ioctl($fh, 0xae00, 0);
1497
1498     return $kvm_api_version;
1499 }
1500
1501 my $kvm_user_version = {};
1502 my $kvm_mtime = {};
1503
1504 sub kvm_user_version {
1505     my ($binary) = @_;
1506
1507     $binary //= get_command_for_arch(get_host_arch()); # get the native arch by default
1508     my $st = stat($binary);
1509
1510     my $cachedmtime = $kvm_mtime->{$binary} // -1;
1511     return $kvm_user_version->{$binary} if $kvm_user_version->{$binary} &&
1512         $cachedmtime == $st->mtime;
1513
1514     $kvm_user_version->{$binary} = 'unknown';
1515     $kvm_mtime->{$binary} = $st->mtime;
1516
1517     my $code = sub {
1518         my $line = shift;
1519         if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1520             $kvm_user_version->{$binary} = $2;
1521         }
1522     };
1523
1524     eval { run_command([$binary, '--version'], outfunc => $code); };
1525     warn $@ if $@;
1526
1527     return $kvm_user_version->{$binary};
1528
1529 }
1530
1531 sub kernel_has_vhost_net {
1532     return -c '/dev/vhost-net';
1533 }
1534
1535 sub valid_drive_names {
1536     # order is important - used to autoselect boot disk
1537     return ((map { "ide$_" } (0 .. ($MAX_IDE_DISKS - 1))),
1538             (map { "scsi$_" } (0 .. ($MAX_SCSI_DISKS - 1))),
1539             (map { "virtio$_" } (0 .. ($MAX_VIRTIO_DISKS - 1))),
1540             (map { "sata$_" } (0 .. ($MAX_SATA_DISKS - 1))),
1541             'efidisk0');
1542 }
1543
1544 sub is_valid_drivename {
1545     my $dev = shift;
1546
1547     return defined($drivename_hash->{$dev});
1548 }
1549
1550 sub option_exists {
1551     my $key = shift;
1552     return defined($confdesc->{$key});
1553 }
1554
1555 my $cdrom_path;
1556 sub get_cdrom_path {
1557
1558     return  $cdrom_path if $cdrom_path;
1559
1560     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
1561     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
1562     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
1563 }
1564
1565 sub get_iso_path {
1566     my ($storecfg, $vmid, $cdrom) = @_;
1567
1568     if ($cdrom eq 'cdrom') {
1569         return get_cdrom_path();
1570     } elsif ($cdrom eq 'none') {
1571         return '';
1572     } elsif ($cdrom =~ m|^/|) {
1573         return $cdrom;
1574     } else {
1575         return PVE::Storage::path($storecfg, $cdrom);
1576     }
1577 }
1578
1579 # try to convert old style file names to volume IDs
1580 sub filename_to_volume_id {
1581     my ($vmid, $file, $media) = @_;
1582
1583      if (!($file eq 'none' || $file eq 'cdrom' ||
1584           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1585
1586         return undef if $file =~ m|/|;
1587
1588         if ($media && $media eq 'cdrom') {
1589             $file = "local:iso/$file";
1590         } else {
1591             $file = "local:$vmid/$file";
1592         }
1593     }
1594
1595     return $file;
1596 }
1597
1598 sub verify_media_type {
1599     my ($opt, $vtype, $media) = @_;
1600
1601     return if !$media;
1602
1603     my $etype;
1604     if ($media eq 'disk') {
1605         $etype = 'images';
1606     } elsif ($media eq 'cdrom') {
1607         $etype = 'iso';
1608     } else {
1609         die "internal error";
1610     }
1611
1612     return if ($vtype eq $etype);
1613
1614     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1615 }
1616
1617 sub cleanup_drive_path {
1618     my ($opt, $storecfg, $drive) = @_;
1619
1620     # try to convert filesystem paths to volume IDs
1621
1622     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1623         ($drive->{file} !~ m|^/dev/.+|) &&
1624         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1625         ($drive->{file} !~ m/^\d+$/)) {
1626         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1627         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"}) if !$vtype;
1628         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1629         verify_media_type($opt, $vtype, $drive->{media});
1630         $drive->{file} = $volid;
1631     }
1632
1633     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1634 }
1635
1636 sub parse_hotplug_features {
1637     my ($data) = @_;
1638
1639     my $res = {};
1640
1641     return $res if $data eq '0';
1642
1643     $data = $confdesc->{hotplug}->{default} if $data eq '1';
1644
1645     foreach my $feature (PVE::Tools::split_list($data)) {
1646         if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
1647             $res->{$1} = 1;
1648         } else {
1649             die "invalid hotplug feature '$feature'\n";
1650         }
1651     }
1652     return $res;
1653 }
1654
1655 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1656 sub pve_verify_hotplug_features {
1657     my ($value, $noerr) = @_;
1658
1659     return $value if parse_hotplug_features($value);
1660
1661     return undef if $noerr;
1662
1663     die "unable to parse hotplug option\n";
1664 }
1665
1666 # ideX = [volume=]volume-id[,media=d][,cyls=c,heads=h,secs=s[,trans=t]]
1667 #        [,snapshot=on|off][,cache=on|off][,format=f][,backup=yes|no]
1668 #        [,rerror=ignore|report|stop][,werror=enospc|ignore|report|stop]
1669 #        [,aio=native|threads][,discard=ignore|on][,detect_zeroes=on|off]
1670 #        [,iothread=on][,serial=serial][,model=model]
1671
1672 sub parse_drive {
1673     my ($key, $data) = @_;
1674
1675     my ($interface, $index);
1676
1677     if ($key =~ m/^([^\d]+)(\d+)$/) {
1678         $interface = $1;
1679         $index = $2;
1680     } else {
1681         return undef;
1682     }
1683
1684     my $desc = $key =~ /^unused\d+$/ ? $alldrive_fmt
1685                                      : $confdesc->{$key}->{format};
1686     if (!$desc) {
1687         warn "invalid drive key: $key\n";
1688         return undef;
1689     }
1690     my $res = eval { PVE::JSONSchema::parse_property_string($desc, $data) };
1691     return undef if !$res;
1692     $res->{interface} = $interface;
1693     $res->{index} = $index;
1694
1695     my $error = 0;
1696     foreach my $opt (qw(bps bps_rd bps_wr)) {
1697         if (my $bps = defined(delete $res->{$opt})) {
1698             if (defined($res->{"m$opt"})) {
1699                 warn "both $opt and m$opt specified\n";
1700                 ++$error;
1701                 next;
1702             }
1703             $res->{"m$opt"} = sprintf("%.3f", $bps / (1024*1024.0));
1704         }
1705     }
1706
1707     # can't use the schema's 'requires' because of the mbps* => bps* "transforming aliases"
1708     for my $requirement (
1709         [mbps_max => 'mbps'],
1710         [mbps_rd_max => 'mbps_rd'],
1711         [mbps_wr_max => 'mbps_wr'],
1712         [miops_max => 'miops'],
1713         [miops_rd_max => 'miops_rd'],
1714         [miops_wr_max => 'miops_wr'],
1715         [bps_max_length => 'mbps_max'],
1716         [bps_rd_max_length => 'mbps_rd_max'],
1717         [bps_wr_max_length => 'mbps_wr_max'],
1718         [iops_max_length => 'iops_max'],
1719         [iops_rd_max_length => 'iops_rd_max'],
1720         [iops_wr_max_length => 'iops_wr_max']) {
1721         my ($option, $requires) = @$requirement;
1722         if ($res->{$option} && !$res->{$requires}) {
1723             warn "$option requires $requires\n";
1724             ++$error;
1725         }
1726     }
1727
1728     return undef if $error;
1729
1730     return undef if $res->{mbps_rd} && $res->{mbps};
1731     return undef if $res->{mbps_wr} && $res->{mbps};
1732     return undef if $res->{iops_rd} && $res->{iops};
1733     return undef if $res->{iops_wr} && $res->{iops};
1734
1735     if ($res->{media} && ($res->{media} eq 'cdrom')) {
1736         return undef if $res->{snapshot} || $res->{trans} || $res->{format};
1737         return undef if $res->{heads} || $res->{secs} || $res->{cyls};
1738         return undef if $res->{interface} eq 'virtio';
1739     }
1740
1741     if (my $size = $res->{size}) {
1742         return undef if !defined($res->{size} = PVE::JSONSchema::parse_size($size));
1743     }
1744
1745     return $res;
1746 }
1747
1748 sub print_drive {
1749     my ($vmid, $drive) = @_;
1750     my $data = { %$drive };
1751     delete $data->{$_} for qw(index interface);
1752     return PVE::JSONSchema::print_property_string($data, $alldrive_fmt);
1753 }
1754
1755 sub scsi_inquiry {
1756     my($fh, $noerr) = @_;
1757
1758     my $SG_IO = 0x2285;
1759     my $SG_GET_VERSION_NUM = 0x2282;
1760
1761     my $versionbuf = "\x00" x 8;
1762     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1763     if (!$ret) {
1764         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1765         return undef;
1766     }
1767     my $version = unpack("I", $versionbuf);
1768     if ($version < 30000) {
1769         die "scsi generic interface too old\n"  if !$noerr;
1770         return undef;
1771     }
1772
1773     my $buf = "\x00" x 36;
1774     my $sensebuf = "\x00" x 8;
1775     my $cmd = pack("C x3 C x1", 0x12, 36);
1776
1777     # see /usr/include/scsi/sg.h
1778     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1779
1780     my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1781                       length($sensebuf), 0, length($buf), $buf,
1782                       $cmd, $sensebuf, 6000);
1783
1784     $ret = ioctl($fh, $SG_IO, $packet);
1785     if (!$ret) {
1786         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1787         return undef;
1788     }
1789
1790     my @res = unpack($sg_io_hdr_t, $packet);
1791     if ($res[17] || $res[18]) {
1792         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1793         return undef;
1794     }
1795
1796     my $res = {};
1797     (my $byte0, my $byte1, $res->{vendor},
1798      $res->{product}, $res->{revision}) = unpack("C C x6 A8 A16 A4", $buf);
1799
1800     $res->{removable} = $byte1 & 128 ? 1 : 0;
1801     $res->{type} = $byte0 & 31;
1802
1803     return $res;
1804 }
1805
1806 sub path_is_scsi {
1807     my ($path) = @_;
1808
1809     my $fh = IO::File->new("+<$path") || return undef;
1810     my $res = scsi_inquiry($fh, 1);
1811     close($fh);
1812
1813     return $res;
1814 }
1815
1816 sub print_tabletdevice_full {
1817     my ($conf, $arch) = @_;
1818
1819     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
1820
1821     # we use uhci for old VMs because tablet driver was buggy in older qemu
1822     my $usbbus;
1823     if (PVE::QemuServer::Machine::machine_type_is_q35($conf) || $arch eq 'aarch64') {
1824         $usbbus = 'ehci';
1825     } else {
1826         $usbbus = 'uhci';
1827     }
1828
1829     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1830 }
1831
1832 sub print_keyboarddevice_full {
1833     my ($conf, $arch, $machine) = @_;
1834
1835     return undef if $arch ne 'aarch64';
1836
1837     return "usb-kbd,id=keyboard,bus=ehci.0,port=2";
1838 }
1839
1840 sub print_drivedevice_full {
1841     my ($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type) = @_;
1842
1843     my $device = '';
1844     my $maxdev = 0;
1845
1846     if ($drive->{interface} eq 'virtio') {
1847         my $pciaddr = print_pci_addr("$drive->{interface}$drive->{index}", $bridges, $arch, $machine_type);
1848         $device = "virtio-blk-pci,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}$pciaddr";
1849         $device .= ",iothread=iothread-$drive->{interface}$drive->{index}" if $drive->{iothread};
1850     } elsif ($drive->{interface} eq 'scsi') {
1851
1852         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1853         my $unit = $drive->{index} % $maxdev;
1854         my $devicetype = 'hd';
1855         my $path = '';
1856         if (drive_is_cdrom($drive)) {
1857             $devicetype = 'cd';
1858         } else {
1859             if ($drive->{file} =~ m|^/|) {
1860                 $path = $drive->{file};
1861                 if (my $info = path_is_scsi($path)) {
1862                     if ($info->{type} == 0 && $drive->{scsiblock}) {
1863                         $devicetype = 'block';
1864                     } elsif ($info->{type} == 1) { # tape
1865                         $devicetype = 'generic';
1866                     }
1867                 }
1868             } else {
1869                  $path = PVE::Storage::path($storecfg, $drive->{file});
1870             }
1871
1872             # for compatibility only, we prefer scsi-hd (#2408, #2355, #2380)
1873             my $version = PVE::QemuServer::Machine::extract_version($machine_type) // kvm_user_version();
1874             if ($path =~ m/^iscsi\:\/\// &&
1875                !min_version($version, 4, 1)) {
1876                 $devicetype = 'generic';
1877             }
1878         }
1879
1880         if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)){
1881             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1882         } else {
1883             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0,lun=$drive->{index},drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1884         }
1885
1886         if ($drive->{ssd} && ($devicetype eq 'block' || $devicetype eq 'hd')) {
1887             $device .= ",rotation_rate=1";
1888         }
1889         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1890
1891     } elsif ($drive->{interface} eq 'ide' || $drive->{interface} eq 'sata') {
1892         my $maxdev = ($drive->{interface} eq 'sata') ? $MAX_SATA_DISKS : 2;
1893         my $controller = int($drive->{index} / $maxdev);
1894         my $unit = $drive->{index} % $maxdev;
1895         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1896
1897         $device = "ide-$devicetype";
1898         if ($drive->{interface} eq 'ide') {
1899             $device .= ",bus=ide.$controller,unit=$unit";
1900         } else {
1901             $device .= ",bus=ahci$controller.$unit";
1902         }
1903         $device .= ",drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1904
1905         if ($devicetype eq 'hd') {
1906             if (my $model = $drive->{model}) {
1907                 $model = URI::Escape::uri_unescape($model);
1908                 $device .= ",model=$model";
1909             }
1910             if ($drive->{ssd}) {
1911                 $device .= ",rotation_rate=1";
1912             }
1913         }
1914         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1915     } elsif ($drive->{interface} eq 'usb') {
1916         die "implement me";
1917         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1918     } else {
1919         die "unsupported interface type";
1920     }
1921
1922     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1923
1924     if (my $serial = $drive->{serial}) {
1925         $serial = URI::Escape::uri_unescape($serial);
1926         $device .= ",serial=$serial";
1927     }
1928
1929
1930     return $device;
1931 }
1932
1933 sub get_initiator_name {
1934     my $initiator;
1935
1936     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return undef;
1937     while (defined(my $line = <$fh>)) {
1938         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1939         $initiator = $1;
1940         last;
1941     }
1942     $fh->close();
1943
1944     return $initiator;
1945 }
1946
1947 sub print_drive_full {
1948     my ($storecfg, $vmid, $drive) = @_;
1949
1950     my $path;
1951     my $volid = $drive->{file};
1952     my $format;
1953
1954     if (drive_is_cdrom($drive)) {
1955         $path = get_iso_path($storecfg, $vmid, $volid);
1956     } else {
1957         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1958         if ($storeid) {
1959             $path = PVE::Storage::path($storecfg, $volid);
1960             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
1961             $format = qemu_img_format($scfg, $volname);
1962         } else {
1963             $path = $volid;
1964             $format = "raw";
1965         }
1966    }
1967
1968     my $opts = '';
1969     my @qemu_drive_options = qw(heads secs cyls trans media format cache rerror werror aio discard);
1970     foreach my $o (@qemu_drive_options) {
1971         $opts .= ",$o=$drive->{$o}" if defined($drive->{$o});
1972     }
1973
1974     # snapshot only accepts on|off
1975     if (defined($drive->{snapshot})) {
1976         my $v = $drive->{snapshot} ? 'on' : 'off';
1977         $opts .= ",snapshot=$v";
1978     }
1979
1980     foreach my $type (['', '-total'], [_rd => '-read'], [_wr => '-write']) {
1981         my ($dir, $qmpname) = @$type;
1982         if (my $v = $drive->{"mbps$dir"}) {
1983             $opts .= ",throttling.bps$qmpname=".int($v*1024*1024);
1984         }
1985         if (my $v = $drive->{"mbps${dir}_max"}) {
1986             $opts .= ",throttling.bps$qmpname-max=".int($v*1024*1024);
1987         }
1988         if (my $v = $drive->{"bps${dir}_max_length"}) {
1989             $opts .= ",throttling.bps$qmpname-max-length=$v";
1990         }
1991         if (my $v = $drive->{"iops${dir}"}) {
1992             $opts .= ",throttling.iops$qmpname=$v";
1993         }
1994         if (my $v = $drive->{"iops${dir}_max"}) {
1995             $opts .= ",throttling.iops$qmpname-max=$v";
1996         }
1997         if (my $v = $drive->{"iops${dir}_max_length"}) {
1998             $opts .= ",throttling.iops$qmpname-max-length=$v";
1999         }
2000     }
2001
2002     $opts .= ",format=$format" if $format && !$drive->{format};
2003
2004     my $cache_direct = 0;
2005
2006     if (my $cache = $drive->{cache}) {
2007         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
2008     } elsif (!drive_is_cdrom($drive)) {
2009         $opts .= ",cache=none";
2010         $cache_direct = 1;
2011     }
2012
2013     # aio native works only with O_DIRECT
2014     if (!$drive->{aio}) {
2015         if($cache_direct) {
2016             $opts .= ",aio=native";
2017         } else {
2018             $opts .= ",aio=threads";
2019         }
2020     }
2021
2022     if (!drive_is_cdrom($drive)) {
2023         my $detectzeroes;
2024         if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
2025             $detectzeroes = 'off';
2026         } elsif ($drive->{discard}) {
2027             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
2028         } else {
2029             # This used to be our default with discard not being specified:
2030             $detectzeroes = 'on';
2031         }
2032         $opts .= ",detect-zeroes=$detectzeroes" if $detectzeroes;
2033     }
2034
2035     my $pathinfo = $path ? "file=$path," : '';
2036
2037     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
2038 }
2039
2040 sub print_netdevice_full {
2041     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files, $arch, $machine_type) = @_;
2042
2043     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
2044
2045     my $device = $net->{model};
2046     if ($net->{model} eq 'virtio') {
2047          $device = 'virtio-net-pci';
2048      };
2049
2050     my $pciaddr = print_pci_addr("$netid", $bridges, $arch, $machine_type);
2051     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
2052     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
2053         #Consider we have N queues, the number of vectors needed is 2*N + 2 (plus one config interrupt and control vq)
2054         my $vectors = $net->{queues} * 2 + 2;
2055         $tmpstr .= ",vectors=$vectors,mq=on";
2056     }
2057     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
2058
2059     if ($use_old_bios_files) {
2060         my $romfile;
2061         if ($device eq 'virtio-net-pci') {
2062             $romfile = 'pxe-virtio.rom';
2063         } elsif ($device eq 'e1000') {
2064             $romfile = 'pxe-e1000.rom';
2065         } elsif ($device eq 'ne2k') {
2066             $romfile = 'pxe-ne2k_pci.rom';
2067         } elsif ($device eq 'pcnet') {
2068             $romfile = 'pxe-pcnet.rom';
2069         } elsif ($device eq 'rtl8139') {
2070             $romfile = 'pxe-rtl8139.rom';
2071         }
2072         $tmpstr .= ",romfile=$romfile" if $romfile;
2073     }
2074
2075     return $tmpstr;
2076 }
2077
2078 sub print_netdev_full {
2079     my ($vmid, $conf, $arch, $net, $netid, $hotplug) = @_;
2080
2081     my $i = '';
2082     if ($netid =~ m/^net(\d+)$/) {
2083         $i = int($1);
2084     }
2085
2086     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
2087
2088     my $ifname = "tap${vmid}i$i";
2089
2090     # kvm uses TUNSETIFF ioctl, and that limits ifname length
2091     die "interface name '$ifname' is too long (max 15 character)\n"
2092         if length($ifname) >= 16;
2093
2094     my $vhostparam = '';
2095     if (is_native($arch)) {
2096         $vhostparam = ',vhost=on' if kernel_has_vhost_net() && $net->{model} eq 'virtio';
2097     }
2098
2099     my $vmname = $conf->{name} || "vm$vmid";
2100
2101     my $netdev = "";
2102     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
2103
2104     if ($net->{bridge}) {
2105         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script,downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
2106     } else {
2107         $netdev = "type=user,id=$netid,hostname=$vmname";
2108     }
2109
2110     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
2111
2112     return $netdev;
2113 }
2114
2115
2116 sub print_cpu_device {
2117     my ($conf, $id) = @_;
2118
2119     my $kvm = $conf->{kvm} // 1;
2120     my $cpu = $kvm ? "kvm64" : "qemu64";
2121     if (my $cputype = $conf->{cpu}) {
2122         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
2123             or die "Cannot parse cpu description: $cputype\n";
2124         $cpu = $cpuconf->{cputype};
2125     }
2126
2127     my $cores = $conf->{cores} || 1;
2128
2129     my $current_core = ($id - 1) % $cores;
2130     my $current_socket = int(($id - 1 - $current_core)/$cores);
2131
2132     return "$cpu-x86_64-cpu,id=cpu$id,socket-id=$current_socket,core-id=$current_core,thread-id=0";
2133 }
2134
2135 my $vga_map = {
2136     'cirrus' => 'cirrus-vga',
2137     'std' => 'VGA',
2138     'vmware' => 'vmware-svga',
2139     'virtio' => 'virtio-vga',
2140 };
2141
2142 sub print_vga_device {
2143     my ($conf, $vga, $arch, $machine_version, $machine, $id, $qxlnum, $bridges) = @_;
2144
2145     my $type = $vga_map->{$vga->{type}};
2146     if ($arch eq 'aarch64' && defined($type) && $type eq 'virtio-vga') {
2147         $type = 'virtio-gpu';
2148     }
2149     my $vgamem_mb = $vga->{memory};
2150
2151     my $max_outputs = '';
2152     if ($qxlnum) {
2153         $type = $id ? 'qxl' : 'qxl-vga';
2154
2155         if (!$conf->{ostype} || $conf->{ostype} =~ m/^(?:l\d\d)|(?:other)$/) {
2156             # set max outputs so linux can have up to 4 qxl displays with one device
2157             if (min_version($machine_version, 4, 1)) {
2158                 $max_outputs = ",max_outputs=4";
2159             }
2160         }
2161     }
2162
2163     die "no devicetype for $vga->{type}\n" if !$type;
2164
2165     my $memory = "";
2166     if ($vgamem_mb) {
2167         if ($vga->{type} eq 'virtio') {
2168             my $bytes = PVE::Tools::convert_size($vgamem_mb, "mb" => "b");
2169             $memory = ",max_hostmem=$bytes";
2170         } elsif ($qxlnum) {
2171             # from https://www.spice-space.org/multiple-monitors.html
2172             $memory = ",vgamem_mb=$vga->{memory}";
2173             my $ram = $vgamem_mb * 4;
2174             my $vram = $vgamem_mb * 2;
2175             $memory .= ",ram_size_mb=$ram,vram_size_mb=$vram";
2176         } else {
2177             $memory = ",vgamem_mb=$vga->{memory}";
2178         }
2179     } elsif ($qxlnum && $id) {
2180         $memory = ",ram_size=67108864,vram_size=33554432";
2181     }
2182
2183     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
2184     my $vgaid = "vga" . ($id // '');
2185     my $pciaddr;
2186
2187     if ($q35 && $vgaid eq 'vga') {
2188         # the first display uses pcie.0 bus on q35 machines
2189         $pciaddr = print_pcie_addr($vgaid, $bridges, $arch, $machine);
2190     } else {
2191         $pciaddr = print_pci_addr($vgaid, $bridges, $arch, $machine);
2192     }
2193
2194     return "$type,id=${vgaid}${memory}${max_outputs}${pciaddr}";
2195 }
2196
2197 sub drive_is_cloudinit {
2198     my ($drive) = @_;
2199     return $drive->{file} =~ m@[:/]vm-\d+-cloudinit(?:\.$QEMU_FORMAT_RE)?$@;
2200 }
2201
2202 sub drive_is_cdrom {
2203     my ($drive, $exclude_cloudinit) = @_;
2204
2205     return 0 if $exclude_cloudinit && drive_is_cloudinit($drive);
2206
2207     return $drive && $drive->{media} && ($drive->{media} eq 'cdrom');
2208
2209 }
2210
2211 sub parse_number_sets {
2212     my ($set) = @_;
2213     my $res = [];
2214     foreach my $part (split(/;/, $set)) {
2215         if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
2216             die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
2217             push @$res, [ $1, $2 ];
2218         } else {
2219             die "invalid range: $part\n";
2220         }
2221     }
2222     return $res;
2223 }
2224
2225 sub parse_numa {
2226     my ($data) = @_;
2227
2228     my $res = PVE::JSONSchema::parse_property_string($numa_fmt, $data);
2229     $res->{cpus} = parse_number_sets($res->{cpus}) if defined($res->{cpus});
2230     $res->{hostnodes} = parse_number_sets($res->{hostnodes}) if defined($res->{hostnodes});
2231     return $res;
2232 }
2233
2234 sub parse_hostpci {
2235     my ($value) = @_;
2236
2237     return undef if !$value;
2238
2239     my $res = PVE::JSONSchema::parse_property_string($hostpci_fmt, $value);
2240
2241     my @idlist = split(/;/, $res->{host});
2242     delete $res->{host};
2243     foreach my $id (@idlist) {
2244         if ($id =~ m/\./) { # full id 00:00.1
2245             push @{$res->{pciid}}, {
2246                 id => $id,
2247             };
2248         } else { # partial id 00:00
2249             $res->{pciid} = PVE::SysFSTools::lspci($id);
2250         }
2251     }
2252     return $res;
2253 }
2254
2255 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
2256 sub parse_net {
2257     my ($data) = @_;
2258
2259     my $res = eval { PVE::JSONSchema::parse_property_string($net_fmt, $data) };
2260     if ($@) {
2261         warn $@;
2262         return undef;
2263     }
2264     if (!defined($res->{macaddr})) {
2265         my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
2266         $res->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix});
2267     }
2268     return $res;
2269 }
2270
2271 # ipconfigX ip=cidr,gw=ip,ip6=cidr,gw6=ip
2272 sub parse_ipconfig {
2273     my ($data) = @_;
2274
2275     my $res = eval { PVE::JSONSchema::parse_property_string($ipconfig_fmt, $data) };
2276     if ($@) {
2277         warn $@;
2278         return undef;
2279     }
2280
2281     if ($res->{gw} && !$res->{ip}) {
2282         warn 'gateway specified without specifying an IP address';
2283         return undef;
2284     }
2285     if ($res->{gw6} && !$res->{ip6}) {
2286         warn 'IPv6 gateway specified without specifying an IPv6 address';
2287         return undef;
2288     }
2289     if ($res->{gw} && $res->{ip} eq 'dhcp') {
2290         warn 'gateway specified together with DHCP';
2291         return undef;
2292     }
2293     if ($res->{gw6} && $res->{ip6} !~ /^$IPV6RE/) {
2294         # gw6 + auto/dhcp
2295         warn "IPv6 gateway specified together with $res->{ip6} address";
2296         return undef;
2297     }
2298
2299     if (!$res->{ip} && !$res->{ip6}) {
2300         return { ip => 'dhcp', ip6 => 'dhcp' };
2301     }
2302
2303     return $res;
2304 }
2305
2306 sub print_net {
2307     my $net = shift;
2308
2309     return PVE::JSONSchema::print_property_string($net, $net_fmt);
2310 }
2311
2312 sub add_random_macs {
2313     my ($settings) = @_;
2314
2315     foreach my $opt (keys %$settings) {
2316         next if $opt !~ m/^net(\d+)$/;
2317         my $net = parse_net($settings->{$opt});
2318         next if !$net;
2319         $settings->{$opt} = print_net($net);
2320     }
2321 }
2322
2323 sub vm_is_volid_owner {
2324     my ($storecfg, $vmid, $volid) = @_;
2325
2326     if ($volid !~  m|^/|) {
2327         my ($path, $owner);
2328         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
2329         if ($owner && ($owner == $vmid)) {
2330             return 1;
2331         }
2332     }
2333
2334     return undef;
2335 }
2336
2337 sub vmconfig_register_unused_drive {
2338     my ($storecfg, $vmid, $conf, $drive) = @_;
2339
2340     if (drive_is_cloudinit($drive)) {
2341         eval { PVE::Storage::vdisk_free($storecfg, $drive->{file}) };
2342         warn $@ if $@;
2343     } elsif (!drive_is_cdrom($drive)) {
2344         my $volid = $drive->{file};
2345         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
2346             PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
2347         }
2348     }
2349 }
2350
2351 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str][,base64=bool]
2352 my $smbios1_fmt = {
2353     uuid => {
2354         type => 'string',
2355         pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
2356         format_description => 'UUID',
2357         description => "Set SMBIOS1 UUID.",
2358         optional => 1,
2359     },
2360     version => {
2361         type => 'string',
2362         pattern => '[A-Za-z0-9+\/]+={0,2}',
2363         format_description => 'Base64 encoded string',
2364         description => "Set SMBIOS1 version.",
2365         optional => 1,
2366     },
2367     serial => {
2368         type => 'string',
2369         pattern => '[A-Za-z0-9+\/]+={0,2}',
2370         format_description => 'Base64 encoded string',
2371         description => "Set SMBIOS1 serial number.",
2372         optional => 1,
2373     },
2374     manufacturer => {
2375         type => 'string',
2376         pattern => '[A-Za-z0-9+\/]+={0,2}',
2377         format_description => 'Base64 encoded string',
2378         description => "Set SMBIOS1 manufacturer.",
2379         optional => 1,
2380     },
2381     product => {
2382         type => 'string',
2383         pattern => '[A-Za-z0-9+\/]+={0,2}',
2384         format_description => 'Base64 encoded string',
2385         description => "Set SMBIOS1 product ID.",
2386         optional => 1,
2387     },
2388     sku => {
2389         type => 'string',
2390         pattern => '[A-Za-z0-9+\/]+={0,2}',
2391         format_description => 'Base64 encoded string',
2392         description => "Set SMBIOS1 SKU string.",
2393         optional => 1,
2394     },
2395     family => {
2396         type => 'string',
2397         pattern => '[A-Za-z0-9+\/]+={0,2}',
2398         format_description => 'Base64 encoded string',
2399         description => "Set SMBIOS1 family string.",
2400         optional => 1,
2401     },
2402     base64 => {
2403         type => 'boolean',
2404         description => 'Flag to indicate that the SMBIOS values are base64 encoded',
2405         optional => 1,
2406     },
2407 };
2408
2409 sub parse_smbios1 {
2410     my ($data) = @_;
2411
2412     my $res = eval { PVE::JSONSchema::parse_property_string($smbios1_fmt, $data) };
2413     warn $@ if $@;
2414     return $res;
2415 }
2416
2417 sub print_smbios1 {
2418     my ($smbios1) = @_;
2419     return PVE::JSONSchema::print_property_string($smbios1, $smbios1_fmt);
2420 }
2421
2422 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_fmt);
2423
2424 PVE::JSONSchema::register_format('pve-qm-bootdisk', \&verify_bootdisk);
2425 sub verify_bootdisk {
2426     my ($value, $noerr) = @_;
2427
2428     return $value if is_valid_drivename($value);
2429
2430     return undef if $noerr;
2431
2432     die "invalid boot disk '$value'\n";
2433 }
2434
2435 sub parse_watchdog {
2436     my ($value) = @_;
2437
2438     return undef if !$value;
2439
2440     my $res = eval { PVE::JSONSchema::parse_property_string($watchdog_fmt, $value) };
2441     warn $@ if $@;
2442     return $res;
2443 }
2444
2445 sub parse_guest_agent {
2446     my ($value) = @_;
2447
2448     return {} if !defined($value->{agent});
2449
2450     my $res = eval { PVE::JSONSchema::parse_property_string($agent_fmt, $value->{agent}) };
2451     warn $@ if $@;
2452
2453     # if the agent is disabled ignore the other potentially set properties
2454     return {} if !$res->{enabled};
2455     return $res;
2456 }
2457
2458 sub parse_vga {
2459     my ($value) = @_;
2460
2461     return {} if !$value;
2462     my $res = eval { PVE::JSONSchema::parse_property_string($vga_fmt, $value) };
2463     warn $@ if $@;
2464     return $res;
2465 }
2466
2467 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
2468 sub verify_usb_device {
2469     my ($value, $noerr) = @_;
2470
2471     return $value if parse_usb_device($value);
2472
2473     return undef if $noerr;
2474
2475     die "unable to parse usb device\n";
2476 }
2477
2478 # add JSON properties for create and set function
2479 sub json_config_properties {
2480     my $prop = shift;
2481
2482     foreach my $opt (keys %$confdesc) {
2483         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate' || $opt eq 'runningmachine';
2484         $prop->{$opt} = $confdesc->{$opt};
2485     }
2486
2487     return $prop;
2488 }
2489
2490 # return copy of $confdesc_cloudinit to generate documentation
2491 sub cloudinit_config_properties {
2492
2493     return dclone($confdesc_cloudinit);
2494 }
2495
2496 sub check_type {
2497     my ($key, $value) = @_;
2498
2499     die "unknown setting '$key'\n" if !$confdesc->{$key};
2500
2501     my $type = $confdesc->{$key}->{type};
2502
2503     if (!defined($value)) {
2504         die "got undefined value\n";
2505     }
2506
2507     if ($value =~ m/[\n\r]/) {
2508         die "property contains a line feed\n";
2509     }
2510
2511     if ($type eq 'boolean') {
2512         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
2513         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
2514         die "type check ('boolean') failed - got '$value'\n";
2515     } elsif ($type eq 'integer') {
2516         return int($1) if $value =~ m/^(\d+)$/;
2517         die "type check ('integer') failed - got '$value'\n";
2518     } elsif ($type eq 'number') {
2519         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
2520         die "type check ('number') failed - got '$value'\n";
2521     } elsif ($type eq 'string') {
2522         if (my $fmt = $confdesc->{$key}->{format}) {
2523             PVE::JSONSchema::check_format($fmt, $value);
2524             return $value;
2525         }
2526         $value =~ s/^\"(.*)\"$/$1/;
2527         return $value;
2528     } else {
2529         die "internal error"
2530     }
2531 }
2532
2533 sub destroy_vm {
2534     my ($storecfg, $vmid, $skiplock, $replacement_conf) = @_;
2535
2536     my $conf = PVE::QemuConfig->load_config($vmid);
2537
2538     PVE::QemuConfig->check_lock($conf) if !$skiplock;
2539
2540     if ($conf->{template}) {
2541         # check if any base image is still used by a linked clone
2542         foreach_drive($conf, sub {
2543                 my ($ds, $drive) = @_;
2544                 return if drive_is_cdrom($drive);
2545
2546                 my $volid = $drive->{file};
2547                 return if !$volid || $volid =~ m|^/|;
2548
2549                 die "base volume '$volid' is still in use by linked cloned\n"
2550                     if PVE::Storage::volume_is_base_and_used($storecfg, $volid);
2551
2552         });
2553     }
2554
2555     # only remove disks owned by this VM
2556     foreach_drive($conf, sub {
2557         my ($ds, $drive) = @_;
2558         return if drive_is_cdrom($drive, 1);
2559
2560         my $volid = $drive->{file};
2561         return if !$volid || $volid =~ m|^/|;
2562
2563         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
2564         return if !$path || !$owner || ($owner != $vmid);
2565
2566         eval { PVE::Storage::vdisk_free($storecfg, $volid) };
2567         warn "Could not remove disk '$volid', check manually: $@" if $@;
2568     });
2569
2570     # also remove unused disk
2571     my $vmdisks = PVE::Storage::vdisk_list($storecfg, undef, $vmid);
2572     PVE::Storage::foreach_volid($vmdisks, sub {
2573         my ($volid, $sid, $volname, $d) = @_;
2574         eval { PVE::Storage::vdisk_free($storecfg, $volid) };
2575         warn $@ if $@;
2576     });
2577
2578     if (defined $replacement_conf) {
2579         PVE::QemuConfig->write_config($vmid, $replacement_conf);
2580     } else {
2581         PVE::QemuConfig->destroy_config($vmid);
2582     }
2583 }
2584
2585 sub parse_vm_config {
2586     my ($filename, $raw) = @_;
2587
2588     return undef if !defined($raw);
2589
2590     my $res = {
2591         digest => Digest::SHA::sha1_hex($raw),
2592         snapshots => {},
2593         pending => {},
2594     };
2595
2596     $filename =~ m|/qemu-server/(\d+)\.conf$|
2597         || die "got strange filename '$filename'";
2598
2599     my $vmid = $1;
2600
2601     my $conf = $res;
2602     my $descr;
2603     my $section = '';
2604
2605     my @lines = split(/\n/, $raw);
2606     foreach my $line (@lines) {
2607         next if $line =~ m/^\s*$/;
2608
2609         if ($line =~ m/^\[PENDING\]\s*$/i) {
2610             $section = 'pending';
2611             if (defined($descr)) {
2612                 $descr =~ s/\s+$//;
2613                 $conf->{description} = $descr;
2614             }
2615             $descr = undef;
2616             $conf = $res->{$section} = {};
2617             next;
2618
2619         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2620             $section = $1;
2621             if (defined($descr)) {
2622                 $descr =~ s/\s+$//;
2623                 $conf->{description} = $descr;
2624             }
2625             $descr = undef;
2626             $conf = $res->{snapshots}->{$section} = {};
2627             next;
2628         }
2629
2630         if ($line =~ m/^\#(.*)\s*$/) {
2631             $descr = '' if !defined($descr);
2632             $descr .= PVE::Tools::decode_text($1) . "\n";
2633             next;
2634         }
2635
2636         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2637             $descr = '' if !defined($descr);
2638             $descr .= PVE::Tools::decode_text($2);
2639         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2640             $conf->{snapstate} = $1;
2641         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2642             my $key = $1;
2643             my $value = $2;
2644             $conf->{$key} = $value;
2645         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2646             my $value = $1;
2647             if ($section eq 'pending') {
2648                 $conf->{delete} = $value; # we parse this later
2649             } else {
2650                 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
2651             }
2652         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(.+?)\s*$/) {
2653             my $key = $1;
2654             my $value = $2;
2655             eval { $value = check_type($key, $value); };
2656             if ($@) {
2657                 warn "vm $vmid - unable to parse value of '$key' - $@";
2658             } else {
2659                 $key = 'ide2' if $key eq 'cdrom';
2660                 my $fmt = $confdesc->{$key}->{format};
2661                 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2662                     my $v = parse_drive($key, $value);
2663                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2664                         $v->{file} = $volid;
2665                         $value = print_drive($vmid, $v);
2666                     } else {
2667                         warn "vm $vmid - unable to parse value of '$key'\n";
2668                         next;
2669                     }
2670                 }
2671
2672                 $conf->{$key} = $value;
2673             }
2674         }
2675     }
2676
2677     if (defined($descr)) {
2678         $descr =~ s/\s+$//;
2679         $conf->{description} = $descr;
2680     }
2681     delete $res->{snapstate}; # just to be sure
2682
2683     return $res;
2684 }
2685
2686 sub write_vm_config {
2687     my ($filename, $conf) = @_;
2688
2689     delete $conf->{snapstate}; # just to be sure
2690
2691     if ($conf->{cdrom}) {
2692         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2693         $conf->{ide2} = $conf->{cdrom};
2694         delete $conf->{cdrom};
2695     }
2696
2697     # we do not use 'smp' any longer
2698     if ($conf->{sockets}) {
2699         delete $conf->{smp};
2700     } elsif ($conf->{smp}) {
2701         $conf->{sockets} = $conf->{smp};
2702         delete $conf->{cores};
2703         delete $conf->{smp};
2704     }
2705
2706     my $used_volids = {};
2707
2708     my $cleanup_config = sub {
2709         my ($cref, $pending, $snapname) = @_;
2710
2711         foreach my $key (keys %$cref) {
2712             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2713                 $key eq 'snapstate' || $key eq 'pending';
2714             my $value = $cref->{$key};
2715             if ($key eq 'delete') {
2716                 die "propertry 'delete' is only allowed in [PENDING]\n"
2717                     if !$pending;
2718                 # fixme: check syntax?
2719                 next;
2720             }
2721             eval { $value = check_type($key, $value); };
2722             die "unable to parse value of '$key' - $@" if $@;
2723
2724             $cref->{$key} = $value;
2725
2726             if (!$snapname && is_valid_drivename($key)) {
2727                 my $drive = parse_drive($key, $value);
2728                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2729             }
2730         }
2731     };
2732
2733     &$cleanup_config($conf);
2734
2735     &$cleanup_config($conf->{pending}, 1);
2736
2737     foreach my $snapname (keys %{$conf->{snapshots}}) {
2738         die "internal error" if $snapname eq 'pending';
2739         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2740     }
2741
2742     # remove 'unusedX' settings if we re-add a volume
2743     foreach my $key (keys %$conf) {
2744         my $value = $conf->{$key};
2745         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2746             delete $conf->{$key};
2747         }
2748     }
2749
2750     my $generate_raw_config = sub {
2751         my ($conf, $pending) = @_;
2752
2753         my $raw = '';
2754
2755         # add description as comment to top of file
2756         if (defined(my $descr = $conf->{description})) {
2757             if ($descr) {
2758                 foreach my $cl (split(/\n/, $descr)) {
2759                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2760                 }
2761             } else {
2762                 $raw .= "#\n" if $pending;
2763             }
2764         }
2765
2766         foreach my $key (sort keys %$conf) {
2767             next if $key eq 'digest' || $key eq 'description' || $key eq 'pending' || $key eq 'snapshots';
2768             $raw .= "$key: $conf->{$key}\n";
2769         }
2770         return $raw;
2771     };
2772
2773     my $raw = &$generate_raw_config($conf);
2774
2775     if (scalar(keys %{$conf->{pending}})){
2776         $raw .= "\n[PENDING]\n";
2777         $raw .= &$generate_raw_config($conf->{pending}, 1);
2778     }
2779
2780     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2781         $raw .= "\n[$snapname]\n";
2782         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2783     }
2784
2785     return $raw;
2786 }
2787
2788 sub load_defaults {
2789
2790     my $res = {};
2791
2792     # we use static defaults from our JSON schema configuration
2793     foreach my $key (keys %$confdesc) {
2794         if (defined(my $default = $confdesc->{$key}->{default})) {
2795             $res->{$key} = $default;
2796         }
2797     }
2798
2799     return $res;
2800 }
2801
2802 sub config_list {
2803     my $vmlist = PVE::Cluster::get_vmlist();
2804     my $res = {};
2805     return $res if !$vmlist || !$vmlist->{ids};
2806     my $ids = $vmlist->{ids};
2807
2808     foreach my $vmid (keys %$ids) {
2809         my $d = $ids->{$vmid};
2810         next if !$d->{node} || $d->{node} ne $nodename;
2811         next if !$d->{type} || $d->{type} ne 'qemu';
2812         $res->{$vmid}->{exists} = 1;
2813     }
2814     return $res;
2815 }
2816
2817 # test if VM uses local resources (to prevent migration)
2818 sub check_local_resources {
2819     my ($conf, $noerr) = @_;
2820
2821     my @loc_res = ();
2822
2823     push @loc_res, "hostusb" if $conf->{hostusb}; # old syntax
2824     push @loc_res, "hostpci" if $conf->{hostpci}; # old syntax
2825
2826     push @loc_res, "ivshmem" if $conf->{ivshmem};
2827
2828     foreach my $k (keys %$conf) {
2829         next if $k =~ m/^usb/ && ($conf->{$k} =~ m/^spice(?![^,])/);
2830         # sockets are safe: they will recreated be on the target side post-migrate
2831         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2832         push @loc_res, $k if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2833     }
2834
2835     die "VM uses local resources\n" if scalar @loc_res && !$noerr;
2836
2837     return \@loc_res;
2838 }
2839
2840 # check if used storages are available on all nodes (use by migrate)
2841 sub check_storage_availability {
2842     my ($storecfg, $conf, $node) = @_;
2843
2844     foreach_drive($conf, sub {
2845         my ($ds, $drive) = @_;
2846
2847         my $volid = $drive->{file};
2848         return if !$volid;
2849
2850         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2851         return if !$sid;
2852
2853         # check if storage is available on both nodes
2854         my $scfg = PVE::Storage::storage_check_node($storecfg, $sid);
2855         PVE::Storage::storage_check_node($storecfg, $sid, $node);
2856    });
2857 }
2858
2859 # list nodes where all VM images are available (used by has_feature API)
2860 sub shared_nodes {
2861     my ($conf, $storecfg) = @_;
2862
2863     my $nodelist = PVE::Cluster::get_nodelist();
2864     my $nodehash = { map { $_ => 1 } @$nodelist };
2865     my $nodename = PVE::INotify::nodename();
2866
2867     foreach_drive($conf, sub {
2868         my ($ds, $drive) = @_;
2869
2870         my $volid = $drive->{file};
2871         return if !$volid;
2872
2873         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2874         if ($storeid) {
2875             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2876             if ($scfg->{disable}) {
2877                 $nodehash = {};
2878             } elsif (my $avail = $scfg->{nodes}) {
2879                 foreach my $node (keys %$nodehash) {
2880                     delete $nodehash->{$node} if !$avail->{$node};
2881                 }
2882             } elsif (!$scfg->{shared}) {
2883                 foreach my $node (keys %$nodehash) {
2884                     delete $nodehash->{$node} if $node ne $nodename
2885                 }
2886             }
2887         }
2888     });
2889
2890     return $nodehash
2891 }
2892
2893 sub check_local_storage_availability {
2894     my ($conf, $storecfg) = @_;
2895
2896     my $nodelist = PVE::Cluster::get_nodelist();
2897     my $nodehash = { map { $_ => {} } @$nodelist };
2898
2899     foreach_drive($conf, sub {
2900         my ($ds, $drive) = @_;
2901
2902         my $volid = $drive->{file};
2903         return if !$volid;
2904
2905         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2906         if ($storeid) {
2907             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2908
2909             if ($scfg->{disable}) {
2910                 foreach my $node (keys %$nodehash) {
2911                     $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2912                 }
2913             } elsif (my $avail = $scfg->{nodes}) {
2914                 foreach my $node (keys %$nodehash) {
2915                     if (!$avail->{$node}) {
2916                         $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2917                     }
2918                 }
2919             }
2920         }
2921     });
2922
2923     foreach my $node (values %$nodehash) {
2924         if (my $unavail = $node->{unavailable_storages}) {
2925             $node->{unavailable_storages} = [ sort keys %$unavail ];
2926         }
2927     }
2928
2929     return $nodehash
2930 }
2931
2932 # Compat only, use assert_config_exists_on_node and vm_running_locally where possible
2933 sub check_running {
2934     my ($vmid, $nocheck, $node) = @_;
2935
2936     PVE::QemuConfig::assert_config_exists_on_node($vmid, $node) if !$nocheck;
2937     return PVE::QemuServer::Helpers::vm_running_locally($vmid);
2938 }
2939
2940 sub vzlist {
2941
2942     my $vzlist = config_list();
2943
2944     my $fd = IO::Dir->new($PVE::QemuServer::Helpers::var_run_tmpdir) || return $vzlist;
2945
2946     while (defined(my $de = $fd->read)) {
2947         next if $de !~ m/^(\d+)\.pid$/;
2948         my $vmid = $1;
2949         next if !defined($vzlist->{$vmid});
2950         if (my $pid = check_running($vmid)) {
2951             $vzlist->{$vmid}->{pid} = $pid;
2952         }
2953     }
2954
2955     return $vzlist;
2956 }
2957
2958 sub disksize {
2959     my ($storecfg, $conf) = @_;
2960
2961     my $bootdisk = $conf->{bootdisk};
2962     return undef if !$bootdisk;
2963     return undef if !is_valid_drivename($bootdisk);
2964
2965     return undef if !$conf->{$bootdisk};
2966
2967     my $drive = parse_drive($bootdisk, $conf->{$bootdisk});
2968     return undef if !defined($drive);
2969
2970     return undef if drive_is_cdrom($drive);
2971
2972     my $volid = $drive->{file};
2973     return undef if !$volid;
2974
2975     return $drive->{size};
2976 }
2977
2978 our $vmstatus_return_properties = {
2979     vmid => get_standard_option('pve-vmid'),
2980     status => {
2981         description => "Qemu process status.",
2982         type => 'string',
2983         enum => ['stopped', 'running'],
2984     },
2985     maxmem => {
2986         description => "Maximum memory in bytes.",
2987         type => 'integer',
2988         optional => 1,
2989         renderer => 'bytes',
2990     },
2991     maxdisk => {
2992         description => "Root disk size in bytes.",
2993         type => 'integer',
2994         optional => 1,
2995         renderer => 'bytes',
2996     },
2997     name => {
2998         description => "VM name.",
2999         type => 'string',
3000         optional => 1,
3001     },
3002     qmpstatus => {
3003         description => "Qemu QMP agent status.",
3004         type => 'string',
3005         optional => 1,
3006     },
3007     pid => {
3008         description => "PID of running qemu process.",
3009         type => 'integer',
3010         optional => 1,
3011     },
3012     uptime => {
3013         description => "Uptime.",
3014         type => 'integer',
3015         optional => 1,
3016         renderer => 'duration',
3017     },
3018     cpus => {
3019         description => "Maximum usable CPUs.",
3020         type => 'number',
3021         optional => 1,
3022     },
3023     lock => {
3024         description => "The current config lock, if any.",
3025         type => 'string',
3026         optional => 1,
3027     }
3028 };
3029
3030 my $last_proc_pid_stat;
3031
3032 # get VM status information
3033 # This must be fast and should not block ($full == false)
3034 # We only query KVM using QMP if $full == true (this can be slow)
3035 sub vmstatus {
3036     my ($opt_vmid, $full) = @_;
3037
3038     my $res = {};
3039
3040     my $storecfg = PVE::Storage::config();
3041
3042     my $list = vzlist();
3043     my $defaults = load_defaults();
3044
3045     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
3046
3047     my $cpucount = $cpuinfo->{cpus} || 1;
3048
3049     foreach my $vmid (keys %$list) {
3050         next if $opt_vmid && ($vmid ne $opt_vmid);
3051
3052         my $conf = PVE::QemuConfig->load_config($vmid);
3053
3054         my $d = { vmid => $vmid };
3055         $d->{pid} = $list->{$vmid}->{pid};
3056
3057         # fixme: better status?
3058         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
3059
3060         my $size = disksize($storecfg, $conf);
3061         if (defined($size)) {
3062             $d->{disk} = 0; # no info available
3063             $d->{maxdisk} = $size;
3064         } else {
3065             $d->{disk} = 0;
3066             $d->{maxdisk} = 0;
3067         }
3068
3069         $d->{cpus} = ($conf->{sockets} || $defaults->{sockets})
3070             * ($conf->{cores} || $defaults->{cores});
3071         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
3072         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
3073
3074         $d->{name} = $conf->{name} || "VM $vmid";
3075         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024)
3076             : $defaults->{memory}*(1024*1024);
3077
3078         if ($conf->{balloon}) {
3079             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
3080             $d->{shares} = defined($conf->{shares}) ? $conf->{shares}
3081                 : $defaults->{shares};
3082         }
3083
3084         $d->{uptime} = 0;
3085         $d->{cpu} = 0;
3086         $d->{mem} = 0;
3087
3088         $d->{netout} = 0;
3089         $d->{netin} = 0;
3090
3091         $d->{diskread} = 0;
3092         $d->{diskwrite} = 0;
3093
3094         $d->{template} = PVE::QemuConfig->is_template($conf);
3095
3096         $d->{serial} = 1 if conf_has_serial($conf);
3097         $d->{lock} = $conf->{lock} if $conf->{lock};
3098
3099         $res->{$vmid} = $d;
3100     }
3101
3102     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
3103     foreach my $dev (keys %$netdev) {
3104         next if $dev !~ m/^tap([1-9]\d*)i/;
3105         my $vmid = $1;
3106         my $d = $res->{$vmid};
3107         next if !$d;
3108
3109         $d->{netout} += $netdev->{$dev}->{receive};
3110         $d->{netin} += $netdev->{$dev}->{transmit};
3111
3112         if ($full) {
3113             $d->{nics}->{$dev}->{netout} = $netdev->{$dev}->{receive};
3114             $d->{nics}->{$dev}->{netin} = $netdev->{$dev}->{transmit};
3115         }
3116
3117     }
3118
3119     my $ctime = gettimeofday;
3120
3121     foreach my $vmid (keys %$list) {
3122
3123         my $d = $res->{$vmid};
3124         my $pid = $d->{pid};
3125         next if !$pid;
3126
3127         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
3128         next if !$pstat; # not running
3129
3130         my $used = $pstat->{utime} + $pstat->{stime};
3131
3132         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
3133
3134         if ($pstat->{vsize}) {
3135             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
3136         }
3137
3138         my $old = $last_proc_pid_stat->{$pid};
3139         if (!$old) {
3140             $last_proc_pid_stat->{$pid} = {
3141                 time => $ctime,
3142                 used => $used,
3143                 cpu => 0,
3144             };
3145             next;
3146         }
3147
3148         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
3149
3150         if ($dtime > 1000) {
3151             my $dutime = $used -  $old->{used};
3152
3153             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
3154             $last_proc_pid_stat->{$pid} = {
3155                 time => $ctime,
3156                 used => $used,
3157                 cpu => $d->{cpu},
3158             };
3159         } else {
3160             $d->{cpu} = $old->{cpu};
3161         }
3162     }
3163
3164     return $res if !$full;
3165
3166     my $qmpclient = PVE::QMPClient->new();
3167
3168     my $ballooncb = sub {
3169         my ($vmid, $resp) = @_;
3170
3171         my $info = $resp->{'return'};
3172         return if !$info->{max_mem};
3173
3174         my $d = $res->{$vmid};
3175
3176         # use memory assigned to VM
3177         $d->{maxmem} = $info->{max_mem};
3178         $d->{balloon} = $info->{actual};
3179
3180         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
3181             $d->{mem} = $info->{total_mem} - $info->{free_mem};
3182             $d->{freemem} = $info->{free_mem};
3183         }
3184
3185         $d->{ballooninfo} = $info;
3186     };
3187
3188     my $blockstatscb = sub {
3189         my ($vmid, $resp) = @_;
3190         my $data = $resp->{'return'} || [];
3191         my $totalrdbytes = 0;
3192         my $totalwrbytes = 0;
3193
3194         for my $blockstat (@$data) {
3195             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
3196             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
3197
3198             $blockstat->{device} =~ s/drive-//;
3199             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
3200         }
3201         $res->{$vmid}->{diskread} = $totalrdbytes;
3202         $res->{$vmid}->{diskwrite} = $totalwrbytes;
3203     };
3204
3205     my $statuscb = sub {
3206         my ($vmid, $resp) = @_;
3207
3208         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
3209         # this fails if ballon driver is not loaded, so this must be
3210         # the last commnand (following command are aborted if this fails).
3211         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
3212
3213         my $status = 'unknown';
3214         if (!defined($status = $resp->{'return'}->{status})) {
3215             warn "unable to get VM status\n";
3216             return;
3217         }
3218
3219         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
3220     };
3221
3222     foreach my $vmid (keys %$list) {
3223         next if $opt_vmid && ($vmid ne $opt_vmid);
3224         next if !$res->{$vmid}->{pid}; # not running
3225         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
3226     }
3227
3228     $qmpclient->queue_execute(undef, 2);
3229
3230     foreach my $vmid (keys %$list) {
3231         next if $opt_vmid && ($vmid ne $opt_vmid);
3232         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
3233     }
3234
3235     return $res;
3236 }
3237
3238 sub foreach_drive {
3239     my ($conf, $func, @param) = @_;
3240
3241     foreach my $ds (valid_drive_names()) {
3242         next if !defined($conf->{$ds});
3243
3244         my $drive = parse_drive($ds, $conf->{$ds});
3245         next if !$drive;
3246
3247         &$func($ds, $drive, @param);
3248     }
3249 }
3250
3251 sub foreach_volid {
3252     my ($conf, $func, @param) = @_;
3253
3254     my $volhash = {};
3255
3256     my $test_volid = sub {
3257         my ($volid, $is_cdrom, $replicate, $shared, $snapname, $size) = @_;
3258
3259         return if !$volid;
3260
3261         $volhash->{$volid}->{cdrom} //= 1;
3262         $volhash->{$volid}->{cdrom} = 0 if !$is_cdrom;
3263
3264         $volhash->{$volid}->{replicate} //= 0;
3265         $volhash->{$volid}->{replicate} = 1 if $replicate;
3266
3267         $volhash->{$volid}->{shared} //= 0;
3268         $volhash->{$volid}->{shared} = 1 if $shared;
3269
3270         $volhash->{$volid}->{referenced_in_config} //= 0;
3271         $volhash->{$volid}->{referenced_in_config} = 1 if !defined($snapname);
3272
3273         $volhash->{$volid}->{referenced_in_snapshot}->{$snapname} = 1
3274             if defined($snapname);
3275         $volhash->{$volid}->{size} = $size if $size;
3276     };
3277
3278     foreach_drive($conf, sub {
3279         my ($ds, $drive) = @_;
3280         $test_volid->($drive->{file}, drive_is_cdrom($drive), $drive->{replicate} // 1, $drive->{shared}, undef, $drive->{size});
3281     });
3282
3283     foreach my $snapname (keys %{$conf->{snapshots}}) {
3284         my $snap = $conf->{snapshots}->{$snapname};
3285         $test_volid->($snap->{vmstate}, 0, 1, $snapname);
3286         foreach_drive($snap, sub {
3287             my ($ds, $drive) = @_;
3288             $test_volid->($drive->{file}, drive_is_cdrom($drive), $drive->{replicate} // 1, $drive->{shared}, $snapname);
3289         });
3290     }
3291
3292     foreach my $volid (keys %$volhash) {
3293         &$func($volid, $volhash->{$volid}, @param);
3294     }
3295 }
3296
3297 sub conf_has_serial {
3298     my ($conf) = @_;
3299
3300     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3301         if ($conf->{"serial$i"}) {
3302             return 1;
3303         }
3304     }
3305
3306     return 0;
3307 }
3308
3309 sub conf_has_audio {
3310     my ($conf, $id) = @_;
3311
3312     $id //= 0;
3313     my $audio = $conf->{"audio$id"};
3314     return undef if !defined($audio);
3315
3316     my $audioproperties = PVE::JSONSchema::parse_property_string($audio_fmt, $audio);
3317     my $audiodriver = $audioproperties->{driver} // 'spice';
3318
3319     return {
3320         dev => $audioproperties->{device},
3321         dev_id => "audiodev$id",
3322         backend => $audiodriver,
3323         backend_id => "$audiodriver-backend${id}",
3324     };
3325 }
3326
3327 sub vga_conf_has_spice {
3328     my ($vga) = @_;
3329
3330     my $vgaconf = parse_vga($vga);
3331     my $vgatype = $vgaconf->{type};
3332     return 0 if !$vgatype || $vgatype !~ m/^qxl([234])?$/;
3333
3334     return $1 || 1;
3335 }
3336
3337 sub is_native($) {
3338     my ($arch) = @_;
3339     return get_host_arch() eq $arch;
3340 }
3341
3342 my $default_machines = {
3343     x86_64 => 'pc',
3344     aarch64 => 'virt',
3345 };
3346
3347 sub get_basic_machine_info {
3348     my ($conf, $forcemachine) = @_;
3349
3350     my $arch = $conf->{arch} // get_host_arch();
3351     my $machine = $forcemachine || $conf->{machine} || $default_machines->{$arch};
3352     return ($arch, $machine);
3353 }
3354
3355 sub get_ovmf_files($) {
3356     my ($arch) = @_;
3357
3358     my $ovmf = $OVMF->{$arch}
3359         or die "no OVMF images known for architecture '$arch'\n";
3360
3361     return @$ovmf;
3362 }
3363
3364 my $Arch2Qemu = {
3365     aarch64 => '/usr/bin/qemu-system-aarch64',
3366     x86_64 => '/usr/bin/qemu-system-x86_64',
3367 };
3368 sub get_command_for_arch($) {
3369     my ($arch) = @_;
3370     return '/usr/bin/kvm' if is_native($arch);
3371
3372     my $cmd = $Arch2Qemu->{$arch}
3373         or die "don't know how to emulate architecture '$arch'\n";
3374     return $cmd;
3375 }
3376
3377 sub get_cpu_options {
3378     my ($conf, $arch, $kvm, $kvm_off, $machine_version, $winversion, $gpu_passthrough) = @_;
3379
3380     my $cpuFlags = [];
3381     my $ostype = $conf->{ostype};
3382
3383     my $cpu = $kvm ? "kvm64" : "qemu64";
3384     if ($arch eq 'aarch64') {
3385         $cpu = 'cortex-a57';
3386     }
3387     my $hv_vendor_id;
3388     if (my $cputype = $conf->{cpu}) {
3389         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
3390             or die "Cannot parse cpu description: $cputype\n";
3391         $cpu = $cpuconf->{cputype};
3392         $kvm_off = 1 if $cpuconf->{hidden};
3393         $hv_vendor_id = $cpuconf->{'hv-vendor-id'};
3394
3395         if (defined(my $flags = $cpuconf->{flags})) {
3396             push @$cpuFlags, split(";", $flags);
3397         }
3398     }
3399
3400     push @$cpuFlags , '+lahf_lm' if $cpu eq 'kvm64' && $arch eq 'x86_64';
3401
3402     push @$cpuFlags , '-x2apic' if $ostype && $ostype eq 'solaris';
3403
3404     push @$cpuFlags, '+sep' if $cpu eq 'kvm64' || $cpu eq 'kvm32';
3405
3406     push @$cpuFlags, '-rdtscp' if $cpu =~ m/^Opteron/;
3407
3408     if (min_version($machine_version, 2, 3) && $arch eq 'x86_64') {
3409
3410         push @$cpuFlags , '+kvm_pv_unhalt' if $kvm;
3411         push @$cpuFlags , '+kvm_pv_eoi' if $kvm;
3412     }
3413
3414     add_hyperv_enlightenments($cpuFlags, $winversion, $machine_version, $conf->{bios}, $gpu_passthrough, $hv_vendor_id) if $kvm;
3415
3416     push @$cpuFlags, 'enforce' if $cpu ne 'host' && $kvm && $arch eq 'x86_64';
3417
3418     push @$cpuFlags, 'kvm=off' if $kvm_off;
3419
3420     if (my $cpu_vendor = $cpu_vendor_list->{$cpu}) {
3421         push @$cpuFlags, "vendor=${cpu_vendor}"
3422             if $cpu_vendor ne 'default';
3423     } elsif ($arch ne 'aarch64') {
3424         die "internal error"; # should not happen
3425     }
3426
3427     $cpu .= "," . join(',', @$cpuFlags) if scalar(@$cpuFlags);
3428
3429     return ('-cpu', $cpu);
3430 }
3431
3432 sub config_to_command {
3433     my ($storecfg, $vmid, $conf, $defaults, $forcemachine) = @_;
3434
3435     my $cmd = [];
3436     my $globalFlags = [];
3437     my $machineFlags = [];
3438     my $rtcFlags = [];
3439     my $devices = [];
3440     my $pciaddr = '';
3441     my $bridges = {};
3442     my $vernum = 0; # unknown
3443     my $ostype = $conf->{ostype};
3444     my $winversion = windows_version($ostype);
3445     my $kvm = $conf->{kvm};
3446
3447     my ($arch, $machine_type) = get_basic_machine_info($conf, $forcemachine);
3448     my $kvm_binary = get_command_for_arch($arch);
3449     my $kvmver = kvm_user_version($kvm_binary);
3450     my $machine_version = PVE::QemuServer::Machine::extract_version($machine_type) // $kvmver;
3451     $kvm //= 1 if is_native($arch);
3452
3453     if ($kvm) {
3454         die "KVM virtualisation configured, but not available. Either disable in VM configuration or enable in BIOS.\n"
3455             if !defined kvm_version();
3456     }
3457
3458     if ($kvmver =~ m/^(\d+)\.(\d+)$/) {
3459         $vernum = $1*1000000+$2*1000;
3460     } elsif ($kvmver =~ m/^(\d+)\.(\d+)\.(\d+)$/) {
3461         $vernum = $1*1000000+$2*1000+$3;
3462     }
3463
3464     die "detected old qemu-kvm binary ($kvmver)\n" if $vernum < 15000;
3465
3466     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
3467     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
3468     my $use_old_bios_files = undef;
3469     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3470
3471     my $cpuunits = defined($conf->{cpuunits}) ?
3472             $conf->{cpuunits} : $defaults->{cpuunits};
3473
3474     push @$cmd, $kvm_binary;
3475
3476     push @$cmd, '-id', $vmid;
3477
3478     my $vmname = $conf->{name} || "vm$vmid";
3479
3480     push @$cmd, '-name', $vmname;
3481
3482     my $use_virtio = 0;
3483
3484     my $qmpsocket = PVE::QemuServer::Helpers::qmp_socket($vmid);
3485     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server,nowait";
3486     push @$cmd, '-mon', "chardev=qmp,mode=control";
3487
3488     if (min_version($machine_version, 2, 12)) {
3489         push @$cmd, '-chardev', "socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5";
3490         push @$cmd, '-mon', "chardev=qmp-event,mode=control";
3491     }
3492
3493     push @$cmd, '-pidfile' , PVE::QemuServer::Helpers::pidfile_name($vmid);
3494
3495     push @$cmd, '-daemonize';
3496
3497     if ($conf->{smbios1}) {
3498         my $smbios_conf = parse_smbios1($conf->{smbios1});
3499         if ($smbios_conf->{base64}) {
3500             # Do not pass base64 flag to qemu
3501             delete $smbios_conf->{base64};
3502             my $smbios_string = "";
3503             foreach my $key (keys %$smbios_conf) {
3504                 my $value;
3505                 if ($key eq "uuid") {
3506                     $value = $smbios_conf->{uuid}
3507                 } else {
3508                     $value = decode_base64($smbios_conf->{$key});
3509                 }
3510                 # qemu accepts any binary data, only commas need escaping by double comma
3511                 $value =~ s/,/,,/g;
3512                 $smbios_string .= "," . $key . "=" . $value if $value;
3513             }
3514             push @$cmd, '-smbios', "type=1" . $smbios_string;
3515         } else {
3516             push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
3517         }
3518     }
3519
3520     if ($conf->{vmgenid}) {
3521         push @$devices, '-device', 'vmgenid,guid='.$conf->{vmgenid};
3522     }
3523
3524     my ($ovmf_code, $ovmf_vars) = get_ovmf_files($arch);
3525     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3526         die "uefi base image not found\n" if ! -f $ovmf_code;
3527
3528         my $path;
3529         my $format;
3530         if (my $efidisk = $conf->{efidisk0}) {
3531             my $d = PVE::JSONSchema::parse_property_string($efidisk_fmt, $efidisk);
3532             my ($storeid, $volname) = PVE::Storage::parse_volume_id($d->{file}, 1);
3533             $format = $d->{format};
3534             if ($storeid) {
3535                 $path = PVE::Storage::path($storecfg, $d->{file});
3536                 if (!defined($format)) {
3537                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
3538                     $format = qemu_img_format($scfg, $volname);
3539                 }
3540             } else {
3541                 $path = $d->{file};
3542                 die "efidisk format must be specified\n"
3543                     if !defined($format);
3544             }
3545         } else {
3546             warn "no efidisk configured! Using temporary efivars disk.\n";
3547             $path = "/tmp/$vmid-ovmf.fd";
3548             PVE::Tools::file_copy($ovmf_vars, $path, -s $ovmf_vars);
3549             $format = 'raw';
3550         }
3551
3552         push @$cmd, '-drive', "if=pflash,unit=0,format=raw,readonly,file=$ovmf_code";
3553         push @$cmd, '-drive', "if=pflash,unit=1,format=$format,id=drive-efidisk0,file=$path";
3554     }
3555
3556     # load q35 config
3557     if ($q35) {
3558         # we use different pcie-port hardware for qemu >= 4.0 for passthrough
3559         if (min_version($machine_version, 4, 0)) {
3560             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35-4.0.cfg';
3561         } else {
3562             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35.cfg';
3563         }
3564     }
3565
3566     # add usb controllers
3567     my @usbcontrollers = PVE::QemuServer::USB::get_usb_controllers($conf, $bridges, $arch, $machine_type, $usbdesc->{format}, $MAX_USB_DEVICES);
3568     push @$devices, @usbcontrollers if @usbcontrollers;
3569     my $vga = parse_vga($conf->{vga});
3570
3571     my $qxlnum = vga_conf_has_spice($conf->{vga});
3572     $vga->{type} = 'qxl' if $qxlnum;
3573
3574     if (!$vga->{type}) {
3575         if ($arch eq 'aarch64') {
3576             $vga->{type} = 'virtio';
3577         } elsif (min_version($machine_version, 2, 9)) {
3578             $vga->{type} = (!$winversion || $winversion >= 6) ? 'std' : 'cirrus';
3579         } else {
3580             $vga->{type} = ($winversion >= 6) ? 'std' : 'cirrus';
3581         }
3582     }
3583
3584     # enable absolute mouse coordinates (needed by vnc)
3585     my $tablet;
3586     if (defined($conf->{tablet})) {
3587         $tablet = $conf->{tablet};
3588     } else {
3589         $tablet = $defaults->{tablet};
3590         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
3591         $tablet = 0 if $vga->{type} =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
3592     }
3593
3594     if ($tablet) {
3595         push @$devices, '-device', print_tabletdevice_full($conf, $arch) if $tablet;
3596         my $kbd = print_keyboarddevice_full($conf, $arch);
3597         push @$devices, '-device', $kbd if defined($kbd);
3598     }
3599
3600     my $kvm_off = 0;
3601     my $gpu_passthrough;
3602
3603     # host pci devices
3604     for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
3605         my $id = "hostpci$i";
3606         my $d = parse_hostpci($conf->{$id});
3607         next if !$d;
3608
3609         if (my $pcie = $d->{pcie}) {
3610             die "q35 machine model is not enabled" if !$q35;
3611             # win7 wants to have the pcie devices directly on the pcie bus
3612             # instead of in the root port
3613             if ($winversion == 7) {
3614                 $pciaddr = print_pcie_addr("${id}bus0");
3615             } else {
3616                 # add more root ports if needed, 4 are present by default
3617                 # by pve-q35 cfgs, rest added here on demand.
3618                 if ($i > 3) {
3619                     push @$devices, '-device', print_pcie_root_port($i);
3620                 }
3621                 $pciaddr = print_pcie_addr($id);
3622             }
3623         } else {
3624             $pciaddr = print_pci_addr($id, $bridges, $arch, $machine_type);
3625         }
3626
3627         my $xvga = '';
3628         if ($d->{'x-vga'}) {
3629             $xvga = ',x-vga=on' if !($conf->{bios} && $conf->{bios} eq 'ovmf');
3630             $kvm_off = 1;
3631             $vga->{type} = 'none' if !defined($conf->{vga});
3632             $gpu_passthrough = 1;
3633         }
3634
3635         my $pcidevices = $d->{pciid};
3636         my $multifunction = 1 if @$pcidevices > 1;
3637
3638         my $sysfspath;
3639         if ($d->{mdev} && scalar(@$pcidevices) == 1) {
3640             my $pci_id = $pcidevices->[0]->{id};
3641             my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $i);
3642             $sysfspath = "/sys/bus/pci/devices/0000:$pci_id/$uuid";
3643         } elsif ($d->{mdev}) {
3644             warn "ignoring mediated device '$id' with multifunction device\n";
3645         }
3646
3647         my $j=0;
3648         foreach my $pcidevice (@$pcidevices) {
3649             my $devicestr = "vfio-pci";
3650
3651             if ($sysfspath) {
3652                 $devicestr .= ",sysfsdev=$sysfspath";
3653             } else {
3654                 $devicestr .= ",host=$pcidevice->{id}";
3655             }
3656
3657             my $mf_addr = $multifunction ? ".$j" : '';
3658             $devicestr .= ",id=${id}${mf_addr}${pciaddr}${mf_addr}";
3659
3660             if ($j == 0) {
3661                 $devicestr .= ',rombar=0' if defined($d->{rombar}) && !$d->{rombar};
3662                 $devicestr .= "$xvga";
3663                 $devicestr .= ",multifunction=on" if $multifunction;
3664                 $devicestr .= ",romfile=/usr/share/kvm/$d->{romfile}" if $d->{romfile};
3665             }
3666
3667             push @$devices, '-device', $devicestr;
3668             $j++;
3669         }
3670     }
3671
3672     # usb devices
3673     my $usb_dev_features = {};
3674     $usb_dev_features->{spice_usb3} = 1 if min_version($machine_version, 4, 0);
3675
3676     my @usbdevices = PVE::QemuServer::USB::get_usb_devices($conf, $usbdesc->{format}, $MAX_USB_DEVICES, $usb_dev_features);
3677     push @$devices, @usbdevices if @usbdevices;
3678     # serial devices
3679     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3680         if (my $path = $conf->{"serial$i"}) {
3681             if ($path eq 'socket') {
3682                 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
3683                 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server,nowait";
3684                 # On aarch64, serial0 is the UART device. Qemu only allows
3685                 # connecting UART devices via the '-serial' command line, as
3686                 # the device has a fixed slot on the hardware...
3687                 if ($arch eq 'aarch64' && $i == 0) {
3688                     push @$devices, '-serial', "chardev:serial$i";
3689                 } else {
3690                     push @$devices, '-device', "isa-serial,chardev=serial$i";
3691                 }
3692             } else {
3693                 die "no such serial device\n" if ! -c $path;
3694                 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
3695                 push @$devices, '-device', "isa-serial,chardev=serial$i";
3696             }
3697         }
3698     }
3699
3700     # parallel devices
3701     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
3702         if (my $path = $conf->{"parallel$i"}) {
3703             die "no such parallel device\n" if ! -c $path;
3704             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
3705             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
3706             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
3707         }
3708     }
3709
3710     if (my $audio = conf_has_audio($conf)) {
3711
3712         my $audiopciaddr = print_pci_addr("audio0", $bridges, $arch, $machine_type);
3713
3714         my $id = $audio->{dev_id};
3715         if ($audio->{dev} eq 'AC97') {
3716             push @$devices, '-device', "AC97,id=${id}${audiopciaddr}";
3717         } elsif ($audio->{dev} =~ /intel\-hda$/) {
3718             push @$devices, '-device', "$audio->{dev},id=${id}${audiopciaddr}";
3719             push @$devices, '-device', "hda-micro,id=${id}-codec0,bus=${id}.0,cad=0";
3720             push @$devices, '-device', "hda-duplex,id=${id}-codec1,bus=${id}.0,cad=1";
3721         } else {
3722             die "unkown audio device '$audio->{dev}', implement me!";
3723         }
3724
3725         push @$devices, '-audiodev', "$audio->{backend},id=$audio->{backend_id}";
3726     }
3727
3728     my $sockets = 1;
3729     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3730     $sockets = $conf->{sockets} if  $conf->{sockets};
3731
3732     my $cores = $conf->{cores} || 1;
3733
3734     my $maxcpus = $sockets * $cores;
3735
3736     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
3737
3738     my $allowed_vcpus = $cpuinfo->{cpus};
3739
3740     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n"
3741         if ($allowed_vcpus < $maxcpus);
3742
3743     if($hotplug_features->{cpu} && min_version($machine_version, 2, 7)) {
3744
3745         push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3746         for (my $i = 2; $i <= $vcpus; $i++)  {
3747             my $cpustr = print_cpu_device($conf,$i);
3748             push @$cmd, '-device', $cpustr;
3749         }
3750
3751     } else {
3752
3753         push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3754     }
3755     push @$cmd, '-nodefaults';
3756
3757     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
3758
3759     my $bootindex_hash = {};
3760     my $i = 1;
3761     foreach my $o (split(//, $bootorder)) {
3762         $bootindex_hash->{$o} = $i*100;
3763         $i++;
3764     }
3765
3766     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
3767
3768     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
3769
3770     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
3771
3772     if ($vga->{type} && $vga->{type} !~ m/^serial\d+$/ && $vga->{type} ne 'none'){
3773         push @$devices, '-device', print_vga_device($conf, $vga, $arch, $machine_version, $machine_type, undef, $qxlnum, $bridges);
3774         my $socket = PVE::QemuServer::Helpers::vnc_socket($vmid);
3775         push @$cmd,  '-vnc', "unix:$socket,password";
3776     } else {
3777         push @$cmd, '-vga', 'none' if $vga->{type} eq 'none';
3778         push @$cmd, '-nographic';
3779     }
3780
3781     # time drift fix
3782     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
3783
3784     my $useLocaltime = $conf->{localtime};
3785
3786     if ($winversion >= 5) { # windows
3787         $useLocaltime = 1 if !defined($conf->{localtime});
3788
3789         # use time drift fix when acpi is enabled
3790         if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
3791             $tdf = 1 if !defined($conf->{tdf});
3792         }
3793     }
3794
3795     if ($winversion >= 6) {
3796         push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3797         push @$cmd, '-no-hpet';
3798     }
3799
3800     push @$rtcFlags, 'driftfix=slew' if $tdf;
3801
3802     if (!$kvm) {
3803         push @$machineFlags, 'accel=tcg';
3804     }
3805
3806     if ($machine_type) {
3807         push @$machineFlags, "type=${machine_type}";
3808     }
3809
3810     if (($conf->{startdate}) && ($conf->{startdate} ne 'now')) {
3811         push @$rtcFlags, "base=$conf->{startdate}";
3812     } elsif ($useLocaltime) {
3813         push @$rtcFlags, 'base=localtime';
3814     }
3815
3816     push @$cmd, get_cpu_options($conf, $arch, $kvm, $kvm_off, $machine_version, $winversion, $gpu_passthrough);
3817
3818     PVE::QemuServer::Memory::config($conf, $vmid, $sockets, $cores, $defaults, $hotplug_features, $cmd);
3819
3820     push @$cmd, '-S' if $conf->{freeze};
3821
3822     push @$cmd, '-k', $conf->{keyboard} if defined($conf->{keyboard});
3823
3824     my $guest_agent = parse_guest_agent($conf);
3825
3826     if ($guest_agent->{enabled}) {
3827         my $qgasocket = PVE::QemuServer::Helpers::qmp_socket($vmid, 1);
3828         push @$devices, '-chardev', "socket,path=$qgasocket,server,nowait,id=qga0";
3829
3830         if (!$guest_agent->{type} || $guest_agent->{type} eq 'virtio') {
3831             my $pciaddr = print_pci_addr("qga0", $bridges, $arch, $machine_type);
3832             push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3833             push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3834         } elsif ($guest_agent->{type} eq 'isa') {
3835             push @$devices, '-device', "isa-serial,chardev=qga0";
3836         }
3837     }
3838
3839     my $spice_port;
3840
3841     if ($qxlnum) {
3842         if ($qxlnum > 1) {
3843             if ($winversion){
3844                 for(my $i = 1; $i < $qxlnum; $i++){
3845                     push @$devices, '-device', print_vga_device($conf, $vga, $arch, $machine_version, $machine_type, $i, $qxlnum, $bridges);
3846                 }
3847             } else {
3848                 # assume other OS works like Linux
3849                 my ($ram, $vram) = ("134217728", "67108864");
3850                 if ($vga->{memory}) {
3851                     $ram = PVE::Tools::convert_size($qxlnum*4*$vga->{memory}, 'mb' => 'b');
3852                     $vram = PVE::Tools::convert_size($qxlnum*2*$vga->{memory}, 'mb' => 'b');
3853                 }
3854                 push @$cmd, '-global', "qxl-vga.ram_size=$ram";
3855                 push @$cmd, '-global', "qxl-vga.vram_size=$vram";
3856             }
3857         }
3858
3859         my $pciaddr = print_pci_addr("spice", $bridges, $arch, $machine_type);
3860
3861         my $nodename = PVE::INotify::nodename();
3862         my $pfamily = PVE::Tools::get_host_address_family($nodename);
3863         my @nodeaddrs = PVE::Tools::getaddrinfo_all('localhost', family => $pfamily);
3864         die "failed to get an ip address of type $pfamily for 'localhost'\n" if !@nodeaddrs;
3865
3866         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3867         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3868         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3869
3870         my $localhost = PVE::Network::addr_to_ip($nodeaddrs[0]->{addr});
3871         $spice_port = PVE::Tools::next_spice_port($pfamily, $localhost);
3872
3873         my $spice_enhancement = PVE::JSONSchema::parse_property_string($spice_enhancements_fmt, $conf->{spice_enhancements} // '');
3874         if ($spice_enhancement->{foldersharing}) {
3875             push @$devices, '-chardev', "spiceport,id=foldershare,name=org.spice-space.webdav.0";
3876             push @$devices, '-device', "virtserialport,chardev=foldershare,name=org.spice-space.webdav.0";
3877         }
3878
3879         my $spice_opts = "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";
3880         $spice_opts .= ",streaming-video=$spice_enhancement->{videostreaming}" if $spice_enhancement->{videostreaming};
3881         push @$devices, '-spice', "$spice_opts";
3882     }
3883
3884     # enable balloon by default, unless explicitly disabled
3885     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3886         $pciaddr = print_pci_addr("balloon0", $bridges, $arch, $machine_type);
3887         push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
3888     }
3889
3890     if ($conf->{watchdog}) {
3891         my $wdopts = parse_watchdog($conf->{watchdog});
3892         $pciaddr = print_pci_addr("watchdog", $bridges, $arch, $machine_type);
3893         my $watchdog = $wdopts->{model} || 'i6300esb';
3894         push @$devices, '-device', "$watchdog$pciaddr";
3895         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3896     }
3897
3898     my $vollist = [];
3899     my $scsicontroller = {};
3900     my $ahcicontroller = {};
3901     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3902
3903     # Add iscsi initiator name if available
3904     if (my $initiator = get_initiator_name()) {
3905         push @$devices, '-iscsi', "initiator-name=$initiator";
3906     }
3907
3908     foreach_drive($conf, sub {
3909         my ($ds, $drive) = @_;
3910
3911         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3912             push @$vollist, $drive->{file};
3913         }
3914
3915         # ignore efidisk here, already added in bios/fw handling code above
3916         return if $drive->{interface} eq 'efidisk';
3917
3918         $use_virtio = 1 if $ds =~ m/^virtio/;
3919
3920         if (drive_is_cdrom ($drive)) {
3921             if ($bootindex_hash->{d}) {
3922                 $drive->{bootindex} = $bootindex_hash->{d};
3923                 $bootindex_hash->{d} += 1;
3924             }
3925         } else {
3926             if ($bootindex_hash->{c}) {
3927                 $drive->{bootindex} = $bootindex_hash->{c} if $conf->{bootdisk} && ($conf->{bootdisk} eq $ds);
3928                 $bootindex_hash->{c} += 1;
3929             }
3930         }
3931
3932         if($drive->{interface} eq 'virtio'){
3933            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3934         }
3935
3936         if ($drive->{interface} eq 'scsi') {
3937
3938             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
3939
3940             $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges, $arch, $machine_type);
3941             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
3942
3943             my $iothread = '';
3944             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
3945                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3946                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3947             } elsif ($drive->{iothread}) {
3948                 warn "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n";
3949             }
3950
3951             my $queues = '';
3952             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
3953                 $queues = ",num_queues=$drive->{queues}";
3954             }
3955
3956             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues" if !$scsicontroller->{$controller};
3957             $scsicontroller->{$controller}=1;
3958         }
3959
3960         if ($drive->{interface} eq 'sata') {
3961            my $controller = int($drive->{index} / $MAX_SATA_DISKS);
3962            $pciaddr = print_pci_addr("ahci$controller", $bridges, $arch, $machine_type);
3963            push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr" if !$ahcicontroller->{$controller};
3964            $ahcicontroller->{$controller}=1;
3965         }
3966
3967         my $drive_cmd = print_drive_full($storecfg, $vmid, $drive);
3968         push @$devices, '-drive',$drive_cmd;
3969         push @$devices, '-device', print_drivedevice_full($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type);
3970     });
3971
3972     for (my $i = 0; $i < $MAX_NETS; $i++) {
3973          next if !$conf->{"net$i"};
3974          my $d = parse_net($conf->{"net$i"});
3975          next if !$d;
3976
3977          $use_virtio = 1 if $d->{model} eq 'virtio';
3978
3979          if ($bootindex_hash->{n}) {
3980             $d->{bootindex} = $bootindex_hash->{n};
3981             $bootindex_hash->{n} += 1;
3982          }
3983
3984          my $netdevfull = print_netdev_full($vmid, $conf, $arch, $d, "net$i");
3985          push @$devices, '-netdev', $netdevfull;
3986
3987          my $netdevicefull = print_netdevice_full($vmid, $conf, $d, "net$i", $bridges, $use_old_bios_files, $arch, $machine_type);
3988          push @$devices, '-device', $netdevicefull;
3989     }
3990
3991     if ($conf->{ivshmem}) {
3992         my $ivshmem = PVE::JSONSchema::parse_property_string($ivshmem_fmt, $conf->{ivshmem});
3993
3994         my $bus;
3995         if ($q35) {
3996             $bus = print_pcie_addr("ivshmem");
3997         } else {
3998             $bus = print_pci_addr("ivshmem", $bridges, $arch, $machine_type);
3999         }
4000
4001         my $ivshmem_name = $ivshmem->{name} // $vmid;
4002         my $path = '/dev/shm/pve-shm-' . $ivshmem_name;
4003
4004         push @$devices, '-device', "ivshmem-plain,memdev=ivshmem$bus,";
4005         push @$devices, '-object', "memory-backend-file,id=ivshmem,share=on,mem-path=$path,size=$ivshmem->{size}M";
4006     }
4007
4008     if (!$q35) {
4009         # add pci bridges
4010         if (min_version($machine_version, 2, 3)) {
4011            $bridges->{1} = 1;
4012            $bridges->{2} = 1;
4013         }
4014
4015         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
4016
4017         for my $k (sort {$b cmp $a} keys %$bridges) {
4018             $pciaddr = print_pci_addr("pci.$k", undef, $arch, $machine_type);
4019             unshift @$devices, '-device', "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr" if $k > 0;
4020         }
4021     }
4022
4023     push @$cmd, @$devices;
4024     push @$cmd, '-rtc', join(',', @$rtcFlags)
4025         if scalar(@$rtcFlags);
4026     push @$cmd, '-machine', join(',', @$machineFlags)
4027         if scalar(@$machineFlags);
4028     push @$cmd, '-global', join(',', @$globalFlags)
4029         if scalar(@$globalFlags);
4030
4031     if (my $vmstate = $conf->{vmstate}) {
4032         my $statepath = PVE::Storage::path($storecfg, $vmstate);
4033         push @$vollist, $vmstate;
4034         push @$cmd, '-loadstate', $statepath;
4035     }
4036
4037     # add custom args
4038     if ($conf->{args}) {
4039         my $aa = PVE::Tools::split_args($conf->{args});
4040         push @$cmd, @$aa;
4041     }
4042
4043     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
4044 }
4045
4046 sub spice_port {
4047     my ($vmid) = @_;
4048
4049     my $res = mon_cmd($vmid, 'query-spice');
4050
4051     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
4052 }
4053
4054 sub vm_devices_list {
4055     my ($vmid) = @_;
4056
4057     my $res = mon_cmd($vmid, 'query-pci');
4058     my $devices_to_check = [];
4059     my $devices = {};
4060     foreach my $pcibus (@$res) {
4061         push @$devices_to_check, @{$pcibus->{devices}},
4062     }
4063
4064     while (@$devices_to_check) {
4065         my $to_check = [];
4066         for my $d (@$devices_to_check) {
4067             $devices->{$d->{'qdev_id'}} = 1 if $d->{'qdev_id'};
4068             next if !$d->{'pci_bridge'};
4069
4070             $devices->{$d->{'qdev_id'}} += scalar(@{$d->{'pci_bridge'}->{devices}});
4071             push @$to_check, @{$d->{'pci_bridge'}->{devices}};
4072         }
4073         $devices_to_check = $to_check;
4074     }
4075
4076     my $resblock = mon_cmd($vmid, 'query-block');
4077     foreach my $block (@$resblock) {
4078         if($block->{device} =~ m/^drive-(\S+)/){
4079                 $devices->{$1} = 1;
4080         }
4081     }
4082
4083     my $resmice = mon_cmd($vmid, 'query-mice');
4084     foreach my $mice (@$resmice) {
4085         if ($mice->{name} eq 'QEMU HID Tablet') {
4086             $devices->{tablet} = 1;
4087             last;
4088         }
4089     }
4090
4091     # for usb devices there is no query-usb
4092     # but we can iterate over the entries in
4093     # qom-list path=/machine/peripheral
4094     my $resperipheral = mon_cmd($vmid, 'qom-list', path => '/machine/peripheral');
4095     foreach my $per (@$resperipheral) {
4096         if ($per->{name} =~ m/^usb\d+$/) {
4097             $devices->{$per->{name}} = 1;
4098         }
4099     }
4100
4101     return $devices;
4102 }
4103
4104 sub vm_deviceplug {
4105     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4106
4107     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
4108
4109     my $devices_list = vm_devices_list($vmid);
4110     return 1 if defined($devices_list->{$deviceid});
4111
4112     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid, $arch, $machine_type); # add PCI bridge if we need it for the device
4113
4114     if ($deviceid eq 'tablet') {
4115
4116         qemu_deviceadd($vmid, print_tabletdevice_full($conf, $arch));
4117
4118     } elsif ($deviceid eq 'keyboard') {
4119
4120         qemu_deviceadd($vmid, print_keyboarddevice_full($conf, $arch));
4121
4122     } elsif ($deviceid =~ m/^usb(\d+)$/) {
4123
4124         die "usb hotplug currently not reliable\n";
4125         # since we can't reliably hot unplug all added usb devices
4126         # and usb passthrough disables live migration
4127         # we disable usb hotplugging for now
4128         qemu_deviceadd($vmid, PVE::QemuServer::USB::print_usbdevice_full($conf, $deviceid, $device));
4129
4130     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4131
4132         qemu_iothread_add($vmid, $deviceid, $device);
4133
4134         qemu_driveadd($storecfg, $vmid, $device);
4135         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, $arch, $machine_type);
4136
4137         qemu_deviceadd($vmid, $devicefull);
4138         eval { qemu_deviceaddverify($vmid, $deviceid); };
4139         if (my $err = $@) {
4140             eval { qemu_drivedel($vmid, $deviceid); };
4141             warn $@ if $@;
4142             die $err;
4143         }
4144
4145     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4146
4147
4148         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
4149         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
4150         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
4151
4152         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
4153
4154         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
4155             qemu_iothread_add($vmid, $deviceid, $device);
4156             $devicefull .= ",iothread=iothread-$deviceid";
4157         }
4158
4159         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
4160             $devicefull .= ",num_queues=$device->{queues}";
4161         }
4162
4163         qemu_deviceadd($vmid, $devicefull);
4164         qemu_deviceaddverify($vmid, $deviceid);
4165
4166     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4167
4168         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device, $arch, $machine_type);
4169         qemu_driveadd($storecfg, $vmid, $device);
4170
4171         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, $arch, $machine_type);
4172         eval { qemu_deviceadd($vmid, $devicefull); };
4173         if (my $err = $@) {
4174             eval { qemu_drivedel($vmid, $deviceid); };
4175             warn $@ if $@;
4176             die $err;
4177         }
4178
4179     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4180
4181         return undef if !qemu_netdevadd($vmid, $conf, $arch, $device, $deviceid);
4182
4183         my $machine_type = PVE::QemuServer::Machine::qemu_machine_pxe($vmid, $conf);
4184         my $use_old_bios_files = undef;
4185         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
4186
4187         my $netdevicefull = print_netdevice_full($vmid, $conf, $device, $deviceid, undef, $use_old_bios_files, $arch, $machine_type);
4188         qemu_deviceadd($vmid, $netdevicefull);
4189         eval {
4190             qemu_deviceaddverify($vmid, $deviceid);
4191             qemu_set_link_status($vmid, $deviceid, !$device->{link_down});
4192         };
4193         if (my $err = $@) {
4194             eval { qemu_netdevdel($vmid, $deviceid); };
4195             warn $@ if $@;
4196             die $err;
4197         }
4198
4199     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
4200
4201         my $bridgeid = $2;
4202         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
4203         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
4204
4205         qemu_deviceadd($vmid, $devicefull);
4206         qemu_deviceaddverify($vmid, $deviceid);
4207
4208     } else {
4209         die "can't hotplug device '$deviceid'\n";
4210     }
4211
4212     return 1;
4213 }
4214
4215 # fixme: this should raise exceptions on error!
4216 sub vm_deviceunplug {
4217     my ($vmid, $conf, $deviceid) = @_;
4218
4219     my $devices_list = vm_devices_list($vmid);
4220     return 1 if !defined($devices_list->{$deviceid});
4221
4222     die "can't unplug bootdisk" if $conf->{bootdisk} && $conf->{bootdisk} eq $deviceid;
4223
4224     if ($deviceid eq 'tablet' || $deviceid eq 'keyboard') {
4225
4226         qemu_devicedel($vmid, $deviceid);
4227
4228     } elsif ($deviceid =~ m/^usb\d+$/) {
4229
4230         die "usb hotplug currently not reliable\n";
4231         # when unplugging usb devices this way,
4232         # there may be remaining usb controllers/hubs
4233         # so we disable it for now
4234         qemu_devicedel($vmid, $deviceid);
4235         qemu_devicedelverify($vmid, $deviceid);
4236
4237     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4238
4239         qemu_devicedel($vmid, $deviceid);
4240         qemu_devicedelverify($vmid, $deviceid);
4241         qemu_drivedel($vmid, $deviceid);
4242         qemu_iothread_del($conf, $vmid, $deviceid);
4243
4244     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4245
4246         qemu_devicedel($vmid, $deviceid);
4247         qemu_devicedelverify($vmid, $deviceid);
4248         qemu_iothread_del($conf, $vmid, $deviceid);
4249
4250     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4251
4252         qemu_devicedel($vmid, $deviceid);
4253         qemu_drivedel($vmid, $deviceid);
4254         qemu_deletescsihw($conf, $vmid, $deviceid);
4255
4256     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4257
4258         qemu_devicedel($vmid, $deviceid);
4259         qemu_devicedelverify($vmid, $deviceid);
4260         qemu_netdevdel($vmid, $deviceid);
4261
4262     } else {
4263         die "can't unplug device '$deviceid'\n";
4264     }
4265
4266     return 1;
4267 }
4268
4269 sub qemu_deviceadd {
4270     my ($vmid, $devicefull) = @_;
4271
4272     $devicefull = "driver=".$devicefull;
4273     my %options =  split(/[=,]/, $devicefull);
4274
4275     mon_cmd($vmid, "device_add" , %options);
4276 }
4277
4278 sub qemu_devicedel {
4279     my ($vmid, $deviceid) = @_;
4280
4281     my $ret = mon_cmd($vmid, "device_del", id => $deviceid);
4282 }
4283
4284 sub qemu_iothread_add {
4285     my($vmid, $deviceid, $device) = @_;
4286
4287     if ($device->{iothread}) {
4288         my $iothreads = vm_iothreads_list($vmid);
4289         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
4290     }
4291 }
4292
4293 sub qemu_iothread_del {
4294     my($conf, $vmid, $deviceid) = @_;
4295
4296     my $confid = $deviceid;
4297     if ($deviceid =~ m/^(?:virtioscsi|scsihw)(\d+)$/) {
4298         $confid = 'scsi' . $1;
4299     }
4300     my $device = parse_drive($confid, $conf->{$confid});
4301     if ($device->{iothread}) {
4302         my $iothreads = vm_iothreads_list($vmid);
4303         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
4304     }
4305 }
4306
4307 sub qemu_objectadd {
4308     my($vmid, $objectid, $qomtype) = @_;
4309
4310     mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
4311
4312     return 1;
4313 }
4314
4315 sub qemu_objectdel {
4316     my($vmid, $objectid) = @_;
4317
4318     mon_cmd($vmid, "object-del", id => $objectid);
4319
4320     return 1;
4321 }
4322
4323 sub qemu_driveadd {
4324     my ($storecfg, $vmid, $device) = @_;
4325
4326     my $drive = print_drive_full($storecfg, $vmid, $device);
4327     $drive =~ s/\\/\\\\/g;
4328     my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_add auto \"$drive\"");
4329
4330     # If the command succeeds qemu prints: "OK"
4331     return 1 if $ret =~ m/OK/s;
4332
4333     die "adding drive failed: $ret\n";
4334 }
4335
4336 sub qemu_drivedel {
4337     my($vmid, $deviceid) = @_;
4338
4339     my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_del drive-$deviceid");
4340     $ret =~ s/^\s+//;
4341
4342     return 1 if $ret eq "";
4343
4344     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
4345     return 1 if $ret =~ m/Device \'.*?\' not found/s;
4346
4347     die "deleting drive $deviceid failed : $ret\n";
4348 }
4349
4350 sub qemu_deviceaddverify {
4351     my ($vmid, $deviceid) = @_;
4352
4353     for (my $i = 0; $i <= 5; $i++) {
4354          my $devices_list = vm_devices_list($vmid);
4355          return 1 if defined($devices_list->{$deviceid});
4356          sleep 1;
4357     }
4358
4359     die "error on hotplug device '$deviceid'\n";
4360 }
4361
4362
4363 sub qemu_devicedelverify {
4364     my ($vmid, $deviceid) = @_;
4365
4366     # need to verify that the device is correctly removed as device_del
4367     # is async and empty return is not reliable
4368
4369     for (my $i = 0; $i <= 5; $i++) {
4370          my $devices_list = vm_devices_list($vmid);
4371          return 1 if !defined($devices_list->{$deviceid});
4372          sleep 1;
4373     }
4374
4375     die "error on hot-unplugging device '$deviceid'\n";
4376 }
4377
4378 sub qemu_findorcreatescsihw {
4379     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4380
4381     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4382
4383     my $scsihwid="$controller_prefix$controller";
4384     my $devices_list = vm_devices_list($vmid);
4385
4386     if(!defined($devices_list->{$scsihwid})) {
4387         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device, $arch, $machine_type);
4388     }
4389
4390     return 1;
4391 }
4392
4393 sub qemu_deletescsihw {
4394     my ($conf, $vmid, $opt) = @_;
4395
4396     my $device = parse_drive($opt, $conf->{$opt});
4397
4398     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
4399         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
4400         return 1;
4401     }
4402
4403     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4404
4405     my $devices_list = vm_devices_list($vmid);
4406     foreach my $opt (keys %{$devices_list}) {
4407         if (PVE::QemuServer::is_valid_drivename($opt)) {
4408             my $drive = PVE::QemuServer::parse_drive($opt, $conf->{$opt});
4409             if($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
4410                 return 1;
4411             }
4412         }
4413     }
4414
4415     my $scsihwid="scsihw$controller";
4416
4417     vm_deviceunplug($vmid, $conf, $scsihwid);
4418
4419     return 1;
4420 }
4421
4422 sub qemu_add_pci_bridge {
4423     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4424
4425     my $bridges = {};
4426
4427     my $bridgeid;
4428
4429     print_pci_addr($device, $bridges, $arch, $machine_type);
4430
4431     while (my ($k, $v) = each %$bridges) {
4432         $bridgeid = $k;
4433     }
4434     return 1 if !defined($bridgeid) || $bridgeid < 1;
4435
4436     my $bridge = "pci.$bridgeid";
4437     my $devices_list = vm_devices_list($vmid);
4438
4439     if (!defined($devices_list->{$bridge})) {
4440         vm_deviceplug($storecfg, $conf, $vmid, $bridge, $arch, $machine_type);
4441     }
4442
4443     return 1;
4444 }
4445
4446 sub qemu_set_link_status {
4447     my ($vmid, $device, $up) = @_;
4448
4449     mon_cmd($vmid, "set_link", name => $device,
4450                up => $up ? JSON::true : JSON::false);
4451 }
4452
4453 sub qemu_netdevadd {
4454     my ($vmid, $conf, $arch, $device, $deviceid) = @_;
4455
4456     my $netdev = print_netdev_full($vmid, $conf, $arch, $device, $deviceid, 1);
4457     my %options =  split(/[=,]/, $netdev);
4458
4459     mon_cmd($vmid, "netdev_add",  %options);
4460     return 1;
4461 }
4462
4463 sub qemu_netdevdel {
4464     my ($vmid, $deviceid) = @_;
4465
4466     mon_cmd($vmid, "netdev_del", id => $deviceid);
4467 }
4468
4469 sub qemu_usb_hotplug {
4470     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4471
4472     return if !$device;
4473
4474     # remove the old one first
4475     vm_deviceunplug($vmid, $conf, $deviceid);
4476
4477     # check if xhci controller is necessary and available
4478     if ($device->{usb3}) {
4479
4480         my $devicelist = vm_devices_list($vmid);
4481
4482         if (!$devicelist->{xhci}) {
4483             my $pciaddr = print_pci_addr("xhci", undef, $arch, $machine_type);
4484             qemu_deviceadd($vmid, "nec-usb-xhci,id=xhci$pciaddr");
4485         }
4486     }
4487     my $d = parse_usb_device($device->{host});
4488     $d->{usb3} = $device->{usb3};
4489
4490     # add the new one
4491     vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $d, $arch, $machine_type);
4492 }
4493
4494 sub qemu_cpu_hotplug {
4495     my ($vmid, $conf, $vcpus) = @_;
4496
4497     my $machine_type = PVE::QemuServer::Machine::get_current_qemu_machine($vmid);
4498
4499     my $sockets = 1;
4500     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
4501     $sockets = $conf->{sockets} if  $conf->{sockets};
4502     my $cores = $conf->{cores} || 1;
4503     my $maxcpus = $sockets * $cores;
4504
4505     $vcpus = $maxcpus if !$vcpus;
4506
4507     die "you can't add more vcpus than maxcpus\n"
4508         if $vcpus > $maxcpus;
4509
4510     my $currentvcpus = $conf->{vcpus} || $maxcpus;
4511
4512     if ($vcpus < $currentvcpus) {
4513
4514         if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4515
4516             for (my $i = $currentvcpus; $i > $vcpus; $i--) {
4517                 qemu_devicedel($vmid, "cpu$i");
4518                 my $retry = 0;
4519                 my $currentrunningvcpus = undef;
4520                 while (1) {
4521                     $currentrunningvcpus = mon_cmd($vmid, "query-cpus");
4522                     last if scalar(@{$currentrunningvcpus}) == $i-1;
4523                     raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
4524                     $retry++;
4525                     sleep 1;
4526                 }
4527                 #update conf after each succesfull cpu unplug
4528                 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4529                 PVE::QemuConfig->write_config($vmid, $conf);
4530             }
4531         } else {
4532             die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
4533         }
4534
4535         return;
4536     }
4537
4538     my $currentrunningvcpus = mon_cmd($vmid, "query-cpus");
4539     die "vcpus in running vm does not match its configuration\n"
4540         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
4541
4542     if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4543
4544         for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
4545             my $cpustr = print_cpu_device($conf, $i);
4546             qemu_deviceadd($vmid, $cpustr);
4547
4548             my $retry = 0;
4549             my $currentrunningvcpus = undef;
4550             while (1) {
4551                 $currentrunningvcpus = mon_cmd($vmid, "query-cpus");
4552                 last if scalar(@{$currentrunningvcpus}) == $i;
4553                 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
4554                 sleep 1;
4555                 $retry++;
4556             }
4557             #update conf after each succesfull cpu hotplug
4558             $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4559             PVE::QemuConfig->write_config($vmid, $conf);
4560         }
4561     } else {
4562
4563         for (my $i = $currentvcpus; $i < $vcpus; $i++) {
4564             mon_cmd($vmid, "cpu-add", id => int($i));
4565         }
4566     }
4567 }
4568
4569 sub qemu_block_set_io_throttle {
4570     my ($vmid, $deviceid,
4571         $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
4572         $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max,
4573         $bps_max_length, $bps_rd_max_length, $bps_wr_max_length,
4574         $iops_max_length, $iops_rd_max_length, $iops_wr_max_length) = @_;
4575
4576     return if !check_running($vmid) ;
4577
4578     mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
4579         bps => int($bps),
4580         bps_rd => int($bps_rd),
4581         bps_wr => int($bps_wr),
4582         iops => int($iops),
4583         iops_rd => int($iops_rd),
4584         iops_wr => int($iops_wr),
4585         bps_max => int($bps_max),
4586         bps_rd_max => int($bps_rd_max),
4587         bps_wr_max => int($bps_wr_max),
4588         iops_max => int($iops_max),
4589         iops_rd_max => int($iops_rd_max),
4590         iops_wr_max => int($iops_wr_max),
4591         bps_max_length => int($bps_max_length),
4592         bps_rd_max_length => int($bps_rd_max_length),
4593         bps_wr_max_length => int($bps_wr_max_length),
4594         iops_max_length => int($iops_max_length),
4595         iops_rd_max_length => int($iops_rd_max_length),
4596         iops_wr_max_length => int($iops_wr_max_length),
4597     );
4598
4599 }
4600
4601 # old code, only used to shutdown old VM after update
4602 sub __read_avail {
4603     my ($fh, $timeout) = @_;
4604
4605     my $sel = new IO::Select;
4606     $sel->add($fh);
4607
4608     my $res = '';
4609     my $buf;
4610
4611     my @ready;
4612     while (scalar (@ready = $sel->can_read($timeout))) {
4613         my $count;
4614         if ($count = $fh->sysread($buf, 8192)) {
4615             if ($buf =~ /^(.*)\(qemu\) $/s) {
4616                 $res .= $1;
4617                 last;
4618             } else {
4619                 $res .= $buf;
4620             }
4621         } else {
4622             if (!defined($count)) {
4623                 die "$!\n";
4624             }
4625             last;
4626         }
4627     }
4628
4629     die "monitor read timeout\n" if !scalar(@ready);
4630
4631     return $res;
4632 }
4633
4634 sub qemu_block_resize {
4635     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4636
4637     my $running = check_running($vmid);
4638
4639     $size = 0 if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4640
4641     return if !$running;
4642
4643     mon_cmd($vmid, "block_resize", device => $deviceid, size => int($size));
4644
4645 }
4646
4647 sub qemu_volume_snapshot {
4648     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4649
4650     my $running = check_running($vmid);
4651
4652     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4653         mon_cmd($vmid, 'blockdev-snapshot-internal-sync', device => $deviceid, name => $snap);
4654     } else {
4655         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4656     }
4657 }
4658
4659 sub qemu_volume_snapshot_delete {
4660     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4661
4662     my $running = check_running($vmid);
4663
4664     if($running) {
4665
4666         $running = undef;
4667         my $conf = PVE::QemuConfig->load_config($vmid);
4668         foreach_drive($conf, sub {
4669             my ($ds, $drive) = @_;
4670             $running = 1 if $drive->{file} eq $volid;
4671         });
4672     }
4673
4674     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4675         mon_cmd($vmid, 'blockdev-snapshot-delete-internal-sync', device => $deviceid, name => $snap);
4676     } else {
4677         PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4678     }
4679 }
4680
4681 sub set_migration_caps {
4682     my ($vmid) = @_;
4683
4684     my $cap_ref = [];
4685
4686     my $enabled_cap = {
4687         "auto-converge" => 1,
4688         "xbzrle" => 1,
4689         "x-rdma-pin-all" => 0,
4690         "zero-blocks" => 0,
4691         "compress" => 0
4692     };
4693
4694     my $supported_capabilities = mon_cmd($vmid, "query-migrate-capabilities");
4695
4696     for my $supported_capability (@$supported_capabilities) {
4697         push @$cap_ref, {
4698             capability => $supported_capability->{capability},
4699             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4700         };
4701     }
4702
4703     mon_cmd($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4704 }
4705
4706 my $fast_plug_option = {
4707     'lock' => 1,
4708     'name' => 1,
4709     'onboot' => 1,
4710     'shares' => 1,
4711     'startup' => 1,
4712     'description' => 1,
4713     'protection' => 1,
4714     'vmstatestorage' => 1,
4715     'hookscript' => 1,
4716 };
4717
4718 # hotplug changes in [PENDING]
4719 # $selection hash can be used to only apply specified options, for
4720 # example: { cores => 1 } (only apply changed 'cores')
4721 # $errors ref is used to return error messages
4722 sub vmconfig_hotplug_pending {
4723     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4724
4725     my $defaults = load_defaults();
4726     my ($arch, $machine_type) = get_basic_machine_info($conf, undef);
4727
4728     # commit values which do not have any impact on running VM first
4729     # Note: those option cannot raise errors, we we do not care about
4730     # $selection and always apply them.
4731
4732     my $add_error = sub {
4733         my ($opt, $msg) = @_;
4734         $errors->{$opt} = "hotplug problem - $msg";
4735     };
4736
4737     my $changes = 0;
4738     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4739         if ($fast_plug_option->{$opt}) {
4740             $conf->{$opt} = $conf->{pending}->{$opt};
4741             delete $conf->{pending}->{$opt};
4742             $changes = 1;
4743         }
4744     }
4745
4746     if ($changes) {
4747         PVE::QemuConfig->write_config($vmid, $conf);
4748         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4749     }
4750
4751     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4752
4753     my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4754     foreach my $opt (sort keys %$pending_delete_hash) {
4755         next if $selection && !$selection->{$opt};
4756         my $force = $pending_delete_hash->{$opt}->{force};
4757         eval {
4758             if ($opt eq 'hotplug') {
4759                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
4760             } elsif ($opt eq 'tablet') {
4761                 die "skip\n" if !$hotplug_features->{usb};
4762                 if ($defaults->{tablet}) {
4763                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4764                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4765                         if $arch eq 'aarch64';
4766                 } else {
4767                     vm_deviceunplug($vmid, $conf, 'tablet');
4768                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4769                 }
4770             } elsif ($opt =~ m/^usb\d+/) {
4771                 die "skip\n";
4772                 # since we cannot reliably hot unplug usb devices
4773                 # we are disabling it
4774                 die "skip\n" if !$hotplug_features->{usb} || $conf->{$opt} =~ m/spice/i;
4775                 vm_deviceunplug($vmid, $conf, $opt);
4776             } elsif ($opt eq 'vcpus') {
4777                 die "skip\n" if !$hotplug_features->{cpu};
4778                 qemu_cpu_hotplug($vmid, $conf, undef);
4779             } elsif ($opt eq 'balloon') {
4780                 # enable balloon device is not hotpluggable
4781                 die "skip\n" if defined($conf->{balloon}) && $conf->{balloon} == 0;
4782                 # here we reset the ballooning value to memory
4783                 my $balloon = $conf->{memory} || $defaults->{memory};
4784                 mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4785             } elsif ($fast_plug_option->{$opt}) {
4786                 # do nothing
4787             } elsif ($opt =~ m/^net(\d+)$/) {
4788                 die "skip\n" if !$hotplug_features->{network};
4789                 vm_deviceunplug($vmid, $conf, $opt);
4790             } elsif (is_valid_drivename($opt)) {
4791                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4792                 vm_deviceunplug($vmid, $conf, $opt);
4793                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4794             } elsif ($opt =~ m/^memory$/) {
4795                 die "skip\n" if !$hotplug_features->{memory};
4796                 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
4797             } elsif ($opt eq 'cpuunits') {
4798                 cgroups_write("cpu", $vmid, "cpu.shares", $defaults->{cpuunits});
4799             } elsif ($opt eq 'cpulimit') {
4800                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", -1);
4801             } else {
4802                 die "skip\n";
4803             }
4804         };
4805         if (my $err = $@) {
4806             &$add_error($opt, $err) if $err ne "skip\n";
4807         } else {
4808             # save new config if hotplug was successful
4809             delete $conf->{$opt};
4810             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4811             PVE::QemuConfig->write_config($vmid, $conf);
4812             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4813         }
4814     }
4815
4816     my ($apply_pending_cloudinit, $apply_pending_cloudinit_done);
4817     $apply_pending_cloudinit = sub {
4818         return if $apply_pending_cloudinit_done; # once is enough
4819         $apply_pending_cloudinit_done = 1; # once is enough
4820
4821         my ($key, $value) = @_;
4822
4823         my @cloudinit_opts = keys %$confdesc_cloudinit;
4824         foreach my $opt (keys %{$conf->{pending}}) {
4825             next if !grep { $_ eq $opt } @cloudinit_opts;
4826             $conf->{$opt} = delete $conf->{pending}->{$opt};
4827         }
4828
4829         my $new_conf = { %$conf };
4830         $new_conf->{$key} = $value;
4831         PVE::QemuServer::Cloudinit::generate_cloudinitconfig($new_conf, $vmid);
4832     };
4833
4834     foreach my $opt (keys %{$conf->{pending}}) {
4835         next if $selection && !$selection->{$opt};
4836         my $value = $conf->{pending}->{$opt};
4837         eval {
4838             if ($opt eq 'hotplug') {
4839                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
4840             } elsif ($opt eq 'tablet') {
4841                 die "skip\n" if !$hotplug_features->{usb};
4842                 if ($value == 1) {
4843                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4844                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4845                         if $arch eq 'aarch64';
4846                 } elsif ($value == 0) {
4847                     vm_deviceunplug($vmid, $conf, 'tablet');
4848                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4849                 }
4850             } elsif ($opt =~ m/^usb\d+$/) {
4851                 die "skip\n";
4852                 # since we cannot reliably hot unplug usb devices
4853                 # we are disabling it
4854                 die "skip\n" if !$hotplug_features->{usb} || $value =~ m/spice/i;
4855                 my $d = eval { PVE::JSONSchema::parse_property_string($usbdesc->{format}, $value) };
4856                 die "skip\n" if !$d;
4857                 qemu_usb_hotplug($storecfg, $conf, $vmid, $opt, $d, $arch, $machine_type);
4858             } elsif ($opt eq 'vcpus') {
4859                 die "skip\n" if !$hotplug_features->{cpu};
4860                 qemu_cpu_hotplug($vmid, $conf, $value);
4861             } elsif ($opt eq 'balloon') {
4862                 # enable/disable balloning device is not hotpluggable
4863                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
4864                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
4865                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
4866
4867                 # allow manual ballooning if shares is set to zero
4868                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
4869                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
4870                     mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4871                 }
4872             } elsif ($opt =~ m/^net(\d+)$/) {
4873                 # some changes can be done without hotplug
4874                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
4875                                     $vmid, $opt, $value, $arch, $machine_type);
4876             } elsif (is_valid_drivename($opt)) {
4877                 # some changes can be done without hotplug
4878                 my $drive = parse_drive($opt, $value);
4879                 if (drive_is_cloudinit($drive)) {
4880                     &$apply_pending_cloudinit($opt, $value);
4881                 }
4882                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
4883                                      $vmid, $opt, $value, 1, $arch, $machine_type);
4884             } elsif ($opt =~ m/^memory$/) { #dimms
4885                 die "skip\n" if !$hotplug_features->{memory};
4886                 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
4887             } elsif ($opt eq 'cpuunits') {
4888                 cgroups_write("cpu", $vmid, "cpu.shares", $conf->{pending}->{$opt});
4889             } elsif ($opt eq 'cpulimit') {
4890                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
4891                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", $cpulimit);
4892             } else {
4893                 die "skip\n";  # skip non-hot-pluggable options
4894             }
4895         };
4896         if (my $err = $@) {
4897             &$add_error($opt, $err) if $err ne "skip\n";
4898         } else {
4899             # save new config if hotplug was successful
4900             $conf->{$opt} = $value;
4901             delete $conf->{pending}->{$opt};
4902             PVE::QemuConfig->write_config($vmid, $conf);
4903             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4904         }
4905     }
4906 }
4907
4908 sub try_deallocate_drive {
4909     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
4910
4911     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
4912         my $volid = $drive->{file};
4913         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
4914             my $sid = PVE::Storage::parse_volume_id($volid);
4915             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
4916
4917             # check if the disk is really unused
4918             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
4919                 if is_volume_in_use($storecfg, $conf, $key, $volid);
4920             PVE::Storage::vdisk_free($storecfg, $volid);
4921             return 1;
4922         } else {
4923             # If vm is not owner of this disk remove from config
4924             return 1;
4925         }
4926     }
4927
4928     return undef;
4929 }
4930
4931 sub vmconfig_delete_or_detach_drive {
4932     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
4933
4934     my $drive = parse_drive($opt, $conf->{$opt});
4935
4936     my $rpcenv = PVE::RPCEnvironment::get();
4937     my $authuser = $rpcenv->get_user();
4938
4939     if ($force) {
4940         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
4941         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
4942     } else {
4943         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
4944     }
4945 }
4946
4947
4948
4949 sub vmconfig_apply_pending {
4950     my ($vmid, $conf, $storecfg) = @_;
4951
4952     # cold plug
4953
4954     my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4955     foreach my $opt (sort keys %$pending_delete_hash) {
4956         die "internal error" if $opt =~ m/^unused/;
4957         my $force = $pending_delete_hash->{$opt}->{force};
4958         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4959         if (!defined($conf->{$opt})) {
4960             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4961             PVE::QemuConfig->write_config($vmid, $conf);
4962         } elsif (is_valid_drivename($opt)) {
4963             vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4964             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4965             delete $conf->{$opt};
4966             PVE::QemuConfig->write_config($vmid, $conf);
4967         } else {
4968             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4969             delete $conf->{$opt};
4970             PVE::QemuConfig->write_config($vmid, $conf);
4971         }
4972     }
4973
4974     $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4975
4976     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4977         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4978
4979         if (defined($conf->{$opt}) && ($conf->{$opt} eq $conf->{pending}->{$opt})) {
4980             # skip if nothing changed
4981         } elsif (is_valid_drivename($opt)) {
4982             vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
4983                 if defined($conf->{$opt});
4984             $conf->{$opt} = $conf->{pending}->{$opt};
4985         } else {
4986             $conf->{$opt} = $conf->{pending}->{$opt};
4987         }
4988
4989         delete $conf->{pending}->{$opt};
4990         PVE::QemuConfig->write_config($vmid, $conf);
4991     }
4992 }
4993
4994 my $safe_num_ne = sub {
4995     my ($a, $b) = @_;
4996
4997     return 0 if !defined($a) && !defined($b);
4998     return 1 if !defined($a);
4999     return 1 if !defined($b);
5000
5001     return $a != $b;
5002 };
5003
5004 my $safe_string_ne = sub {
5005     my ($a, $b) = @_;
5006
5007     return 0 if !defined($a) && !defined($b);
5008     return 1 if !defined($a);
5009     return 1 if !defined($b);
5010
5011     return $a ne $b;
5012 };
5013
5014 sub vmconfig_update_net {
5015     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
5016
5017     my $newnet = parse_net($value);
5018
5019     if ($conf->{$opt}) {
5020         my $oldnet = parse_net($conf->{$opt});
5021
5022         if (&$safe_string_ne($oldnet->{model}, $newnet->{model}) ||
5023             &$safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
5024             &$safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
5025             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
5026
5027             # for non online change, we try to hot-unplug
5028             die "skip\n" if !$hotplug;
5029             vm_deviceunplug($vmid, $conf, $opt);
5030         } else {
5031
5032             die "internal error" if $opt !~ m/net(\d+)/;
5033             my $iface = "tap${vmid}i$1";
5034
5035             if (&$safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
5036                 &$safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
5037                 &$safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
5038                 &$safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
5039                 PVE::Network::tap_unplug($iface);
5040                 PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
5041             } elsif (&$safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
5042                 # Rate can be applied on its own but any change above needs to
5043                 # include the rate in tap_plug since OVS resets everything.
5044                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
5045             }
5046
5047             if (&$safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
5048                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
5049             }
5050
5051             return 1;
5052         }
5053     }
5054
5055     if ($hotplug) {
5056         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet, $arch, $machine_type);
5057     } else {
5058         die "skip\n";
5059     }
5060 }
5061
5062 sub vmconfig_update_disk {
5063     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $force, $arch, $machine_type) = @_;
5064
5065     # fixme: do we need force?
5066
5067     my $drive = parse_drive($opt, $value);
5068
5069     if ($conf->{$opt}) {
5070
5071         if (my $old_drive = parse_drive($opt, $conf->{$opt}))  {
5072
5073             my $media = $drive->{media} || 'disk';
5074             my $oldmedia = $old_drive->{media} || 'disk';
5075             die "unable to change media type\n" if $media ne $oldmedia;
5076
5077             if (!drive_is_cdrom($old_drive)) {
5078
5079                 if ($drive->{file} ne $old_drive->{file}) {
5080
5081                     die "skip\n" if !$hotplug;
5082
5083                     # unplug and register as unused
5084                     vm_deviceunplug($vmid, $conf, $opt);
5085                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
5086
5087                 } else {
5088                     # update existing disk
5089
5090                     # skip non hotpluggable value
5091                     if (&$safe_string_ne($drive->{discard}, $old_drive->{discard}) ||
5092                         &$safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
5093                         &$safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
5094                         &$safe_string_ne($drive->{cache}, $old_drive->{cache})) {
5095                         die "skip\n";
5096                     }
5097
5098                     # apply throttle
5099                     if (&$safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
5100                         &$safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
5101                         &$safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
5102                         &$safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
5103                         &$safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
5104                         &$safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
5105                         &$safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
5106                         &$safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
5107                         &$safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
5108                         &$safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
5109                         &$safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
5110                         &$safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max}) ||
5111                         &$safe_num_ne($drive->{bps_max_length}, $old_drive->{bps_max_length}) ||
5112                         &$safe_num_ne($drive->{bps_rd_max_length}, $old_drive->{bps_rd_max_length}) ||
5113                         &$safe_num_ne($drive->{bps_wr_max_length}, $old_drive->{bps_wr_max_length}) ||
5114                         &$safe_num_ne($drive->{iops_max_length}, $old_drive->{iops_max_length}) ||
5115                         &$safe_num_ne($drive->{iops_rd_max_length}, $old_drive->{iops_rd_max_length}) ||
5116                         &$safe_num_ne($drive->{iops_wr_max_length}, $old_drive->{iops_wr_max_length})) {
5117
5118                         qemu_block_set_io_throttle($vmid,"drive-$opt",
5119                                                    ($drive->{mbps} || 0)*1024*1024,
5120                                                    ($drive->{mbps_rd} || 0)*1024*1024,
5121                                                    ($drive->{mbps_wr} || 0)*1024*1024,
5122                                                    $drive->{iops} || 0,
5123                                                    $drive->{iops_rd} || 0,
5124                                                    $drive->{iops_wr} || 0,
5125                                                    ($drive->{mbps_max} || 0)*1024*1024,
5126                                                    ($drive->{mbps_rd_max} || 0)*1024*1024,
5127                                                    ($drive->{mbps_wr_max} || 0)*1024*1024,
5128                                                    $drive->{iops_max} || 0,
5129                                                    $drive->{iops_rd_max} || 0,
5130                                                    $drive->{iops_wr_max} || 0,
5131                                                    $drive->{bps_max_length} || 1,
5132                                                    $drive->{bps_rd_max_length} || 1,
5133                                                    $drive->{bps_wr_max_length} || 1,
5134                                                    $drive->{iops_max_length} || 1,
5135                                                    $drive->{iops_rd_max_length} || 1,
5136                                                    $drive->{iops_wr_max_length} || 1);
5137
5138                     }
5139
5140                     return 1;
5141                 }
5142
5143             } else { # cdrom
5144
5145                 if ($drive->{file} eq 'none') {
5146                     mon_cmd($vmid, "eject",force => JSON::true,device => "drive-$opt");
5147                     if (drive_is_cloudinit($old_drive)) {
5148                         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive);
5149                     }
5150                 } else {
5151                     my $path = get_iso_path($storecfg, $vmid, $drive->{file});
5152                     mon_cmd($vmid, "eject", force => JSON::true,device => "drive-$opt"); # force eject if locked
5153                     mon_cmd($vmid, "change", device => "drive-$opt",target => "$path") if $path;
5154                 }
5155
5156                 return 1;
5157             }
5158         }
5159     }
5160
5161     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
5162     # hotplug new disks
5163     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
5164     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive, $arch, $machine_type);
5165 }
5166
5167 sub vm_start {
5168     my ($storecfg, $vmid, $statefile, $skiplock, $migratedfrom, $paused,
5169         $forcemachine, $spice_ticket, $migration_network, $migration_type, $targetstorage) = @_;
5170
5171     PVE::QemuConfig->lock_config($vmid, sub {
5172         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
5173
5174         die "you can't start a vm if it's a template\n" if PVE::QemuConfig->is_template($conf);
5175
5176         my $is_suspended = PVE::QemuConfig->has_lock($conf, 'suspended');
5177
5178         PVE::QemuConfig->check_lock($conf)
5179             if !($skiplock || $is_suspended);
5180
5181         die "VM $vmid already running\n" if check_running($vmid, undef, $migratedfrom);
5182
5183         # clean up leftover reboot request files
5184         eval { clear_reboot_request($vmid); };
5185         warn $@ if $@;
5186
5187         if (!$statefile && scalar(keys %{$conf->{pending}})) {
5188             vmconfig_apply_pending($vmid, $conf, $storecfg);
5189             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5190         }
5191
5192         PVE::QemuServer::Cloudinit::generate_cloudinitconfig($conf, $vmid);
5193
5194         my $defaults = load_defaults();
5195
5196         # set environment variable useful inside network script
5197         $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
5198
5199         my $local_volumes = {};
5200
5201         if ($targetstorage) {
5202             foreach_drive($conf, sub {
5203                 my ($ds, $drive) = @_;
5204
5205                 return if drive_is_cdrom($drive);
5206
5207                 my $volid = $drive->{file};
5208
5209                 return if !$volid;
5210
5211                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
5212
5213                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5214                 return if $scfg->{shared};
5215                 $local_volumes->{$ds} = [$volid, $storeid, $volname];
5216             });
5217
5218             my $format = undef;
5219
5220             foreach my $opt (sort keys %$local_volumes) {
5221
5222                 my ($volid, $storeid, $volname) = @{$local_volumes->{$opt}};
5223                 my $drive = parse_drive($opt, $conf->{$opt});
5224
5225                 #if remote storage is specified, use default format
5226                 if ($targetstorage && $targetstorage ne "1") {
5227                     $storeid = $targetstorage;
5228                     my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
5229                     $format = $defFormat;
5230                 } else {
5231                     #else we use same format than original
5232                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5233                     $format = qemu_img_format($scfg, $volid);
5234                 }
5235
5236                 my $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $format, undef, ($drive->{size}/1024));
5237                 my $newdrive = $drive;
5238                 $newdrive->{format} = $format;
5239                 $newdrive->{file} = $newvolid;
5240                 my $drivestr = PVE::QemuServer::print_drive($vmid, $newdrive);
5241                 $local_volumes->{$opt} = $drivestr;
5242                 #pass drive to conf for command line
5243                 $conf->{$opt} = $drivestr;
5244             }
5245         }
5246
5247         PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-start', 1);
5248
5249         if ($is_suspended) {
5250             # enforce machine type on suspended vm to ensure HW compatibility
5251             $forcemachine = $conf->{runningmachine};
5252             print "Resuming suspended VM\n";
5253         }
5254
5255         my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine);
5256
5257         my $migration_ip;
5258         my $get_migration_ip = sub {
5259             my ($cidr, $nodename) = @_;
5260
5261             return $migration_ip if defined($migration_ip);
5262
5263             if (!defined($cidr)) {
5264                 my $dc_conf = PVE::Cluster::cfs_read_file('datacenter.cfg');
5265                 $cidr = $dc_conf->{migration}->{network};
5266             }
5267
5268             if (defined($cidr)) {
5269                 my $ips = PVE::Network::get_local_ip_from_cidr($cidr);
5270
5271                 die "could not get IP: no address configured on local " .
5272                     "node for network '$cidr'\n" if scalar(@$ips) == 0;
5273
5274                 die "could not get IP: multiple addresses configured on local " .
5275                     "node for network '$cidr'\n" if scalar(@$ips) > 1;
5276
5277                 $migration_ip = @$ips[0];
5278             }
5279
5280             $migration_ip = PVE::Cluster::remote_node_ip($nodename, 1)
5281                 if !defined($migration_ip);
5282
5283             return $migration_ip;
5284         };
5285
5286         my $migrate_uri;
5287         if ($statefile) {
5288             if ($statefile eq 'tcp') {
5289                 my $localip = "localhost";
5290                 my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
5291                 my $nodename = PVE::INotify::nodename();
5292
5293                 if (!defined($migration_type)) {
5294                     if (defined($datacenterconf->{migration}->{type})) {
5295                         $migration_type = $datacenterconf->{migration}->{type};
5296                     } else {
5297                         $migration_type = 'secure';
5298                     }
5299                 }
5300
5301                 if ($migration_type eq 'insecure') {
5302                     $localip = $get_migration_ip->($migration_network, $nodename);
5303                     $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5304                 }
5305
5306                 my $pfamily = PVE::Tools::get_host_address_family($nodename);
5307                 my $migrate_port = PVE::Tools::next_migrate_port($pfamily);
5308                 $migrate_uri = "tcp:${localip}:${migrate_port}";
5309                 push @$cmd, '-incoming', $migrate_uri;
5310                 push @$cmd, '-S';
5311
5312             } elsif ($statefile eq 'unix') {
5313                 # should be default for secure migrations as a ssh TCP forward
5314                 # tunnel is not deterministic reliable ready and fails regurarly
5315                 # to set up in time, so use UNIX socket forwards
5316                 my $socket_addr = "/run/qemu-server/$vmid.migrate";
5317                 unlink $socket_addr;
5318
5319                 $migrate_uri = "unix:$socket_addr";
5320
5321                 push @$cmd, '-incoming', $migrate_uri;
5322                 push @$cmd, '-S';
5323
5324             } elsif (-e $statefile) {
5325                 push @$cmd, '-loadstate', $statefile;
5326             } else {
5327                 my $statepath = PVE::Storage::path($storecfg, $statefile);
5328                 push @$vollist, $statefile;
5329                 push @$cmd, '-loadstate', $statepath;
5330             }
5331         } elsif ($paused) {
5332             push @$cmd, '-S';
5333         }
5334
5335         # host pci devices
5336         for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
5337           my $d = parse_hostpci($conf->{"hostpci$i"});
5338           next if !$d;
5339           my $pcidevices = $d->{pciid};
5340           foreach my $pcidevice (@$pcidevices) {
5341                 my $pciid = $pcidevice->{id};
5342
5343                 my $info = PVE::SysFSTools::pci_device_info("0000:$pciid");
5344                 die "IOMMU not present\n" if !PVE::SysFSTools::check_iommu_support();
5345                 die "no pci device info for device '$pciid'\n" if !$info;
5346
5347                 if ($d->{mdev}) {
5348                     my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $i);
5349                     PVE::SysFSTools::pci_create_mdev_device($pciid, $uuid, $d->{mdev});
5350                 } else {
5351                     die "can't unbind/bind pci group to vfio '$pciid'\n"
5352                         if !PVE::SysFSTools::pci_dev_group_bind_to_vfio($pciid);
5353                     die "can't reset pci device '$pciid'\n"
5354                         if $info->{has_fl_reset} and !PVE::SysFSTools::pci_dev_reset($info);
5355                 }
5356           }
5357         }
5358
5359         PVE::Storage::activate_volumes($storecfg, $vollist);
5360
5361         eval {
5362             run_command(['/bin/systemctl', 'stop', "$vmid.scope"],
5363                 outfunc => sub {}, errfunc => sub {});
5364         };
5365         # Issues with the above 'stop' not being fully completed are extremely rare, a very low
5366         # timeout should be more than enough here...
5367         PVE::Systemd::wait_for_unit_removed("$vmid.scope", 5);
5368
5369         my $cpuunits = defined($conf->{cpuunits}) ? $conf->{cpuunits}
5370                                                   : $defaults->{cpuunits};
5371
5372         my $start_timeout = ($conf->{hugepages} || $is_suspended) ? 300 : 30;
5373         my %run_params = (timeout => $statefile ? undef : $start_timeout, umask => 0077);
5374
5375         my %properties = (
5376             Slice => 'qemu.slice',
5377             KillMode => 'none',
5378             CPUShares => $cpuunits
5379         );
5380
5381         if (my $cpulimit = $conf->{cpulimit}) {
5382             $properties{CPUQuota} = int($cpulimit * 100);
5383         }
5384         $properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
5385
5386         my $run_qemu = sub {
5387             PVE::Tools::run_fork sub {
5388                 PVE::Systemd::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
5389                 run_command($cmd, %run_params);
5390             };
5391         };
5392
5393         if ($conf->{hugepages}) {
5394
5395             my $code = sub {
5396                 my $hugepages_topology = PVE::QemuServer::Memory::hugepages_topology($conf);
5397                 my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
5398
5399                 PVE::QemuServer::Memory::hugepages_mount();
5400                 PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
5401
5402                 eval { $run_qemu->() };
5403                 if (my $err = $@) {
5404                     PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology);
5405                     die $err;
5406                 }
5407
5408                 PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology);
5409             };
5410             eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
5411
5412         } else {
5413             eval { $run_qemu->() };
5414         }
5415
5416         if (my $err = $@) {
5417             # deactivate volumes if start fails
5418             eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
5419             die "start failed: $err";
5420         }
5421
5422         print "migration listens on $migrate_uri\n" if $migrate_uri;
5423
5424         if ($statefile && $statefile ne 'tcp' && $statefile ne 'unix')  {
5425             eval { mon_cmd($vmid, "cont"); };
5426             warn $@ if $@;
5427         }
5428
5429         #start nbd server for storage migration
5430         if ($targetstorage) {
5431             my $nodename = PVE::INotify::nodename();
5432             my $localip = $get_migration_ip->($migration_network, $nodename);
5433             my $pfamily = PVE::Tools::get_host_address_family($nodename);
5434             my $storage_migrate_port = PVE::Tools::next_migrate_port($pfamily);
5435
5436             mon_cmd($vmid, "nbd-server-start", addr => { type => 'inet', data => { host => "${localip}", port => "${storage_migrate_port}" } } );
5437
5438             $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5439
5440             foreach my $opt (sort keys %$local_volumes) {
5441                 my $volid = $local_volumes->{$opt};
5442                 mon_cmd($vmid, "nbd-server-add", device => "drive-$opt", writable => JSON::true );
5443                 my $migrate_storage_uri = "nbd:${localip}:${storage_migrate_port}:exportname=drive-$opt";
5444                 print "storage migration listens on $migrate_storage_uri volume:$volid\n";
5445             }
5446         }
5447
5448         if ($migratedfrom) {
5449             eval {
5450                 set_migration_caps($vmid);
5451             };
5452             warn $@ if $@;
5453
5454             if ($spice_port) {
5455                 print "spice listens on port $spice_port\n";
5456                 if ($spice_ticket) {
5457                     mon_cmd($vmid, "set_password", protocol => 'spice', password => $spice_ticket);
5458                     mon_cmd($vmid, "expire_password", protocol => 'spice', time => "+30");
5459                 }
5460             }
5461
5462         } else {
5463             mon_cmd($vmid, "balloon", value => $conf->{balloon}*1024*1024)
5464                 if !$statefile && $conf->{balloon};
5465
5466             foreach my $opt (keys %$conf) {
5467                 next if $opt !~  m/^net\d+$/;
5468                 my $nicconf = parse_net($conf->{$opt});
5469                 qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
5470             }
5471         }
5472
5473         mon_cmd($vmid, 'qom-set',
5474                     path => "machine/peripheral/balloon0",
5475                     property => "guest-stats-polling-interval",
5476                     value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
5477
5478         if ($is_suspended && (my $vmstate = $conf->{vmstate})) {
5479             print "Resumed VM, removing state\n";
5480             delete $conf->@{qw(lock vmstate runningmachine)};
5481             PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
5482             PVE::Storage::vdisk_free($storecfg, $vmstate);
5483             PVE::QemuConfig->write_config($vmid, $conf);
5484         }
5485
5486         PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'post-start');
5487     });
5488 }
5489
5490 sub vm_commandline {
5491     my ($storecfg, $vmid, $snapname) = @_;
5492
5493     my $conf = PVE::QemuConfig->load_config($vmid);
5494     my $forcemachine;
5495
5496     if ($snapname) {
5497         my $snapshot = $conf->{snapshots}->{$snapname};
5498         die "snapshot '$snapname' does not exist\n" if !defined($snapshot);
5499
5500         # check for a 'runningmachine' in snapshot
5501         $forcemachine = $snapshot->{runningmachine} if $snapshot->{runningmachine};
5502
5503         $snapshot->{digest} = $conf->{digest}; # keep file digest for API
5504
5505         $conf = $snapshot;
5506     }
5507
5508     my $defaults = load_defaults();
5509
5510     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine);
5511
5512     return PVE::Tools::cmd2string($cmd);
5513 }
5514
5515 sub vm_reset {
5516     my ($vmid, $skiplock) = @_;
5517
5518     PVE::QemuConfig->lock_config($vmid, sub {
5519
5520         my $conf = PVE::QemuConfig->load_config($vmid);
5521
5522         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5523
5524         mon_cmd($vmid, "system_reset");
5525     });
5526 }
5527
5528 sub get_vm_volumes {
5529     my ($conf) = @_;
5530
5531     my $vollist = [];
5532     foreach_volid($conf, sub {
5533         my ($volid, $attr) = @_;
5534
5535         return if $volid =~ m|^/|;
5536
5537         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5538         return if !$sid;
5539
5540         push @$vollist, $volid;
5541     });
5542
5543     return $vollist;
5544 }
5545
5546 sub vm_stop_cleanup {
5547     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
5548
5549     eval {
5550
5551         if (!$keepActive) {
5552             my $vollist = get_vm_volumes($conf);
5553             PVE::Storage::deactivate_volumes($storecfg, $vollist);
5554         }
5555
5556         foreach my $ext (qw(mon qmp pid vnc qga)) {
5557             unlink "/var/run/qemu-server/${vmid}.$ext";
5558         }
5559
5560         if ($conf->{ivshmem}) {
5561             my $ivshmem = PVE::JSONSchema::parse_property_string($ivshmem_fmt, $conf->{ivshmem});
5562             # just delete it for now, VMs which have this already open do not
5563             # are affected, but new VMs will get a separated one. If this
5564             # becomes an issue we either add some sort of ref-counting or just
5565             # add a "don't delete on stop" flag to the ivshmem format.
5566             unlink '/dev/shm/pve-shm-' . ($ivshmem->{name} // $vmid);
5567         }
5568
5569         foreach my $key (keys %$conf) {
5570             next if $key !~ m/^hostpci(\d+)$/;
5571             my $hostpciindex = $1;
5572             my $d = parse_hostpci($conf->{$key});
5573             my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $hostpciindex);
5574
5575             foreach my $pci (@{$d->{pciid}}) {
5576                 my $pciid = $pci->{id};
5577                 PVE::SysFSTools::pci_cleanup_mdev_device($pciid, $uuid);
5578             }
5579         }
5580
5581         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
5582     };
5583     warn $@ if $@; # avoid errors - just warn
5584 }
5585
5586 # call only in locked context
5587 sub _do_vm_stop {
5588     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive) = @_;
5589
5590     my $pid = check_running($vmid, $nocheck);
5591     return if !$pid;
5592
5593     my $conf;
5594     if (!$nocheck) {
5595         $conf = PVE::QemuConfig->load_config($vmid);
5596         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5597         if (!defined($timeout) && $shutdown && $conf->{startup}) {
5598             my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
5599             $timeout = $opts->{down} if $opts->{down};
5600         }
5601         PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-stop');
5602     }
5603
5604     eval {
5605         if ($shutdown) {
5606             if (defined($conf) && parse_guest_agent($conf)->{enabled}) {
5607                 mon_cmd($vmid, "guest-shutdown", timeout => $timeout);
5608             } else {
5609                 mon_cmd($vmid, "system_powerdown");
5610             }
5611         } else {
5612             mon_cmd($vmid, "quit");
5613         }
5614     };
5615     my $err = $@;
5616
5617     if (!$err) {
5618         $timeout = 60 if !defined($timeout);
5619
5620         my $count = 0;
5621         while (($count < $timeout) && check_running($vmid, $nocheck)) {
5622             $count++;
5623             sleep 1;
5624         }
5625
5626         if ($count >= $timeout) {
5627             if ($force) {
5628                 warn "VM still running - terminating now with SIGTERM\n";
5629                 kill 15, $pid;
5630             } else {
5631                 die "VM quit/powerdown failed - got timeout\n";
5632             }
5633         } else {
5634             vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5635             return;
5636         }
5637     } else {
5638         if ($force) {
5639             warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
5640             kill 15, $pid;
5641         } else {
5642             die "VM quit/powerdown failed\n";
5643         }
5644     }
5645
5646     # wait again
5647     $timeout = 10;
5648
5649     my $count = 0;
5650     while (($count < $timeout) && check_running($vmid, $nocheck)) {
5651         $count++;
5652         sleep 1;
5653     }
5654
5655     if ($count >= $timeout) {
5656         warn "VM still running - terminating now with SIGKILL\n";
5657         kill 9, $pid;
5658         sleep 1;
5659     }
5660
5661     vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5662 }
5663
5664 # Note: use $nocheck to skip tests if VM configuration file exists.
5665 # We need that when migration VMs to other nodes (files already moved)
5666 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
5667 sub vm_stop {
5668     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
5669
5670     $force = 1 if !defined($force) && !$shutdown;
5671
5672     if ($migratedfrom){
5673         my $pid = check_running($vmid, $nocheck, $migratedfrom);
5674         kill 15, $pid if $pid;
5675         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
5676         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
5677         return;
5678     }
5679
5680     PVE::QemuConfig->lock_config($vmid, sub {
5681         _do_vm_stop($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive);
5682    });
5683 }
5684
5685 sub vm_reboot {
5686     my ($vmid, $timeout) = @_;
5687
5688     PVE::QemuConfig->lock_config($vmid, sub {
5689         eval {
5690
5691             # only reboot if running, as qmeventd starts it again on a stop event
5692             return if !check_running($vmid);
5693
5694             create_reboot_request($vmid);
5695
5696             my $storecfg = PVE::Storage::config();
5697             _do_vm_stop($storecfg, $vmid, undef, undef, $timeout, 1);
5698
5699         };
5700         if (my $err = $@) {
5701             # avoid that the next normal shutdown will be confused for a reboot
5702             clear_reboot_request($vmid);
5703             die $err;
5704         }
5705    });
5706 }
5707
5708 sub vm_suspend {
5709     my ($vmid, $skiplock, $includestate, $statestorage) = @_;
5710
5711     my $conf;
5712     my $path;
5713     my $storecfg;
5714     my $vmstate;
5715
5716     PVE::QemuConfig->lock_config($vmid, sub {
5717
5718         $conf = PVE::QemuConfig->load_config($vmid);
5719
5720         my $is_backing_up = PVE::QemuConfig->has_lock($conf, 'backup');
5721         PVE::QemuConfig->check_lock($conf)
5722             if !($skiplock || $is_backing_up);
5723
5724         die "cannot suspend to disk during backup\n"
5725             if $is_backing_up && $includestate;
5726
5727         if ($includestate) {
5728             $conf->{lock} = 'suspending';
5729             my $date = strftime("%Y-%m-%d", localtime(time()));
5730             $storecfg = PVE::Storage::config();
5731             $vmstate = PVE::QemuConfig->__snapshot_save_vmstate($vmid, $conf, "suspend-$date", $storecfg, $statestorage, 1);
5732             $path = PVE::Storage::path($storecfg, $vmstate);
5733             PVE::QemuConfig->write_config($vmid, $conf);
5734         } else {
5735             mon_cmd($vmid, "stop");
5736         }
5737     });
5738
5739     if ($includestate) {
5740         # save vm state
5741         PVE::Storage::activate_volumes($storecfg, [$vmstate]);
5742
5743         eval {
5744             mon_cmd($vmid, "savevm-start", statefile => $path);
5745             for(;;) {
5746                 my $state = mon_cmd($vmid, "query-savevm");
5747                 if (!$state->{status}) {
5748                     die "savevm not active\n";
5749                 } elsif ($state->{status} eq 'active') {
5750                     sleep(1);
5751                     next;
5752                 } elsif ($state->{status} eq 'completed') {
5753                     print "State saved, quitting\n";
5754                     last;
5755                 } elsif ($state->{status} eq 'failed' && $state->{error}) {
5756                     die "query-savevm failed with error '$state->{error}'\n"
5757                 } else {
5758                     die "query-savevm returned status '$state->{status}'\n";
5759                 }
5760             }
5761         };
5762         my $err = $@;
5763
5764         PVE::QemuConfig->lock_config($vmid, sub {
5765             $conf = PVE::QemuConfig->load_config($vmid);
5766             if ($err) {
5767                 # cleanup, but leave suspending lock, to indicate something went wrong
5768                 eval {
5769                     mon_cmd($vmid, "savevm-end");
5770                     PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
5771                     PVE::Storage::vdisk_free($storecfg, $vmstate);
5772                     delete $conf->@{qw(vmstate runningmachine)};
5773                     PVE::QemuConfig->write_config($vmid, $conf);
5774                 };
5775                 warn $@ if $@;
5776                 die $err;
5777             }
5778
5779             die "lock changed unexpectedly\n"
5780                 if !PVE::QemuConfig->has_lock($conf, 'suspending');
5781
5782             mon_cmd($vmid, "quit");
5783             $conf->{lock} = 'suspended';
5784             PVE::QemuConfig->write_config($vmid, $conf);
5785         });
5786     }
5787 }
5788
5789 sub vm_resume {
5790     my ($vmid, $skiplock, $nocheck) = @_;
5791
5792     PVE::QemuConfig->lock_config($vmid, sub {
5793         my $res = mon_cmd($vmid, 'query-status');
5794         my $resume_cmd = 'cont';
5795
5796         if ($res->{status} && $res->{status} eq 'suspended') {
5797             $resume_cmd = 'system_wakeup';
5798         }
5799
5800         if (!$nocheck) {
5801
5802             my $conf = PVE::QemuConfig->load_config($vmid);
5803
5804             PVE::QemuConfig->check_lock($conf)
5805                 if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
5806         }
5807
5808         mon_cmd($vmid, $resume_cmd);
5809     });
5810 }
5811
5812 sub vm_sendkey {
5813     my ($vmid, $skiplock, $key) = @_;
5814
5815     PVE::QemuConfig->lock_config($vmid, sub {
5816
5817         my $conf = PVE::QemuConfig->load_config($vmid);
5818
5819         # there is no qmp command, so we use the human monitor command
5820         my $res = PVE::QemuServer::Monitor::hmp_cmd($vmid, "sendkey $key");
5821         die $res if $res ne '';
5822     });
5823 }
5824
5825 # vzdump restore implementaion
5826
5827 sub tar_archive_read_firstfile {
5828     my $archive = shift;
5829
5830     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
5831
5832     # try to detect archive type first
5833     my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
5834         die "unable to open file '$archive'\n";
5835     my $firstfile = <$fh>;
5836     kill 15, $pid;
5837     close $fh;
5838
5839     die "ERROR: archive contaions no data\n" if !$firstfile;
5840     chomp $firstfile;
5841
5842     return $firstfile;
5843 }
5844
5845 sub tar_restore_cleanup {
5846     my ($storecfg, $statfile) = @_;
5847
5848     print STDERR "starting cleanup\n";
5849
5850     if (my $fd = IO::File->new($statfile, "r")) {
5851         while (defined(my $line = <$fd>)) {
5852             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5853                 my $volid = $2;
5854                 eval {
5855                     if ($volid =~ m|^/|) {
5856                         unlink $volid || die 'unlink failed\n';
5857                     } else {
5858                         PVE::Storage::vdisk_free($storecfg, $volid);
5859                     }
5860                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
5861                 };
5862                 print STDERR "unable to cleanup '$volid' - $@" if $@;
5863             } else {
5864                 print STDERR "unable to parse line in statfile - $line";
5865             }
5866         }
5867         $fd->close();
5868     }
5869 }
5870
5871 sub restore_archive {
5872     my ($archive, $vmid, $user, $opts) = @_;
5873
5874     my $format = $opts->{format};
5875     my $comp;
5876
5877     if ($archive =~ m/\.tgz$/ || $archive =~ m/\.tar\.gz$/) {
5878         $format = 'tar' if !$format;
5879         $comp = 'gzip';
5880     } elsif ($archive =~ m/\.tar$/) {
5881         $format = 'tar' if !$format;
5882     } elsif ($archive =~ m/.tar.lzo$/) {
5883         $format = 'tar' if !$format;
5884         $comp = 'lzop';
5885     } elsif ($archive =~ m/\.vma$/) {
5886         $format = 'vma' if !$format;
5887     } elsif ($archive =~ m/\.vma\.gz$/) {
5888         $format = 'vma' if !$format;
5889         $comp = 'gzip';
5890     } elsif ($archive =~ m/\.vma\.lzo$/) {
5891         $format = 'vma' if !$format;
5892         $comp = 'lzop';
5893     } else {
5894         $format = 'vma' if !$format; # default
5895     }
5896
5897     # try to detect archive format
5898     if ($format eq 'tar') {
5899         return restore_tar_archive($archive, $vmid, $user, $opts);
5900     } else {
5901         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
5902     }
5903 }
5904
5905 sub restore_update_config_line {
5906     my ($outfd, $cookie, $vmid, $map, $line, $unique) = @_;
5907
5908     return if $line =~ m/^\#qmdump\#/;
5909     return if $line =~ m/^\#vzdump\#/;
5910     return if $line =~ m/^lock:/;
5911     return if $line =~ m/^unused\d+:/;
5912     return if $line =~ m/^parent:/;
5913
5914     my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
5915     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
5916         # try to convert old 1.X settings
5917         my ($id, $ind, $ethcfg) = ($1, $2, $3);
5918         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
5919             my ($model, $macaddr) = split(/\=/, $devconfig);
5920             $macaddr = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if !$macaddr || $unique;
5921             my $net = {
5922                 model => $model,
5923                 bridge => "vmbr$ind",
5924                 macaddr => $macaddr,
5925             };
5926             my $netstr = print_net($net);
5927
5928             print $outfd "net$cookie->{netcount}: $netstr\n";
5929             $cookie->{netcount}++;
5930         }
5931     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
5932         my ($id, $netstr) = ($1, $2);
5933         my $net = parse_net($netstr);
5934         $net->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if $net->{macaddr};
5935         $netstr = print_net($net);
5936         print $outfd "$id: $netstr\n";
5937     } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk)\d+):\s*(\S+)\s*$/) {
5938         my $virtdev = $1;
5939         my $value = $3;
5940         my $di = parse_drive($virtdev, $value);
5941         if (defined($di->{backup}) && !$di->{backup}) {
5942             print $outfd "#$line";
5943         } elsif ($map->{$virtdev}) {
5944             delete $di->{format}; # format can change on restore
5945             $di->{file} = $map->{$virtdev};
5946             $value = print_drive($vmid, $di);
5947             print $outfd "$virtdev: $value\n";
5948         } else {
5949             print $outfd $line;
5950         }
5951     } elsif (($line =~ m/^vmgenid: (.*)/)) {
5952         my $vmgenid = $1;
5953         if ($vmgenid ne '0') {
5954             # always generate a new vmgenid if there was a valid one setup
5955             $vmgenid = generate_uuid();
5956         }
5957         print $outfd "vmgenid: $vmgenid\n";
5958     } elsif (($line =~ m/^(smbios1: )(.*)/) && $unique) {
5959         my ($uuid, $uuid_str);
5960         UUID::generate($uuid);
5961         UUID::unparse($uuid, $uuid_str);
5962         my $smbios1 = parse_smbios1($2);
5963         $smbios1->{uuid} = $uuid_str;
5964         print $outfd $1.print_smbios1($smbios1)."\n";
5965     } else {
5966         print $outfd $line;
5967     }
5968 }
5969
5970 sub scan_volids {
5971     my ($cfg, $vmid) = @_;
5972
5973     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid);
5974
5975     my $volid_hash = {};
5976     foreach my $storeid (keys %$info) {
5977         foreach my $item (@{$info->{$storeid}}) {
5978             next if !($item->{volid} && $item->{size});
5979             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
5980             $volid_hash->{$item->{volid}} = $item;
5981         }
5982     }
5983
5984     return $volid_hash;
5985 }
5986
5987 sub is_volume_in_use {
5988     my ($storecfg, $conf, $skip_drive, $volid) = @_;
5989
5990     my $path = PVE::Storage::path($storecfg, $volid);
5991
5992     my $scan_config = sub {
5993         my ($cref, $snapname) = @_;
5994
5995         foreach my $key (keys %$cref) {
5996             my $value = $cref->{$key};
5997             if (is_valid_drivename($key)) {
5998                 next if $skip_drive && $key eq $skip_drive;
5999                 my $drive = parse_drive($key, $value);
6000                 next if !$drive || !$drive->{file} || drive_is_cdrom($drive);
6001                 return 1 if $volid eq $drive->{file};
6002                 if ($drive->{file} =~ m!^/!) {
6003                     return 1 if $drive->{file} eq $path;
6004                 } else {
6005                     my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file}, 1);
6006                     next if !$storeid;
6007                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid, 1);
6008                     next if !$scfg;
6009                     return 1 if $path eq PVE::Storage::path($storecfg, $drive->{file}, $snapname);
6010                 }
6011             }
6012         }
6013
6014         return 0;
6015     };
6016
6017     return 1 if &$scan_config($conf);
6018
6019     undef $skip_drive;
6020
6021     foreach my $snapname (keys %{$conf->{snapshots}}) {
6022         return 1 if &$scan_config($conf->{snapshots}->{$snapname}, $snapname);
6023     }
6024
6025     return 0;
6026 }
6027
6028 sub update_disksize {
6029     my ($vmid, $conf, $volid_hash) = @_;
6030
6031     my $changes;
6032     my $prefix = "VM $vmid:";
6033
6034     # used and unused disks
6035     my $referenced = {};
6036
6037     # Note: it is allowed to define multiple storages with same path (alias), so
6038     # we need to check both 'volid' and real 'path' (two different volid can point
6039     # to the same path).
6040
6041     my $referencedpath = {};
6042
6043     # update size info
6044     foreach my $opt (keys %$conf) {
6045         if (is_valid_drivename($opt)) {
6046             my $drive = parse_drive($opt, $conf->{$opt});
6047             my $volid = $drive->{file};
6048             next if !$volid;
6049
6050             $referenced->{$volid} = 1;
6051             if ($volid_hash->{$volid} &&
6052                 (my $path = $volid_hash->{$volid}->{path})) {
6053                 $referencedpath->{$path} = 1;
6054             }
6055
6056             next if drive_is_cdrom($drive);
6057             next if !$volid_hash->{$volid};
6058
6059             $drive->{size} = $volid_hash->{$volid}->{size};
6060             my $new = print_drive($vmid, $drive);
6061             if ($new ne $conf->{$opt}) {
6062                 $changes = 1;
6063                 $conf->{$opt} = $new;
6064                 print "$prefix update disk '$opt' information.\n";
6065             }
6066         }
6067     }
6068
6069     # remove 'unusedX' entry if volume is used
6070     foreach my $opt (keys %$conf) {
6071         next if $opt !~ m/^unused\d+$/;
6072         my $volid = $conf->{$opt};
6073         my $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
6074         if ($referenced->{$volid} || ($path && $referencedpath->{$path})) {
6075             print "$prefix remove entry '$opt', its volume '$volid' is in use.\n";
6076             $changes = 1;
6077             delete $conf->{$opt};
6078         }
6079
6080         $referenced->{$volid} = 1;
6081         $referencedpath->{$path} = 1 if $path;
6082     }
6083
6084     foreach my $volid (sort keys %$volid_hash) {
6085         next if $volid =~ m/vm-$vmid-state-/;
6086         next if $referenced->{$volid};
6087         my $path = $volid_hash->{$volid}->{path};
6088         next if !$path; # just to be sure
6089         next if $referencedpath->{$path};
6090         $changes = 1;
6091         my $key = PVE::QemuConfig->add_unused_volume($conf, $volid);
6092         print "$prefix add unreferenced volume '$volid' as '$key' to config.\n";
6093         $referencedpath->{$path} = 1; # avoid to add more than once (aliases)
6094     }
6095
6096     return $changes;
6097 }
6098
6099 sub rescan {
6100     my ($vmid, $nolock, $dryrun) = @_;
6101
6102     my $cfg = PVE::Storage::config();
6103
6104     # FIXME: Remove once our RBD plugin can handle CT and VM on a single storage
6105     # see: https://pve.proxmox.com/pipermail/pve-devel/2018-July/032900.html
6106     foreach my $stor (keys %{$cfg->{ids}}) {
6107         delete($cfg->{ids}->{$stor}) if ! $cfg->{ids}->{$stor}->{content}->{images};
6108     }
6109
6110     print "rescan volumes...\n";
6111     my $volid_hash = scan_volids($cfg, $vmid);
6112
6113     my $updatefn =  sub {
6114         my ($vmid) = @_;
6115
6116         my $conf = PVE::QemuConfig->load_config($vmid);
6117
6118         PVE::QemuConfig->check_lock($conf);
6119
6120         my $vm_volids = {};
6121         foreach my $volid (keys %$volid_hash) {
6122             my $info = $volid_hash->{$volid};
6123             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
6124         }
6125
6126         my $changes = update_disksize($vmid, $conf, $vm_volids);
6127
6128         PVE::QemuConfig->write_config($vmid, $conf) if $changes && !$dryrun;
6129     };
6130
6131     if (defined($vmid)) {
6132         if ($nolock) {
6133             &$updatefn($vmid);
6134         } else {
6135             PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6136         }
6137     } else {
6138         my $vmlist = config_list();
6139         foreach my $vmid (keys %$vmlist) {
6140             if ($nolock) {
6141                 &$updatefn($vmid);
6142             } else {
6143                 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6144             }
6145         }
6146     }
6147 }
6148
6149 sub restore_vma_archive {
6150     my ($archive, $vmid, $user, $opts, $comp) = @_;
6151
6152     my $readfrom = $archive;
6153
6154     my $cfg = PVE::Storage::config();
6155     my $commands = [];
6156     my $bwlimit = $opts->{bwlimit};
6157
6158     my $dbg_cmdstring = '';
6159     my $add_pipe = sub {
6160         my ($cmd) = @_;
6161         push @$commands, $cmd;
6162         $dbg_cmdstring .= ' | ' if length($dbg_cmdstring);
6163         $dbg_cmdstring .= PVE::Tools::cmd2string($cmd);
6164         $readfrom = '-';
6165     };
6166
6167     my $input = undef;
6168     if ($archive eq '-') {
6169         $input = '<&STDIN';
6170     } else {
6171         # If we use a backup from a PVE defined storage we also consider that
6172         # storage's rate limit:
6173         my (undef, $volid) = PVE::Storage::path_to_volume_id($cfg, $archive);
6174         if (defined($volid)) {
6175             my ($sid, undef) = PVE::Storage::parse_volume_id($volid);
6176             my $readlimit = PVE::Storage::get_bandwidth_limit('restore', [$sid], $bwlimit);
6177             if ($readlimit) {
6178                 print STDERR "applying read rate limit: $readlimit\n";
6179                 my $cstream = ['cstream', '-t', $readlimit*1024, '--', $readfrom];
6180                 $add_pipe->($cstream);
6181             }
6182         }
6183     }
6184
6185     if ($comp) {
6186         my $cmd;
6187         if ($comp eq 'gzip') {
6188             $cmd = ['zcat', $readfrom];
6189         } elsif ($comp eq 'lzop') {
6190             $cmd = ['lzop', '-d', '-c', $readfrom];
6191         } else {
6192             die "unknown compression method '$comp'\n";
6193         }
6194         $add_pipe->($cmd);
6195     }
6196
6197     my $tmpdir = "/var/tmp/vzdumptmp$$";
6198     rmtree $tmpdir;
6199
6200     # disable interrupts (always do cleanups)
6201     local $SIG{INT} =
6202         local $SIG{TERM} =
6203         local $SIG{QUIT} =
6204         local $SIG{HUP} = sub { warn "got interrupt - ignored\n"; };
6205
6206     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
6207     POSIX::mkfifo($mapfifo, 0600);
6208     my $fifofh;
6209
6210     my $openfifo = sub {
6211         open($fifofh, '>', $mapfifo) || die $!;
6212     };
6213
6214     $add_pipe->(['vma', 'extract', '-v', '-r', $mapfifo, $readfrom, $tmpdir]);
6215
6216     my $oldtimeout;
6217     my $timeout = 5;
6218
6219     my $devinfo = {};
6220
6221     my $rpcenv = PVE::RPCEnvironment::get();
6222
6223     my $conffile = PVE::QemuConfig->config_file($vmid);
6224     my $tmpfn = "$conffile.$$.tmp";
6225
6226     # Note: $oldconf is undef if VM does not exists
6227     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
6228     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
6229
6230     my %storage_limits;
6231
6232     my $print_devmap = sub {
6233         my $virtdev_hash = {};
6234
6235         my $cfgfn = "$tmpdir/qemu-server.conf";
6236
6237         # we can read the config - that is already extracted
6238         my $fh = IO::File->new($cfgfn, "r") ||
6239             "unable to read qemu-server.conf - $!\n";
6240
6241         my $fwcfgfn = "$tmpdir/qemu-server.fw";
6242         if (-f $fwcfgfn) {
6243             my $pve_firewall_dir = '/etc/pve/firewall';
6244             mkdir $pve_firewall_dir; # make sure the dir exists
6245             PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
6246         }
6247
6248         while (defined(my $line = <$fh>)) {
6249             if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
6250                 my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
6251                 die "archive does not contain data for drive '$virtdev'\n"
6252                     if !$devinfo->{$devname};
6253                 if (defined($opts->{storage})) {
6254                     $storeid = $opts->{storage} || 'local';
6255                 } elsif (!$storeid) {
6256                     $storeid = 'local';
6257                 }
6258                 $format = 'raw' if !$format;
6259                 $devinfo->{$devname}->{devname} = $devname;
6260                 $devinfo->{$devname}->{virtdev} = $virtdev;
6261                 $devinfo->{$devname}->{format} = $format;
6262                 $devinfo->{$devname}->{storeid} = $storeid;
6263
6264                 # check permission on storage
6265                 my $pool = $opts->{pool}; # todo: do we need that?
6266                 if ($user ne 'root@pam') {
6267                     $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
6268                 }
6269
6270                 $storage_limits{$storeid} = $bwlimit;
6271
6272                 $virtdev_hash->{$virtdev} = $devinfo->{$devname};
6273             } elsif ($line =~ m/^((?:ide|sata|scsi)\d+):\s*(.*)\s*$/) {
6274                 my $virtdev = $1;
6275                 my $drive = parse_drive($virtdev, $2);
6276                 if (drive_is_cloudinit($drive)) {
6277                     my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6278                     my $scfg = PVE::Storage::storage_config($cfg, $storeid);
6279                     my $format = qemu_img_format($scfg, $volname); # has 'raw' fallback
6280
6281                     my $d = {
6282                         format => $format,
6283                         storeid => $opts->{storage} // $storeid,
6284                         size => PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE,
6285                         file => $drive->{file}, # to make drive_is_cloudinit check possible
6286                         name => "vm-$vmid-cloudinit",
6287                         is_cloudinit => 1,
6288                     };
6289                     $virtdev_hash->{$virtdev} = $d;
6290                 }
6291             }
6292         }
6293
6294         foreach my $key (keys %storage_limits) {
6295             my $limit = PVE::Storage::get_bandwidth_limit('restore', [$key], $bwlimit);
6296             next if !$limit;
6297             print STDERR "rate limit for storage $key: $limit KiB/s\n";
6298             $storage_limits{$key} = $limit * 1024;
6299         }
6300
6301         foreach my $devname (keys %$devinfo) {
6302             die "found no device mapping information for device '$devname'\n"
6303                 if !$devinfo->{$devname}->{virtdev};
6304         }
6305
6306         # create empty/temp config
6307         if ($oldconf) {
6308             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
6309             foreach_drive($oldconf, sub {
6310                 my ($ds, $drive) = @_;
6311
6312                 return if drive_is_cdrom($drive, 1);
6313
6314                 my $volid = $drive->{file};
6315                 return if !$volid || $volid =~ m|^/|;
6316
6317                 my ($path, $owner) = PVE::Storage::path($cfg, $volid);
6318                 return if !$path || !$owner || ($owner != $vmid);
6319
6320                 # Note: only delete disk we want to restore
6321                 # other volumes will become unused
6322                 if ($virtdev_hash->{$ds}) {
6323                     eval { PVE::Storage::vdisk_free($cfg, $volid); };
6324                     if (my $err = $@) {
6325                         warn $err;
6326                     }
6327                 }
6328             });
6329
6330             # delete vmstate files, after the restore we have no snapshots anymore
6331             foreach my $snapname (keys %{$oldconf->{snapshots}}) {
6332                 my $snap = $oldconf->{snapshots}->{$snapname};
6333                 if ($snap->{vmstate}) {
6334                     eval { PVE::Storage::vdisk_free($cfg, $snap->{vmstate}); };
6335                     if (my $err = $@) {
6336                         warn $err;
6337                     }
6338                 }
6339             }
6340         }
6341
6342         my $map = {};
6343         foreach my $virtdev (sort keys %$virtdev_hash) {
6344             my $d = $virtdev_hash->{$virtdev};
6345             my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
6346             my $storeid = $d->{storeid};
6347             my $scfg = PVE::Storage::storage_config($cfg, $storeid);
6348
6349             my $map_opts = '';
6350             if (my $limit = $storage_limits{$storeid}) {
6351                 $map_opts .= "throttling.bps=$limit:throttling.group=$storeid:";
6352             }
6353
6354             # test if requested format is supported
6355             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($cfg, $storeid);
6356             my $supported = grep { $_ eq $d->{format} } @$validFormats;
6357             $d->{format} = $defFormat if !$supported;
6358
6359             my $name;
6360             if ($d->{is_cloudinit}) {
6361                 $name = $d->{name};
6362                 $name .= ".$d->{format}" if $d->{format} ne 'raw';
6363             }
6364
6365             my $volid = PVE::Storage::vdisk_alloc($cfg, $storeid, $vmid, $d->{format}, $name, $alloc_size);
6366             print STDERR "new volume ID is '$volid'\n";
6367             $d->{volid} = $volid;
6368
6369             PVE::Storage::activate_volumes($cfg, [$volid]);
6370
6371             my $write_zeros = 1;
6372             if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
6373                 $write_zeros = 0;
6374             }
6375
6376             if (!$d->{is_cloudinit}) {
6377                 my $path = PVE::Storage::path($cfg, $volid);
6378
6379                 print $fifofh "${map_opts}format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
6380
6381                 print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
6382             }
6383             $map->{$virtdev} = $volid;
6384         }
6385
6386         $fh->seek(0, 0) || die "seek failed - $!\n";
6387
6388         my $outfd = new IO::File ($tmpfn, "w") ||
6389             die "unable to write config for VM $vmid\n";
6390
6391         my $cookie = { netcount => 0 };
6392         while (defined(my $line = <$fh>)) {
6393             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
6394         }
6395
6396         $fh->close();
6397         $outfd->close();
6398     };
6399
6400     eval {
6401         # enable interrupts
6402         local $SIG{INT} =
6403             local $SIG{TERM} =
6404             local $SIG{QUIT} =
6405             local $SIG{HUP} =
6406             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6407         local $SIG{ALRM} = sub { die "got timeout\n"; };
6408
6409         $oldtimeout = alarm($timeout);
6410
6411         my $parser = sub {
6412             my $line = shift;
6413
6414             print "$line\n";
6415
6416             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
6417                 my ($dev_id, $size, $devname) = ($1, $2, $3);
6418                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
6419             } elsif ($line =~ m/^CTIME: /) {
6420                 # we correctly received the vma config, so we can disable
6421                 # the timeout now for disk allocation (set to 10 minutes, so
6422                 # that we always timeout if something goes wrong)
6423                 alarm(600);
6424                 &$print_devmap();
6425                 print $fifofh "done\n";
6426                 my $tmp = $oldtimeout || 0;
6427                 $oldtimeout = undef;
6428                 alarm($tmp);
6429                 close($fifofh);
6430             }
6431         };
6432
6433         print "restore vma archive: $dbg_cmdstring\n";
6434         run_command($commands, input => $input, outfunc => $parser, afterfork => $openfifo);
6435     };
6436     my $err = $@;
6437
6438     alarm($oldtimeout) if $oldtimeout;
6439
6440     my $vollist = [];
6441     foreach my $devname (keys %$devinfo) {
6442         my $volid = $devinfo->{$devname}->{volid};
6443         push @$vollist, $volid if $volid;
6444     }
6445
6446     PVE::Storage::deactivate_volumes($cfg, $vollist);
6447
6448     unlink $mapfifo;
6449
6450     if ($err) {
6451         rmtree $tmpdir;
6452         unlink $tmpfn;
6453
6454         foreach my $devname (keys %$devinfo) {
6455             my $volid = $devinfo->{$devname}->{volid};
6456             next if !$volid;
6457             eval {
6458                 if ($volid =~ m|^/|) {
6459                     unlink $volid || die 'unlink failed\n';
6460                 } else {
6461                     PVE::Storage::vdisk_free($cfg, $volid);
6462                 }
6463                 print STDERR "temporary volume '$volid' sucessfuly removed\n";
6464             };
6465             print STDERR "unable to cleanup '$volid' - $@" if $@;
6466         }
6467         die $err;
6468     }
6469
6470     rmtree $tmpdir;
6471
6472     rename($tmpfn, $conffile) ||
6473         die "unable to commit configuration file '$conffile'\n";
6474
6475     PVE::Cluster::cfs_update(); # make sure we read new file
6476
6477     eval { rescan($vmid, 1); };
6478     warn $@ if $@;
6479 }
6480
6481 sub restore_tar_archive {
6482     my ($archive, $vmid, $user, $opts) = @_;
6483
6484     if ($archive ne '-') {
6485         my $firstfile = tar_archive_read_firstfile($archive);
6486         die "ERROR: file '$archive' dos not lock like a QemuServer vzdump backup\n"
6487             if $firstfile ne 'qemu-server.conf';
6488     }
6489
6490     my $storecfg = PVE::Storage::config();
6491
6492     # avoid zombie disks when restoring over an existing VM -> cleanup first
6493     # pass keep_empty_config=1 to keep the config (thus VMID) reserved for us
6494     # skiplock=1 because qmrestore has set the 'create' lock itself already
6495     my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
6496     destroy_vm($storecfg, $vmid, 1, { lock => 'restore' }) if -f $vmcfgfn;
6497
6498     my $tocmd = "/usr/lib/qemu-server/qmextract";
6499
6500     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
6501     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
6502     $tocmd .= ' --prealloc' if $opts->{prealloc};
6503     $tocmd .= ' --info' if $opts->{info};
6504
6505     # tar option "xf" does not autodetect compression when read from STDIN,
6506     # so we pipe to zcat
6507     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
6508         PVE::Tools::shellquote("--to-command=$tocmd");
6509
6510     my $tmpdir = "/var/tmp/vzdumptmp$$";
6511     mkpath $tmpdir;
6512
6513     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
6514     local $ENV{VZDUMP_VMID} = $vmid;
6515     local $ENV{VZDUMP_USER} = $user;
6516
6517     my $conffile = PVE::QemuConfig->config_file($vmid);
6518     my $tmpfn = "$conffile.$$.tmp";
6519
6520     # disable interrupts (always do cleanups)
6521     local $SIG{INT} =
6522         local $SIG{TERM} =
6523         local $SIG{QUIT} =
6524         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
6525
6526     eval {
6527         # enable interrupts
6528         local $SIG{INT} =
6529             local $SIG{TERM} =
6530             local $SIG{QUIT} =
6531             local $SIG{HUP} =
6532             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6533
6534         if ($archive eq '-') {
6535             print "extracting archive from STDIN\n";
6536             run_command($cmd, input => "<&STDIN");
6537         } else {
6538             print "extracting archive '$archive'\n";
6539             run_command($cmd);
6540         }
6541
6542         return if $opts->{info};
6543
6544         # read new mapping
6545         my $map = {};
6546         my $statfile = "$tmpdir/qmrestore.stat";
6547         if (my $fd = IO::File->new($statfile, "r")) {
6548             while (defined (my $line = <$fd>)) {
6549                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
6550                     $map->{$1} = $2 if $1;
6551                 } else {
6552                     print STDERR "unable to parse line in statfile - $line\n";
6553                 }
6554             }
6555             $fd->close();
6556         }
6557
6558         my $confsrc = "$tmpdir/qemu-server.conf";
6559
6560         my $srcfd = new IO::File($confsrc, "r") ||
6561             die "unable to open file '$confsrc'\n";
6562
6563         my $outfd = new IO::File ($tmpfn, "w") ||
6564             die "unable to write config for VM $vmid\n";
6565
6566         my $cookie = { netcount => 0 };
6567         while (defined (my $line = <$srcfd>)) {
6568             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
6569         }
6570
6571         $srcfd->close();
6572         $outfd->close();
6573     };
6574     if (my $err = $@) {
6575         unlink $tmpfn;
6576         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
6577         die $err;
6578     }
6579
6580     rmtree $tmpdir;
6581
6582     rename $tmpfn, $conffile ||
6583         die "unable to commit configuration file '$conffile'\n";
6584
6585     PVE::Cluster::cfs_update(); # make sure we read new file
6586
6587     eval { rescan($vmid, 1); };
6588     warn $@ if $@;
6589 };
6590
6591 sub foreach_storage_used_by_vm {
6592     my ($conf, $func) = @_;
6593
6594     my $sidhash = {};
6595
6596     foreach_drive($conf, sub {
6597         my ($ds, $drive) = @_;
6598         return if drive_is_cdrom($drive);
6599
6600         my $volid = $drive->{file};
6601
6602         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
6603         $sidhash->{$sid} = $sid if $sid;
6604     });
6605
6606     foreach my $sid (sort keys %$sidhash) {
6607         &$func($sid);
6608     }
6609 }
6610
6611 my $qemu_snap_storage = {
6612     rbd => 1,
6613 };
6614 sub do_snapshots_with_qemu {
6615     my ($storecfg, $volid) = @_;
6616
6617     my $storage_name = PVE::Storage::parse_volume_id($volid);
6618     my $scfg = $storecfg->{ids}->{$storage_name};
6619
6620     if ($qemu_snap_storage->{$scfg->{type}} && !$scfg->{krbd}){
6621         return 1;
6622     }
6623
6624     if ($volid =~ m/\.(qcow2|qed)$/){
6625         return 1;
6626     }
6627
6628     return undef;
6629 }
6630
6631 sub qga_check_running {
6632     my ($vmid, $nowarn) = @_;
6633
6634     eval { mon_cmd($vmid, "guest-ping", timeout => 3); };
6635     if ($@) {
6636         warn "Qemu Guest Agent is not running - $@" if !$nowarn;
6637         return 0;
6638     }
6639     return 1;
6640 }
6641
6642 sub template_create {
6643     my ($vmid, $conf, $disk) = @_;
6644
6645     my $storecfg = PVE::Storage::config();
6646
6647     foreach_drive($conf, sub {
6648         my ($ds, $drive) = @_;
6649
6650         return if drive_is_cdrom($drive);
6651         return if $disk && $ds ne $disk;
6652
6653         my $volid = $drive->{file};
6654         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
6655
6656         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
6657         $drive->{file} = $voliddst;
6658         $conf->{$ds} = print_drive($vmid, $drive);
6659         PVE::QemuConfig->write_config($vmid, $conf);
6660     });
6661 }
6662
6663 sub convert_iscsi_path {
6664     my ($path) = @_;
6665
6666     if ($path =~ m|^iscsi://([^/]+)/([^/]+)/(.+)$|) {
6667         my $portal = $1;
6668         my $target = $2;
6669         my $lun = $3;
6670
6671         my $initiator_name = get_initiator_name();
6672
6673         return "file.driver=iscsi,file.transport=tcp,file.initiator-name=$initiator_name,".
6674                "file.portal=$portal,file.target=$target,file.lun=$lun,driver=raw";
6675     }
6676
6677     die "cannot convert iscsi path '$path', unkown format\n";
6678 }
6679
6680 sub qemu_img_convert {
6681     my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized) = @_;
6682
6683     my $storecfg = PVE::Storage::config();
6684     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
6685     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
6686
6687     die "destination '$dst_volid' is not a valid volid form qemu-img convert\n" if !$dst_storeid;
6688
6689     my $cachemode;
6690     my $src_path;
6691     my $src_is_iscsi = 0;
6692     my $src_format = 'raw';
6693
6694     if ($src_storeid) {
6695         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
6696         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
6697         $src_format = qemu_img_format($src_scfg, $src_volname);
6698         $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
6699         $src_is_iscsi = ($src_path =~ m|^iscsi://|);
6700         $cachemode = 'none' if $src_scfg->{type} eq 'zfspool';
6701     } elsif (-f $src_volid) {
6702         $src_path = $src_volid;
6703         if ($src_path =~ m/\.($QEMU_FORMAT_RE)$/) {
6704             $src_format = $1;
6705         }
6706     }
6707
6708     die "source '$src_volid' is not a valid volid nor path for qemu-img convert\n" if !$src_path;
6709
6710     my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6711     my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
6712     my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6713     my $dst_is_iscsi = ($dst_path =~ m|^iscsi://|);
6714
6715     my $cmd = [];
6716     push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
6717     push @$cmd, '-l', "snapshot.name=$snapname" if($snapname && $src_format eq "qcow2");
6718     push @$cmd, '-t', 'none' if $dst_scfg->{type} eq 'zfspool';
6719     push @$cmd, '-T', $cachemode if defined($cachemode);
6720
6721     if ($src_is_iscsi) {
6722         push @$cmd, '--image-opts';
6723         $src_path = convert_iscsi_path($src_path);
6724     } else {
6725         push @$cmd, '-f', $src_format;
6726     }
6727
6728     if ($dst_is_iscsi) {
6729         push @$cmd, '--target-image-opts';
6730         $dst_path = convert_iscsi_path($dst_path);
6731     } else {
6732         push @$cmd, '-O', $dst_format;
6733     }
6734
6735     push @$cmd, $src_path;
6736
6737     if (!$dst_is_iscsi && $is_zero_initialized) {
6738         push @$cmd, "zeroinit:$dst_path";
6739     } else {
6740         push @$cmd, $dst_path;
6741     }
6742
6743     my $parser = sub {
6744         my $line = shift;
6745         if($line =~ m/\((\S+)\/100\%\)/){
6746             my $percent = $1;
6747             my $transferred = int($size * $percent / 100);
6748             my $remaining = $size - $transferred;
6749
6750             print "transferred: $transferred bytes remaining: $remaining bytes total: $size bytes progression: $percent %\n";
6751         }
6752
6753     };
6754
6755     eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
6756     my $err = $@;
6757     die "copy failed: $err" if $err;
6758 }
6759
6760 sub qemu_img_format {
6761     my ($scfg, $volname) = @_;
6762
6763     if ($scfg->{path} && $volname =~ m/\.($QEMU_FORMAT_RE)$/) {
6764         return $1;
6765     } else {
6766         return "raw";
6767     }
6768 }
6769
6770 sub qemu_drive_mirror {
6771     my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $skipcomplete, $qga, $bwlimit) = @_;
6772
6773     $jobs = {} if !$jobs;
6774
6775     my $qemu_target;
6776     my $format;
6777     $jobs->{"drive-$drive"} = {};
6778
6779     if ($dst_volid =~ /^nbd:/) {
6780         $qemu_target = $dst_volid;
6781         $format = "nbd";
6782     } else {
6783         my $storecfg = PVE::Storage::config();
6784         my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
6785
6786         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6787
6788         $format = qemu_img_format($dst_scfg, $dst_volname);
6789
6790         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6791
6792         $qemu_target = $is_zero_initialized ? "zeroinit:$dst_path" : $dst_path;
6793     }
6794
6795     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $qemu_target };
6796     $opts->{format} = $format if $format;
6797
6798     if (defined($bwlimit)) {
6799         $opts->{speed} = $bwlimit * 1024;
6800         print "drive mirror is starting for drive-$drive with bandwidth limit: ${bwlimit} KB/s\n";
6801     } else {
6802         print "drive mirror is starting for drive-$drive\n";
6803     }
6804
6805     # if a job already runs for this device we get an error, catch it for cleanup
6806     eval { mon_cmd($vmid, "drive-mirror", %$opts); };
6807     if (my $err = $@) {
6808         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6809         warn "$@\n" if $@;
6810         die "mirroring error: $err\n";
6811     }
6812
6813     qemu_drive_mirror_monitor ($vmid, $vmiddst, $jobs, $skipcomplete, $qga);
6814 }
6815
6816 sub qemu_drive_mirror_monitor {
6817     my ($vmid, $vmiddst, $jobs, $skipcomplete, $qga) = @_;
6818
6819     eval {
6820         my $err_complete = 0;
6821
6822         while (1) {
6823             die "storage migration timed out\n" if $err_complete > 300;
6824
6825             my $stats = mon_cmd($vmid, "query-block-jobs");
6826
6827             my $running_mirror_jobs = {};
6828             foreach my $stat (@$stats) {
6829                 next if $stat->{type} ne 'mirror';
6830                 $running_mirror_jobs->{$stat->{device}} = $stat;
6831             }
6832
6833             my $readycounter = 0;
6834
6835             foreach my $job (keys %$jobs) {
6836
6837                 if(defined($jobs->{$job}->{complete}) && !defined($running_mirror_jobs->{$job})) {
6838                     print "$job : finished\n";
6839                     delete $jobs->{$job};
6840                     next;
6841                 }
6842
6843                 die "$job: mirroring has been cancelled\n" if !defined($running_mirror_jobs->{$job});
6844
6845                 my $busy = $running_mirror_jobs->{$job}->{busy};
6846                 my $ready = $running_mirror_jobs->{$job}->{ready};
6847                 if (my $total = $running_mirror_jobs->{$job}->{len}) {
6848                     my $transferred = $running_mirror_jobs->{$job}->{offset} || 0;
6849                     my $remaining = $total - $transferred;
6850                     my $percent = sprintf "%.2f", ($transferred * 100 / $total);
6851
6852                     print "$job: transferred: $transferred bytes remaining: $remaining bytes total: $total bytes progression: $percent % busy: $busy ready: $ready \n";
6853                 }
6854
6855                 $readycounter++ if $running_mirror_jobs->{$job}->{ready};
6856             }
6857
6858             last if scalar(keys %$jobs) == 0;
6859
6860             if ($readycounter == scalar(keys %$jobs)) {
6861                 print "all mirroring jobs are ready \n";
6862                 last if $skipcomplete; #do the complete later
6863
6864                 if ($vmiddst && $vmiddst != $vmid) {
6865                     my $agent_running = $qga && qga_check_running($vmid);
6866                     if ($agent_running) {
6867                         print "freeze filesystem\n";
6868                         eval { mon_cmd($vmid, "guest-fsfreeze-freeze"); };
6869                     } else {
6870                         print "suspend vm\n";
6871                         eval { PVE::QemuServer::vm_suspend($vmid, 1); };
6872                     }
6873
6874                     # if we clone a disk for a new target vm, we don't switch the disk
6875                     PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs);
6876
6877                     if ($agent_running) {
6878                         print "unfreeze filesystem\n";
6879                         eval { mon_cmd($vmid, "guest-fsfreeze-thaw"); };
6880                     } else {
6881                         print "resume vm\n";
6882                         eval {  PVE::QemuServer::vm_resume($vmid, 1, 1); };
6883                     }
6884
6885                     last;
6886                 } else {
6887
6888                     foreach my $job (keys %$jobs) {
6889                         # try to switch the disk if source and destination are on the same guest
6890                         print "$job: Completing block job...\n";
6891
6892                         eval { mon_cmd($vmid, "block-job-complete", device => $job) };
6893                         if ($@ =~ m/cannot be completed/) {
6894                             print "$job: Block job cannot be completed, try again.\n";
6895                             $err_complete++;
6896                         }else {
6897                             print "$job: Completed successfully.\n";
6898                             $jobs->{$job}->{complete} = 1;
6899                         }
6900                     }
6901                 }
6902             }
6903             sleep 1;
6904         }
6905     };
6906     my $err = $@;
6907
6908     if ($err) {
6909         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6910         die "mirroring error: $err";
6911     }
6912
6913 }
6914
6915 sub qemu_blockjobs_cancel {
6916     my ($vmid, $jobs) = @_;
6917
6918     foreach my $job (keys %$jobs) {
6919         print "$job: Cancelling block job\n";
6920         eval { mon_cmd($vmid, "block-job-cancel", device => $job); };
6921         $jobs->{$job}->{cancel} = 1;
6922     }
6923
6924     while (1) {
6925         my $stats = mon_cmd($vmid, "query-block-jobs");
6926
6927         my $running_jobs = {};
6928         foreach my $stat (@$stats) {
6929             $running_jobs->{$stat->{device}} = $stat;
6930         }
6931
6932         foreach my $job (keys %$jobs) {
6933
6934             if (defined($jobs->{$job}->{cancel}) && !defined($running_jobs->{$job})) {
6935                 print "$job: Done.\n";
6936                 delete $jobs->{$job};
6937             }
6938         }
6939
6940         last if scalar(keys %$jobs) == 0;
6941
6942         sleep 1;
6943     }
6944 }
6945
6946 sub clone_disk {
6947     my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
6948         $newvmid, $storage, $format, $full, $newvollist, $jobs, $skipcomplete, $qga, $bwlimit) = @_;
6949
6950     my $newvolid;
6951
6952     if (!$full) {
6953         print "create linked clone of drive $drivename ($drive->{file})\n";
6954         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
6955         push @$newvollist, $newvolid;
6956     } else {
6957
6958         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6959         $storeid = $storage if $storage;
6960
6961         my $dst_format = resolve_dst_disk_format($storecfg, $storeid, $volname, $format);
6962         my ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 3);
6963
6964         print "create full clone of drive $drivename ($drive->{file})\n";
6965         my $name = undef;
6966         if (drive_is_cloudinit($drive)) {
6967             $name = "vm-$newvmid-cloudinit";
6968             $name .= ".$dst_format" if $dst_format ne 'raw';
6969             $snapname = undef;
6970             $size = PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE;
6971         }
6972         $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $newvmid, $dst_format, $name, ($size/1024));
6973         push @$newvollist, $newvolid;
6974
6975         PVE::Storage::activate_volumes($storecfg, [$newvolid]);
6976
6977         if (drive_is_cloudinit($drive)) {
6978             goto no_data_clone;
6979         }
6980
6981         my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
6982         if (!$running || $snapname) {
6983             # TODO: handle bwlimits
6984             qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit);
6985         } else {
6986
6987             my $kvmver = get_running_qemu_version ($vmid);
6988             if (!min_version($kvmver, 2, 7)) {
6989                 die "drive-mirror with iothread requires qemu version 2.7 or higher\n"
6990                     if $drive->{iothread};
6991             }
6992
6993             qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid, $sparseinit, $jobs, $skipcomplete, $qga, $bwlimit);
6994         }
6995     }
6996
6997 no_data_clone:
6998     my ($size) = PVE::Storage::volume_size_info($storecfg, $newvolid, 3);
6999
7000     my $disk = $drive;
7001     $disk->{format} = undef;
7002     $disk->{file} = $newvolid;
7003     $disk->{size} = $size;
7004
7005     return $disk;
7006 }
7007
7008 sub get_running_qemu_version {
7009     my ($vmid) = @_;
7010     my $res = mon_cmd($vmid, "query-version");
7011     return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
7012 }
7013
7014 sub qemu_use_old_bios_files {
7015     my ($machine_type) = @_;
7016
7017     return if !$machine_type;
7018
7019     my $use_old_bios_files = undef;
7020
7021     if ($machine_type =~ m/^(\S+)\.pxe$/) {
7022         $machine_type = $1;
7023         $use_old_bios_files = 1;
7024     } else {
7025         my $version = PVE::QemuServer::Machine::extract_version($machine_type) // kvm_user_version();
7026         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
7027         # load new efi bios files on migration. So this hack is required to allow
7028         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
7029         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
7030         $use_old_bios_files = !min_version($version, 2, 4);
7031     }
7032
7033     return ($use_old_bios_files, $machine_type);
7034 }
7035
7036 sub create_efidisk($$$$$) {
7037     my ($storecfg, $storeid, $vmid, $fmt, $arch) = @_;
7038
7039     my (undef, $ovmf_vars) = get_ovmf_files($arch);
7040     die "EFI vars default image not found\n" if ! -f $ovmf_vars;
7041
7042     my $vars_size_b = -s $ovmf_vars;
7043     my $vars_size = PVE::Tools::convert_size($vars_size_b, 'b' => 'kb');
7044     my $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $fmt, undef, $vars_size);
7045     PVE::Storage::activate_volumes($storecfg, [$volid]);
7046
7047     qemu_img_convert($ovmf_vars, $volid, $vars_size_b, undef, 0);
7048
7049     return ($volid, $vars_size);
7050 }
7051
7052 sub vm_iothreads_list {
7053     my ($vmid) = @_;
7054
7055     my $res = mon_cmd($vmid, 'query-iothreads');
7056
7057     my $iothreads = {};
7058     foreach my $iothread (@$res) {
7059         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
7060     }
7061
7062     return $iothreads;
7063 }
7064
7065 sub scsihw_infos {
7066     my ($conf, $drive) = @_;
7067
7068     my $maxdev = 0;
7069
7070     if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)) {
7071         $maxdev = 7;
7072     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
7073         $maxdev = 1;
7074     } else {
7075         $maxdev = 256;
7076     }
7077
7078     my $controller = int($drive->{index} / $maxdev);
7079     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single') ? "virtioscsi" : "scsihw";
7080
7081     return ($maxdev, $controller, $controller_prefix);
7082 }
7083
7084 sub add_hyperv_enlightenments {
7085     my ($cpuFlags, $winversion, $machine_version, $bios, $gpu_passthrough, $hv_vendor_id) = @_;
7086
7087     return if $winversion < 6;
7088     return if $bios && $bios eq 'ovmf' && $winversion < 8;
7089
7090     if ($gpu_passthrough || defined($hv_vendor_id)) {
7091         $hv_vendor_id //= 'proxmox';
7092         push @$cpuFlags , "hv_vendor_id=$hv_vendor_id";
7093     }
7094
7095     if (min_version($machine_version, 2, 3)) {
7096         push @$cpuFlags , 'hv_spinlocks=0x1fff';
7097         push @$cpuFlags , 'hv_vapic';
7098         push @$cpuFlags , 'hv_time';
7099     } else {
7100         push @$cpuFlags , 'hv_spinlocks=0xffff';
7101     }
7102
7103     if (min_version($machine_version, 2, 6)) {
7104         push @$cpuFlags , 'hv_reset';
7105         push @$cpuFlags , 'hv_vpindex';
7106         push @$cpuFlags , 'hv_runtime';
7107     }
7108
7109     if ($winversion >= 7) {
7110         push @$cpuFlags , 'hv_relaxed';
7111
7112         if (min_version($machine_version, 2, 12)) {
7113             push @$cpuFlags , 'hv_synic';
7114             push @$cpuFlags , 'hv_stimer';
7115         }
7116
7117         if (min_version($machine_version, 3, 1)) {
7118             push @$cpuFlags , 'hv_ipi';
7119         }
7120     }
7121 }
7122
7123 sub windows_version {
7124     my ($ostype) = @_;
7125
7126     return 0 if !$ostype;
7127
7128     my $winversion = 0;
7129
7130     if($ostype eq 'wxp' || $ostype eq 'w2k3' || $ostype eq 'w2k') {
7131         $winversion = 5;
7132     } elsif($ostype eq 'w2k8' || $ostype eq 'wvista') {
7133         $winversion = 6;
7134     } elsif ($ostype =~ m/^win(\d+)$/) {
7135         $winversion = $1;
7136     }
7137
7138     return $winversion;
7139 }
7140
7141 sub resolve_dst_disk_format {
7142         my ($storecfg, $storeid, $src_volname, $format) = @_;
7143         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
7144
7145         if (!$format) {
7146             # if no target format is specified, use the source disk format as hint
7147             if ($src_volname) {
7148                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
7149                 $format = qemu_img_format($scfg, $src_volname);
7150             } else {
7151                 return $defFormat;
7152             }
7153         }
7154
7155         # test if requested format is supported - else use default
7156         my $supported = grep { $_ eq $format } @$validFormats;
7157         $format = $defFormat if !$supported;
7158         return $format;
7159 }
7160
7161 sub resolve_first_disk {
7162     my $conf = shift;
7163     my @disks = PVE::QemuServer::valid_drive_names();
7164     my $firstdisk;
7165     foreach my $ds (reverse @disks) {
7166         next if !$conf->{$ds};
7167         my $disk = PVE::QemuServer::parse_drive($ds, $conf->{$ds});
7168         next if PVE::QemuServer::drive_is_cdrom($disk);
7169         $firstdisk = $ds;
7170     }
7171     return $firstdisk;
7172 }
7173
7174 sub generate_uuid {
7175     my ($uuid, $uuid_str);
7176     UUID::generate($uuid);
7177     UUID::unparse($uuid, $uuid_str);
7178     return $uuid_str;
7179 }
7180
7181 sub generate_smbios1_uuid {
7182     return "uuid=".generate_uuid();
7183 }
7184
7185 sub nbd_stop {
7186     my ($vmid) = @_;
7187
7188     mon_cmd($vmid, 'nbd-server-stop');
7189 }
7190
7191 sub create_reboot_request {
7192     my ($vmid) = @_;
7193     open(my $fh, '>', "/run/qemu-server/$vmid.reboot")
7194         or die "failed to create reboot trigger file: $!\n";
7195     close($fh);
7196 }
7197
7198 sub clear_reboot_request {
7199     my ($vmid) = @_;
7200     my $path = "/run/qemu-server/$vmid.reboot";
7201     my $res = 0;
7202
7203     $res = unlink($path);
7204     die "could not remove reboot request for $vmid: $!"
7205         if !$res && $! != POSIX::ENOENT;
7206
7207     return $res;
7208 }
7209
7210 # bash completion helper
7211
7212 sub complete_backup_archives {
7213     my ($cmdname, $pname, $cvalue) = @_;
7214
7215     my $cfg = PVE::Storage::config();
7216
7217     my $storeid;
7218
7219     if ($cvalue =~ m/^([^:]+):/) {
7220         $storeid = $1;
7221     }
7222
7223     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
7224
7225     my $res = [];
7226     foreach my $id (keys %$data) {
7227         foreach my $item (@{$data->{$id}}) {
7228             next if $item->{format} !~ m/^vma\.(gz|lzo)$/;
7229             push @$res, $item->{volid} if defined($item->{volid});
7230         }
7231     }
7232
7233     return $res;
7234 }
7235
7236 my $complete_vmid_full = sub {
7237     my ($running) = @_;
7238
7239     my $idlist = vmstatus();
7240
7241     my $res = [];
7242
7243     foreach my $id (keys %$idlist) {
7244         my $d = $idlist->{$id};
7245         if (defined($running)) {
7246             next if $d->{template};
7247             next if $running && $d->{status} ne 'running';
7248             next if !$running && $d->{status} eq 'running';
7249         }
7250         push @$res, $id;
7251
7252     }
7253     return $res;
7254 };
7255
7256 sub complete_vmid {
7257     return &$complete_vmid_full();
7258 }
7259
7260 sub complete_vmid_stopped {
7261     return &$complete_vmid_full(0);
7262 }
7263
7264 sub complete_vmid_running {
7265     return &$complete_vmid_full(1);
7266 }
7267
7268 sub complete_storage {
7269
7270     my $cfg = PVE::Storage::config();
7271     my $ids = $cfg->{ids};
7272
7273     my $res = [];
7274     foreach my $sid (keys %$ids) {
7275         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
7276         next if !$ids->{$sid}->{content}->{images};
7277         push @$res, $sid;
7278     }
7279
7280     return $res;
7281 }
7282
7283 sub complete_migration_storage {
7284     my ($cmd, $param, $current_value, $all_args) = @_;
7285
7286     my $targetnode = @$all_args[1];
7287
7288     my $cfg = PVE::Storage::config();
7289     my $ids = $cfg->{ids};
7290
7291     my $res = [];
7292     foreach my $sid (keys %$ids) {
7293         next if !PVE::Storage::storage_check_enabled($cfg, $sid, $targetnode, 1);
7294         next if !$ids->{$sid}->{content}->{images};
7295         push @$res, $sid;
7296     }
7297
7298     return $res;
7299 }
7300
7301 1;