PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5
   6 use Cwd 'abs_path';
   7 use Digest::SHA;
   8 use Fcntl ':flock';
   9 use Fcntl;
  10 use File::Basename;
  11 use File::Copy qw(copy);
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use IO::Dir;
  16 use IO::File;
  17 use IO::Handle;
  18 use IO::Select;
  19 use IO::Socket::UNIX;
  20 use IPC::Open3;
  21 use JSON;
  22 use MIME::Base64;
  23 use POSIX;
  24 use Storable qw(dclone);
  25 use Time::HiRes qw(gettimeofday);
  26 use URI::Escape;
  27 use UUID;
  28
  29 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file cfs_lock_file);
  30 use PVE::DataCenterConfig;
  31 use PVE::Exception qw(raise raise_param_exc);
  32 use PVE::GuestHelpers;
  33 use PVE::INotify;
  34 use PVE::JSONSchema qw(get_standard_option);
  35 use PVE::ProcFSTools;
  36 use PVE::RPCEnvironment;
  37 use PVE::Storage;
  38 use PVE::SysFSTools;
  39 use PVE::Systemd;
  40 use PVE::Tools qw(run_command lock_file lock_file_full file_read_firstline dir_glob_foreach get_host_arch $IPV6RE);
  41
  42 use PVE::QMPClient;
  43 use PVE::QemuConfig;
  44 use PVE::QemuServer::Helpers qw(min_version);
  45 use PVE::QemuServer::Cloudinit;
  46 use PVE::QemuServer::Machine;
  47 use PVE::QemuServer::Memory;
  48 use PVE::QemuServer::Monitor qw(mon_cmd);
  49 use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr print_pcie_root_port);
  50 use PVE::QemuServer::USB qw(parse_usb_device);
  51
  52 my $EDK2_FW_BASE = '/usr/share/pve-edk2-firmware/';
  53 my $OVMF = {
  54     x86_64 => [
  55         "$EDK2_FW_BASE/OVMF_CODE.fd",
  56         "$EDK2_FW_BASE/OVMF_VARS.fd"
  57     ],
  58     aarch64 => [
  59         "$EDK2_FW_BASE/AAVMF_CODE.fd",
  60         "$EDK2_FW_BASE/AAVMF_VARS.fd"
  61     ],
  62 };
  63
  64 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  65
  66 my $QEMU_FORMAT_RE = qr/raw|cow|qcow|qcow2|qed|vmdk|cloop/;
  67
  68 # Note about locking: we use flock on the config file protect
  69 # against concurent actions.
  70 # Aditionaly, we have a 'lock' setting in the config file. This
  71 # can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
  72 # allowed when such lock is set. But you can ignore this kind of
  73 # lock with the --skiplock flag.
  74
  75 cfs_register_file('/qemu-server/',
  76                   \&parse_vm_config,
  77                   \&write_vm_config);
  78
  79 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
  80     description => "Some command save/restore state from this location.",
  81     type => 'string',
  82     maxLength => 128,
  83     optional => 1,
  84 });
  85
  86 PVE::JSONSchema::register_standard_option('pve-qm-image-format', {
  87     type => 'string',
  88     enum => [qw(raw cow qcow qed qcow2 vmdk cloop)],
  89     description => "The drive's backing file's data format.",
  90     optional => 1,
  91 });
  92
  93 PVE::JSONSchema::register_standard_option('pve-qemu-machine', {
  94         description => "Specifies the Qemu machine type.",
  95         type => 'string',
  96         pattern => '(pc|pc(-i440fx)?-\d+(\.\d+)+(\.pxe)?|q35|pc-q35-\d+(\.\d+)+(\.pxe)?|virt(?:-\d+(\.\d+)+)?)',
  97         maxLength => 40,
  98         optional => 1,
  99 });
 100
 101 #no warnings 'redefine';
 102
 103 sub cgroups_write {
 104    my ($controller, $vmid, $option, $value) = @_;
 105
 106    my $path = "/sys/fs/cgroup/$controller/qemu.slice/$vmid.scope/$option";
 107    PVE::ProcFSTools::write_proc_entry($path, $value);
 108
 109 }
 110
 111 my $nodename = PVE::INotify::nodename();
 112
 113 my $cpu_vendor_list = {
 114     # Intel CPUs
 115     486 => 'GenuineIntel',
 116     pentium => 'GenuineIntel',
 117     pentium2  => 'GenuineIntel',
 118     pentium3  => 'GenuineIntel',
 119     coreduo => 'GenuineIntel',
 120     core2duo => 'GenuineIntel',
 121     Conroe  => 'GenuineIntel',
 122     Penryn  => 'GenuineIntel',
 123     Nehalem  => 'GenuineIntel',
 124     'Nehalem-IBRS'  => 'GenuineIntel',
 125     Westmere => 'GenuineIntel',
 126     'Westmere-IBRS' => 'GenuineIntel',
 127     SandyBridge => 'GenuineIntel',
 128     'SandyBridge-IBRS' => 'GenuineIntel',
 129     IvyBridge => 'GenuineIntel',
 130     'IvyBridge-IBRS' => 'GenuineIntel',
 131     Haswell => 'GenuineIntel',
 132     'Haswell-IBRS' => 'GenuineIntel',
 133     'Haswell-noTSX' => 'GenuineIntel',
 134     'Haswell-noTSX-IBRS' => 'GenuineIntel',
 135     Broadwell => 'GenuineIntel',
 136     'Broadwell-IBRS' => 'GenuineIntel',
 137     'Broadwell-noTSX' => 'GenuineIntel',
 138     'Broadwell-noTSX-IBRS' => 'GenuineIntel',
 139     'Skylake-Client' => 'GenuineIntel',
 140     'Skylake-Client-IBRS' => 'GenuineIntel',
 141     'Skylake-Server' => 'GenuineIntel',
 142     'Skylake-Server-IBRS' => 'GenuineIntel',
 143     'Cascadelake-Server' => 'GenuineIntel',
 144     KnightsMill => 'GenuineIntel',
 145
 146
 147     # AMD CPUs
 148     athlon => 'AuthenticAMD',
 149     phenom  => 'AuthenticAMD',
 150     Opteron_G1  => 'AuthenticAMD',
 151     Opteron_G2  => 'AuthenticAMD',
 152     Opteron_G3  => 'AuthenticAMD',
 153     Opteron_G4  => 'AuthenticAMD',
 154     Opteron_G5  => 'AuthenticAMD',
 155     EPYC => 'AuthenticAMD',
 156     'EPYC-IBPB' => 'AuthenticAMD',
 157
 158     # generic types, use vendor from host node
 159     host => 'default',
 160     kvm32 => 'default',
 161     kvm64 => 'default',
 162     qemu32 => 'default',
 163     qemu64 => 'default',
 164     max => 'default',
 165 };
 166
 167 my @supported_cpu_flags = (
 168     'pcid',
 169     'spec-ctrl',
 170     'ibpb',
 171     'ssbd',
 172     'virt-ssbd',
 173     'amd-ssbd',
 174     'amd-no-ssb',
 175     'pdpe1gb',
 176     'md-clear',
 177     'hv-tlbflush',
 178     'hv-evmcs',
 179     'aes'
 180 );
 181 my $cpu_flag = qr/[+-](@{[join('|', @supported_cpu_flags)]})/;
 182
 183 my $cpu_fmt = {
 184     cputype => {
 185         description => "Emulated CPU type.",
 186         type => 'string',
 187         enum => [ sort { "\L$a" cmp "\L$b" } keys %$cpu_vendor_list ],
 188         default => 'kvm64',
 189         default_key => 1,
 190     },
 191     hidden => {
 192         description => "Do not identify as a KVM virtual machine.",
 193         type => 'boolean',
 194         optional => 1,
 195         default => 0
 196     },
 197     'hv-vendor-id' => {
 198         type => 'string',
 199         pattern => qr/[a-zA-Z0-9]{1,12}/,
 200         format_description => 'vendor-id',
 201         description => 'The Hyper-V vendor ID. Some drivers or programs inside Windows guests need a specific ID.',
 202         optional => 1,
 203     },
 204     flags => {
 205         description => "List of additional CPU flags separated by ';'."
 206                      . " Use '+FLAG' to enable, '-FLAG' to disable a flag."
 207                      . " Currently supported flags: @{[join(', ', @supported_cpu_flags)]}.",
 208         format_description => '+FLAG[;-FLAG...]',
 209         type => 'string',
 210         pattern => qr/$cpu_flag(;$cpu_flag)*/,
 211         optional => 1,
 212     },
 213 };
 214
 215 my $watchdog_fmt = {
 216     model => {
 217         default_key => 1,
 218         type => 'string',
 219         enum => [qw(i6300esb ib700)],
 220         description => "Watchdog type to emulate.",
 221         default => 'i6300esb',
 222         optional => 1,
 223     },
 224     action => {
 225         type => 'string',
 226         enum => [qw(reset shutdown poweroff pause debug none)],
 227         description => "The action to perform if after activation the guest fails to poll the watchdog in time.",
 228         optional => 1,
 229     },
 230 };
 231 PVE::JSONSchema::register_format('pve-qm-watchdog', $watchdog_fmt);
 232
 233 my $agent_fmt = {
 234     enabled => {
 235         description => "Enable/disable Qemu GuestAgent.",
 236         type => 'boolean',
 237         default => 0,
 238         default_key => 1,
 239     },
 240     fstrim_cloned_disks => {
 241         description => "Run fstrim after cloning/moving a disk.",
 242         type => 'boolean',
 243         optional => 1,
 244         default => 0
 245     },
 246     type => {
 247         description => "Select the agent type",
 248         type => 'string',
 249         default => 'virtio',
 250         optional => 1,
 251         enum => [qw(virtio isa)],
 252     },
 253 };
 254
 255 my $vga_fmt = {
 256     type => {
 257         description => "Select the VGA type.",
 258         type => 'string',
 259         default => 'std',
 260         optional => 1,
 261         default_key => 1,
 262         enum => [qw(cirrus qxl qxl2 qxl3 qxl4 none serial0 serial1 serial2 serial3 std virtio vmware)],
 263     },
 264     memory => {
 265         description => "Sets the VGA memory (in MiB). Has no effect with serial display.",
 266         type => 'integer',
 267         optional => 1,
 268         minimum => 4,
 269         maximum => 512,
 270     },
 271 };
 272
 273 my $ivshmem_fmt = {
 274     size => {
 275         type => 'integer',
 276         minimum => 1,
 277         description => "The size of the file in MB.",
 278     },
 279     name => {
 280         type => 'string',
 281         pattern => '[a-zA-Z0-9\-]+',
 282         optional => 1,
 283         format_description => 'string',
 284         description => "The name of the file. Will be prefixed with 'pve-shm-'. Default is the VMID. Will be deleted when the VM is stopped.",
 285     },
 286 };
 287
 288 my $audio_fmt = {
 289     device => {
 290         type => 'string',
 291         enum => [qw(ich9-intel-hda intel-hda AC97)],
 292         description =>  "Configure an audio device."
 293     },
 294     driver =>  {
 295         type => 'string',
 296         enum => ['spice'],
 297         default => 'spice',
 298         optional => 1,
 299         description => "Driver backend for the audio device."
 300     },
 301 };
 302
 303 my $spice_enhancements_fmt = {
 304     foldersharing => {
 305         type => 'boolean',
 306         optional => 1,
 307         default => '0',
 308         description =>  "Enable folder sharing via SPICE. Needs Spice-WebDAV daemon installed in the VM."
 309     },
 310     videostreaming =>  {
 311         type => 'string',
 312         enum => ['off', 'all', 'filter'],
 313         default => 'off',
 314         optional => 1,
 315         description => "Enable video streaming. Uses compression for detected video streams."
 316     },
 317 };
 318
 319 my $confdesc = {
 320     onboot => {
 321         optional => 1,
 322         type => 'boolean',
 323         description => "Specifies whether a VM will be started during system bootup.",
 324         default => 0,
 325     },
 326     autostart => {
 327         optional => 1,
 328         type => 'boolean',
 329         description => "Automatic restart after crash (currently ignored).",
 330         default => 0,
 331     },
 332     hotplug => {
 333         optional => 1,
 334         type => 'string', format => 'pve-hotplug-features',
 335         description => "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
 336         default => 'network,disk,usb',
 337     },
 338     reboot => {
 339         optional => 1,
 340         type => 'boolean',
 341         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 342         default => 1,
 343     },
 344     lock => {
 345         optional => 1,
 346         type => 'string',
 347         description => "Lock/unlock the VM.",
 348         enum => [qw(backup clone create migrate rollback snapshot snapshot-delete suspending suspended)],
 349     },
 350     cpulimit => {
 351         optional => 1,
 352         type => 'number',
 353         description => "Limit of CPU usage.",
 354         verbose_description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
 355         minimum => 0,
 356         maximum => 128,
 357         default => 0,
 358     },
 359     cpuunits => {
 360         optional => 1,
 361         type => 'integer',
 362         description => "CPU weight for a VM.",
 363         verbose_description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.",
 364         minimum => 2,
 365         maximum => 262144,
 366         default => 1024,
 367     },
 368     memory => {
 369         optional => 1,
 370         type => 'integer',
 371         description => "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
 372         minimum => 16,
 373         default => 512,
 374     },
 375     balloon => {
 376         optional => 1,
 377         type => 'integer',
 378         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 379         minimum => 0,
 380     },
 381     shares => {
 382         optional => 1,
 383         type => 'integer',
 384         description => "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning. Auto-ballooning is done by pvestatd.",
 385         minimum => 0,
 386         maximum => 50000,
 387         default => 1000,
 388     },
 389     keyboard => {
 390         optional => 1,
 391         type => 'string',
 392         description => "Keybord layout for vnc server. Default is read from the '/etc/pve/datacenter.cfg' configuration file.".
 393                        "It should not be necessary to set it.",
 394         enum => PVE::Tools::kvmkeymaplist(),
 395         default => undef,
 396     },
 397     name => {
 398         optional => 1,
 399         type => 'string', format => 'dns-name',
 400         description => "Set a name for the VM. Only used on the configuration web interface.",
 401     },
 402     scsihw => {
 403         optional => 1,
 404         type => 'string',
 405         description => "SCSI controller model",
 406         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 407         default => 'lsi',
 408     },
 409     description => {
 410         optional => 1,
 411         type => 'string',
 412         description => "Description for the VM. Only used on the configuration web interface. This is saved as comment inside the configuration file.",
 413     },
 414     ostype => {
 415         optional => 1,
 416         type => 'string',
 417         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 l24 l26 solaris)],
 418         description => "Specify guest operating system.",
 419         verbose_description => <<EODESC,
 420 Specify guest operating system. This is used to enable special
 421 optimization/features for specific operating systems:
 422
 423 [horizontal]
 424 other;; unspecified OS
 425 wxp;; Microsoft Windows XP
 426 w2k;; Microsoft Windows 2000
 427 w2k3;; Microsoft Windows 2003
 428 w2k8;; Microsoft Windows 2008
 429 wvista;; Microsoft Windows Vista
 430 win7;; Microsoft Windows 7
 431 win8;; Microsoft Windows 8/2012/2012r2
 432 win10;; Microsoft Windows 10/2016
 433 l24;; Linux 2.4 Kernel
 434 l26;; Linux 2.6 - 5.X Kernel
 435 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
 436 EODESC
 437     },
 438     boot => {
 439         optional => 1,
 440         type => 'string',
 441         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n).",
 442         pattern => '[acdn]{1,4}',
 443         default => 'cdn',
 444     },
 445     bootdisk => {
 446         optional => 1,
 447         type => 'string', format => 'pve-qm-bootdisk',
 448         description => "Enable booting from specified disk.",
 449         pattern => '(ide|sata|scsi|virtio)\d+',
 450     },
 451     smp => {
 452         optional => 1,
 453         type => 'integer',
 454         description => "The number of CPUs. Please use option -sockets instead.",
 455         minimum => 1,
 456         default => 1,
 457     },
 458     sockets => {
 459         optional => 1,
 460         type => 'integer',
 461         description => "The number of CPU sockets.",
 462         minimum => 1,
 463         default => 1,
 464     },
 465     cores => {
 466         optional => 1,
 467         type => 'integer',
 468         description => "The number of cores per socket.",
 469         minimum => 1,
 470         default => 1,
 471     },
 472     numa => {
 473         optional => 1,
 474         type => 'boolean',
 475         description => "Enable/disable NUMA.",
 476         default => 0,
 477     },
 478     hugepages => {
 479         optional => 1,
 480         type => 'string',
 481         description => "Enable/disable hugepages memory.",
 482         enum => [qw(any 2 1024)],
 483     },
 484     vcpus => {
 485         optional => 1,
 486         type => 'integer',
 487         description => "Number of hotplugged vcpus.",
 488         minimum => 1,
 489         default => 0,
 490     },
 491     acpi => {
 492         optional => 1,
 493         type => 'boolean',
 494         description => "Enable/disable ACPI.",
 495         default => 1,
 496     },
 497     agent => {
 498         optional => 1,
 499         description => "Enable/disable Qemu GuestAgent and its properties.",
 500         type => 'string',
 501         format => $agent_fmt,
 502     },
 503     kvm => {
 504         optional => 1,
 505         type => 'boolean',
 506         description => "Enable/disable KVM hardware virtualization.",
 507         default => 1,
 508     },
 509     tdf => {
 510         optional => 1,
 511         type => 'boolean',
 512         description => "Enable/disable time drift fix.",
 513         default => 0,
 514     },
 515     localtime => {
 516         optional => 1,
 517         type => 'boolean',
 518         description => "Set the real time clock to local time. This is enabled by default if ostype indicates a Microsoft OS.",
 519     },
 520     freeze => {
 521         optional => 1,
 522         type => 'boolean',
 523         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 524     },
 525     vga => {
 526         optional => 1,
 527         type => 'string', format => $vga_fmt,
 528         description => "Configure the VGA hardware.",
 529         verbose_description => "Configure the VGA Hardware. If you want to use ".
 530             "high resolution modes (>= 1280x1024x16) you may need to increase " .
 531             "the vga memory option. Since QEMU 2.9 the default VGA display type " .
 532             "is 'std' for all OS types besides some Windows versions (XP and " .
 533             "older) which use 'cirrus'. The 'qxl' option enables the SPICE " .
 534             "display server. For win* OS you can select how many independent " .
 535             "displays you want, Linux guests can add displays them self.\n".
 536             "You can also run without any graphic card, using a serial device as terminal.",
 537     },
 538     watchdog => {
 539         optional => 1,
 540         type => 'string', format => 'pve-qm-watchdog',
 541         description => "Create a virtual hardware watchdog device.",
 542         verbose_description => "Create a virtual hardware watchdog device. Once enabled" .
 543             " (by a guest action), the watchdog must be periodically polled " .
 544             "by an agent inside the guest or else the watchdog will reset " .
 545             "the guest (or execute the respective action specified)",
 546     },
 547     startdate => {
 548         optional => 1,
 549         type => 'string',
 550         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 551         description => "Set the initial date of the real time clock. Valid format for date are: 'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 552         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 553         default => 'now',
 554     },
 555     startup =>  get_standard_option('pve-startup-order'),
 556     template => {
 557         optional => 1,
 558         type => 'boolean',
 559         description => "Enable/disable Template.",
 560         default => 0,
 561     },
 562     args => {
 563         optional => 1,
 564         type => 'string',
 565         description => "Arbitrary arguments passed to kvm.",
 566         verbose_description => <<EODESCR,
 567 Arbitrary arguments passed to kvm, for example:
 568
 569 args: -no-reboot -no-hpet
 570
 571 NOTE: this option is for experts only.
 572 EODESCR
 573     },
 574     tablet => {
 575         optional => 1,
 576         type => 'boolean',
 577         default => 1,
 578         description => "Enable/disable the USB tablet device.",
 579         verbose_description => "Enable/disable the USB tablet device. This device is " .
 580             "usually needed to allow absolute mouse positioning with VNC. " .
 581             "Else the mouse runs out of sync with normal VNC clients. " .
 582             "If you're running lots of console-only guests on one host, " .
 583             "you may consider disabling this to save some context switches. " .
 584             "This is turned off by default if you use spice (-vga=qxl).",
 585     },
 586     migrate_speed => {
 587         optional => 1,
 588         type => 'integer',
 589         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 590         minimum => 0,
 591         default => 0,
 592     },
 593     migrate_downtime => {
 594         optional => 1,
 595         type => 'number',
 596         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 597         minimum => 0,
 598         default => 0.1,
 599     },
 600     cdrom => {
 601         optional => 1,
 602         type => 'string', format => 'pve-qm-ide',
 603         typetext => '<volume>',
 604         description => "This is an alias for option -ide2",
 605     },
 606     cpu => {
 607         optional => 1,
 608         description => "Emulated CPU type.",
 609         type => 'string',
 610         format => $cpu_fmt,
 611     },
 612     parent => get_standard_option('pve-snapshot-name', {
 613         optional => 1,
 614         description => "Parent snapshot name. This is used internally, and should not be modified.",
 615     }),
 616     snaptime => {
 617         optional => 1,
 618         description => "Timestamp for snapshots.",
 619         type => 'integer',
 620         minimum => 0,
 621     },
 622     vmstate => {
 623         optional => 1,
 624         type => 'string', format => 'pve-volume-id',
 625         description => "Reference to a volume which stores the VM state. This is used internally for snapshots.",
 626     },
 627     vmstatestorage => get_standard_option('pve-storage-id', {
 628         description => "Default storage for VM state volumes/files.",
 629         optional => 1,
 630     }),
 631     runningmachine => get_standard_option('pve-qemu-machine', {
 632         description => "Specifies the Qemu machine type of the running vm. This is used internally for snapshots.",
 633     }),
 634     machine => get_standard_option('pve-qemu-machine'),
 635     arch => {
 636         description => "Virtual processor architecture. Defaults to the host.",
 637         optional => 1,
 638         type => 'string',
 639         enum => [qw(x86_64 aarch64)],
 640     },
 641     smbios1 => {
 642         description => "Specify SMBIOS type 1 fields.",
 643         type => 'string', format => 'pve-qm-smbios1',
 644         maxLength => 512,
 645         optional => 1,
 646     },
 647     protection => {
 648         optional => 1,
 649         type => 'boolean',
 650         description => "Sets the protection flag of the VM. This will disable the remove VM and remove disk operations.",
 651         default => 0,
 652     },
 653     bios => {
 654         optional => 1,
 655         type => 'string',
 656         enum => [ qw(seabios ovmf) ],
 657         description => "Select BIOS implementation.",
 658         default => 'seabios',
 659     },
 660     vmgenid => {
 661         type => 'string',
 662         pattern => '(?:[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}|[01])',
 663         format_description => 'UUID',
 664         description => "Set VM Generation ID. Use '1' to autogenerate on create or update, pass '0' to disable explicitly.",
 665         verbose_description => "The VM generation ID (vmgenid) device exposes a".
 666             " 128-bit integer value identifier to the guest OS. This allows to".
 667             " notify the guest operating system when the virtual machine is".
 668             " executed with a different configuration (e.g. snapshot execution".
 669             " or creation from a template). The guest operating system notices".
 670             " the change, and is then able to react as appropriate by marking".
 671             " its copies of distributed databases as dirty, re-initializing its".
 672             " random number generator, etc.\n".
 673             "Note that auto-creation only works when done throug API/CLI create".
 674             " or update methods, but not when manually editing the config file.",
 675         default => "1 (autogenerated)",
 676         optional => 1,
 677     },
 678     hookscript => {
 679         type => 'string',
 680         format => 'pve-volume-id',
 681         optional => 1,
 682         description => "Script that will be executed during various steps in the vms lifetime.",
 683     },
 684     ivshmem => {
 685         type => 'string',
 686         format => $ivshmem_fmt,
 687         description => "Inter-VM shared memory. Useful for direct communication between VMs, or to the host.",
 688         optional => 1,
 689     },
 690     audio0 => {
 691         type => 'string',
 692         format => $audio_fmt,
 693         description => "Configure a audio device, useful in combination with QXL/Spice.",
 694         optional => 1
 695     },
 696     spice_enhancements => {
 697         type => 'string',
 698         format => $spice_enhancements_fmt,
 699         description => "Configure additional enhancements for SPICE.",
 700         optional => 1
 701     },
 702     tags => {
 703         type => 'string', format => 'pve-tag-list',
 704         description => 'Tags of the VM. This is only meta information.',
 705         optional => 1,
 706     },
 707 };
 708
 709 my $cicustom_fmt = {
 710     meta => {
 711         type => 'string',
 712         optional => 1,
 713         description => 'Specify a custom file containing all meta data passed to the VM via cloud-init. This is provider specific meaning configdrive2 and nocloud differ.',
 714         format => 'pve-volume-id',
 715         format_description => 'volume',
 716     },
 717     network => {
 718         type => 'string',
 719         optional => 1,
 720         description => 'Specify a custom file containing all network data passed to the VM via cloud-init.',
 721         format => 'pve-volume-id',
 722         format_description => 'volume',
 723     },
 724     user => {
 725         type => 'string',
 726         optional => 1,
 727         description => 'Specify a custom file containing all user data passed to the VM via cloud-init.',
 728         format => 'pve-volume-id',
 729         format_description => 'volume',
 730     },
 731 };
 732 PVE::JSONSchema::register_format('pve-qm-cicustom', $cicustom_fmt);
 733
 734 my $confdesc_cloudinit = {
 735     citype => {
 736         optional => 1,
 737         type => 'string',
 738         description => 'Specifies the cloud-init configuration format. The default depends on the configured operating system type (`ostype`. We use the `nocloud` format for Linux, and `configdrive2` for windows.',
 739         enum => ['configdrive2', 'nocloud'],
 740     },
 741     ciuser => {
 742         optional => 1,
 743         type => 'string',
 744         description => "cloud-init: User name to change ssh keys and password for instead of the image's configured default user.",
 745     },
 746     cipassword => {
 747         optional => 1,
 748         type => 'string',
 749         description => 'cloud-init: Password to assign the user. Using this is generally not recommended. Use ssh keys instead. Also note that older cloud-init versions do not support hashed passwords.',
 750     },
 751     cicustom => {
 752         optional => 1,
 753         type => 'string',
 754         description => 'cloud-init: Specify custom files to replace the automatically generated ones at start.',
 755         format => 'pve-qm-cicustom',
 756     },
 757     searchdomain => {
 758         optional => 1,
 759         type => 'string',
 760         description => "cloud-init: Sets DNS search domains for a container. Create will automatically use the setting from the host if neither searchdomain nor nameserver are set.",
 761     },
 762     nameserver => {
 763         optional => 1,
 764         type => 'string', format => 'address-list',
 765         description => "cloud-init: Sets DNS server IP address for a container. Create will automatically use the setting from the host if neither searchdomain nor nameserver are set.",
 766     },
 767     sshkeys => {
 768         optional => 1,
 769         type => 'string',
 770         format => 'urlencoded',
 771         description => "cloud-init: Setup public SSH keys (one key per line, OpenSSH format).",
 772     },
 773 };
 774
 775 # what about other qemu settings ?
 776 #cpu => 'string',
 777 #machine => 'string',
 778 #fda => 'file',
 779 #fdb => 'file',
 780 #mtdblock => 'file',
 781 #sd => 'file',
 782 #pflash => 'file',
 783 #snapshot => 'bool',
 784 #bootp => 'file',
 785 ##tftp => 'dir',
 786 ##smb => 'dir',
 787 #kernel => 'file',
 788 #append => 'string',
 789 #initrd => 'file',
 790 ##soundhw => 'string',
 791
 792 while (my ($k, $v) = each %$confdesc) {
 793     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 794 }
 795
 796 my $MAX_IDE_DISKS = 4;
 797 my $MAX_SCSI_DISKS = 14;
 798 my $MAX_VIRTIO_DISKS = 16;
 799 my $MAX_SATA_DISKS = 6;
 800 my $MAX_USB_DEVICES = 5;
 801 my $MAX_NETS = 32;
 802 my $MAX_UNUSED_DISKS = 256;
 803 my $MAX_HOSTPCI_DEVICES = 16;
 804 my $MAX_SERIAL_PORTS = 4;
 805 my $MAX_PARALLEL_PORTS = 3;
 806 my $MAX_NUMA = 8;
 807
 808 my $numa_fmt = {
 809     cpus => {
 810         type => "string",
 811         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 812         description => "CPUs accessing this NUMA node.",
 813         format_description => "id[-id];...",
 814     },
 815     memory => {
 816         type => "number",
 817         description => "Amount of memory this NUMA node provides.",
 818         optional => 1,
 819     },
 820     hostnodes => {
 821         type => "string",
 822         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 823         description => "Host NUMA nodes to use.",
 824         format_description => "id[-id];...",
 825         optional => 1,
 826     },
 827     policy => {
 828         type => 'string',
 829         enum => [qw(preferred bind interleave)],
 830         description => "NUMA allocation policy.",
 831         optional => 1,
 832     },
 833 };
 834 PVE::JSONSchema::register_format('pve-qm-numanode', $numa_fmt);
 835 my $numadesc = {
 836     optional => 1,
 837     type => 'string', format => $numa_fmt,
 838     description => "NUMA topology.",
 839 };
 840 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 841
 842 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 843     $confdesc->{"numa$i"} = $numadesc;
 844 }
 845
 846 my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000',  'pcnet',  'virtio',
 847                       'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3',
 848                       'e1000-82540em', 'e1000-82544gc', 'e1000-82545em'];
 849 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 850
 851 my $net_fmt_bridge_descr = <<__EOD__;
 852 Bridge to attach the network device to. The Proxmox VE standard bridge
 853 is called 'vmbr0'.
 854
 855 If you do not specify a bridge, we create a kvm user (NATed) network
 856 device, which provides DHCP and DNS services. The following addresses
 857 are used:
 858
 859  10.0.2.2   Gateway
 860  10.0.2.3   DNS Server
 861  10.0.2.4   SMB Server
 862
 863 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 864 __EOD__
 865
 866 my $net_fmt = {
 867     macaddr  => get_standard_option('mac-addr', {
 868         description => "MAC address. That address must be unique withing your network. This is automatically generated if not specified.",
 869     }),
 870     model => {
 871         type => 'string',
 872         description => "Network Card Model. The 'virtio' model provides the best performance with very low CPU overhead. If your guest does not support this driver, it is usually best to use 'e1000'.",
 873         enum => $nic_model_list,
 874         default_key => 1,
 875     },
 876     (map { $_ => { keyAlias => 'model', alias => 'macaddr' }} @$nic_model_list),
 877     bridge => {
 878         type => 'string',
 879         description => $net_fmt_bridge_descr,
 880         format_description => 'bridge',
 881         optional => 1,
 882     },
 883     queues => {
 884         type => 'integer',
 885         minimum => 0, maximum => 16,
 886         description => 'Number of packet queues to be used on the device.',
 887         optional => 1,
 888     },
 889     rate => {
 890         type => 'number',
 891         minimum => 0,
 892         description => "Rate limit in mbps (megabytes per second) as floating point number.",
 893         optional => 1,
 894     },
 895     tag => {
 896         type => 'integer',
 897         minimum => 1, maximum => 4094,
 898         description => 'VLAN tag to apply to packets on this interface.',
 899         optional => 1,
 900     },
 901     trunks => {
 902         type => 'string',
 903         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 904         description => 'VLAN trunks to pass through this interface.',
 905         format_description => 'vlanid[;vlanid...]',
 906         optional => 1,
 907     },
 908     firewall => {
 909         type => 'boolean',
 910         description => 'Whether this interface should be protected by the firewall.',
 911         optional => 1,
 912     },
 913     link_down => {
 914         type => 'boolean',
 915         description => 'Whether this interface should be disconnected (like pulling the plug).',
 916         optional => 1,
 917     },
 918 };
 919
 920 my $netdesc = {
 921     optional => 1,
 922     type => 'string', format => $net_fmt,
 923     description => "Specify network devices.",
 924 };
 925
 926 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 927
 928 my $ipconfig_fmt = {
 929     ip => {
 930         type => 'string',
 931         format => 'pve-ipv4-config',
 932         format_description => 'IPv4Format/CIDR',
 933         description => 'IPv4 address in CIDR format.',
 934         optional => 1,
 935         default => 'dhcp',
 936     },
 937     gw => {
 938         type => 'string',
 939         format => 'ipv4',
 940         format_description => 'GatewayIPv4',
 941         description => 'Default gateway for IPv4 traffic.',
 942         optional => 1,
 943         requires => 'ip',
 944     },
 945     ip6 => {
 946         type => 'string',
 947         format => 'pve-ipv6-config',
 948         format_description => 'IPv6Format/CIDR',
 949         description => 'IPv6 address in CIDR format.',
 950         optional => 1,
 951         default => 'dhcp',
 952     },
 953     gw6 => {
 954         type => 'string',
 955         format => 'ipv6',
 956         format_description => 'GatewayIPv6',
 957         description => 'Default gateway for IPv6 traffic.',
 958         optional => 1,
 959         requires => 'ip6',
 960     },
 961 };
 962 PVE::JSONSchema::register_format('pve-qm-ipconfig', $ipconfig_fmt);
 963 my $ipconfigdesc = {
 964     optional => 1,
 965     type => 'string', format => 'pve-qm-ipconfig',
 966     description => <<'EODESCR',
 967 cloud-init: Specify IP addresses and gateways for the corresponding interface.
 968
 969 IP addresses use CIDR notation, gateways are optional but need an IP of the same type specified.
 970
 971 The special string 'dhcp' can be used for IP addresses to use DHCP, in which case no explicit gateway should be provided.
 972 For IPv6 the special string 'auto' can be used to use stateless autoconfiguration.
 973
 974 If cloud-init is enabled and neither an IPv4 nor an IPv6 address is specified, it defaults to using dhcp on IPv4.
 975 EODESCR
 976 };
 977 PVE::JSONSchema::register_standard_option("pve-qm-ipconfig", $netdesc);
 978
 979 for (my $i = 0; $i < $MAX_NETS; $i++)  {
 980     $confdesc->{"net$i"} = $netdesc;
 981     $confdesc_cloudinit->{"ipconfig$i"} = $ipconfigdesc;
 982 }
 983
 984 foreach my $key (keys %$confdesc_cloudinit) {
 985     $confdesc->{$key} = $confdesc_cloudinit->{$key};
 986 }
 987
 988 PVE::JSONSchema::register_format('pve-volume-id-or-qm-path', \&verify_volume_id_or_qm_path);
 989 sub verify_volume_id_or_qm_path {
 990     my ($volid, $noerr) = @_;
 991
 992     if ($volid eq 'none' || $volid eq 'cdrom' || $volid =~ m|^/|) {
 993         return $volid;
 994     }
 995
 996     # if its neither 'none' nor 'cdrom' nor a path, check if its a volume-id
 997     $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') };
 998     if ($@) {
 999         return undef if $noerr;
1000         die $@;
1001     }
1002     return $volid;
1003 }
1004
1005 my $drivename_hash;
1006
1007 my %drivedesc_base = (
1008     volume => { alias => 'file' },
1009     file => {
1010         type => 'string',
1011         format => 'pve-volume-id-or-qm-path',
1012         default_key => 1,
1013         format_description => 'volume',
1014         description => "The drive's backing volume.",
1015     },
1016     media => {
1017         type => 'string',
1018         enum => [qw(cdrom disk)],
1019         description => "The drive's media type.",
1020         default => 'disk',
1021         optional => 1
1022     },
1023     cyls => {
1024         type => 'integer',
1025         description => "Force the drive's physical geometry to have a specific cylinder count.",
1026         optional => 1
1027     },
1028     heads => {
1029         type => 'integer',
1030         description => "Force the drive's physical geometry to have a specific head count.",
1031         optional => 1
1032     },
1033     secs => {
1034         type => 'integer',
1035         description => "Force the drive's physical geometry to have a specific sector count.",
1036         optional => 1
1037     },
1038     trans => {
1039         type => 'string',
1040         enum => [qw(none lba auto)],
1041         description => "Force disk geometry bios translation mode.",
1042         optional => 1,
1043     },
1044     snapshot => {
1045         type => 'boolean',
1046         description => "Controls qemu's snapshot mode feature."
1047             . " If activated, changes made to the disk are temporary and will"
1048             . " be discarded when the VM is shutdown.",
1049         optional => 1,
1050     },
1051     cache => {
1052         type => 'string',
1053         enum => [qw(none writethrough writeback unsafe directsync)],
1054         description => "The drive's cache mode",
1055         optional => 1,
1056     },
1057     format => get_standard_option('pve-qm-image-format'),
1058     size => {
1059         type => 'string',
1060         format => 'disk-size',
1061         format_description => 'DiskSize',
1062         description => "Disk size. This is purely informational and has no effect.",
1063         optional => 1,
1064     },
1065     backup => {
1066         type => 'boolean',
1067         description => "Whether the drive should be included when making backups.",
1068         optional => 1,
1069     },
1070     replicate => {
1071         type => 'boolean',
1072         description => 'Whether the drive should considered for replication jobs.',
1073         optional => 1,
1074         default => 1,
1075     },
1076     rerror => {
1077         type => 'string',
1078         enum => [qw(ignore report stop)],
1079         description => 'Read error action.',
1080         optional => 1,
1081     },
1082     werror => {
1083         type => 'string',
1084         enum => [qw(enospc ignore report stop)],
1085         description => 'Write error action.',
1086         optional => 1,
1087     },
1088     aio => {
1089         type => 'string',
1090         enum => [qw(native threads)],
1091         description => 'AIO type to use.',
1092         optional => 1,
1093     },
1094     discard => {
1095         type => 'string',
1096         enum => [qw(ignore on)],
1097         description => 'Controls whether to pass discard/trim requests to the underlying storage.',
1098         optional => 1,
1099     },
1100     detect_zeroes => {
1101         type => 'boolean',
1102         description => 'Controls whether to detect and try to optimize writes of zeroes.',
1103         optional => 1,
1104     },
1105     serial => {
1106         type => 'string',
1107         format => 'urlencoded',
1108         format_description => 'serial',
1109         maxLength => 20*3, # *3 since it's %xx url enoded
1110         description => "The drive's reported serial number, url-encoded, up to 20 bytes long.",
1111         optional => 1,
1112     },
1113     shared => {
1114         type => 'boolean',
1115         description => 'Mark this locally-managed volume as available on all nodes',
1116         verbose_description => "Mark this locally-managed volume as available on all nodes.\n\nWARNING: This option does not share the volume automatically, it assumes it is shared already!",
1117         optional => 1,
1118         default => 0,
1119     }
1120 );
1121
1122 my %iothread_fmt = ( iothread => {
1123         type => 'boolean',
1124         description => "Whether to use iothreads for this drive",
1125         optional => 1,
1126 });
1127
1128 my %model_fmt = (
1129     model => {
1130         type => 'string',
1131         format => 'urlencoded',
1132         format_description => 'model',
1133         maxLength => 40*3, # *3 since it's %xx url enoded
1134         description => "The drive's reported model name, url-encoded, up to 40 bytes long.",
1135         optional => 1,
1136     },
1137 );
1138
1139 my %queues_fmt = (
1140     queues => {
1141         type => 'integer',
1142         description => "Number of queues.",
1143         minimum => 2,
1144         optional => 1
1145     }
1146 );
1147
1148 my %scsiblock_fmt = (
1149     scsiblock => {
1150         type => 'boolean',
1151         description => "whether to use scsi-block for full passthrough of host block device\n\nWARNING: can lead to I/O errors in combination with low memory or high memory fragmentation on host",
1152         optional => 1,
1153         default => 0,
1154     },
1155 );
1156
1157 my %ssd_fmt = (
1158     ssd => {
1159         type => 'boolean',
1160         description => "Whether to expose this drive as an SSD, rather than a rotational hard disk.",
1161         optional => 1,
1162     },
1163 );
1164
1165 my %wwn_fmt = (
1166     wwn => {
1167         type => 'string',
1168         pattern => qr/^(0x)[0-9a-fA-F]{16}/,
1169         format_description => 'wwn',
1170         description => "The drive's worldwide name, encoded as 16 bytes hex string, prefixed by '0x'.",
1171         optional => 1,
1172     },
1173 );
1174
1175 my $add_throttle_desc = sub {
1176     my ($key, $type, $what, $unit, $longunit, $minimum) = @_;
1177     my $d = {
1178         type => $type,
1179         format_description => $unit,
1180         description => "Maximum $what in $longunit.",
1181         optional => 1,
1182     };
1183     $d->{minimum} = $minimum if defined($minimum);
1184     $drivedesc_base{$key} = $d;
1185 };
1186 # throughput: (leaky bucket)
1187 $add_throttle_desc->('bps',     'integer', 'r/w speed',   'bps',  'bytes per second');
1188 $add_throttle_desc->('bps_rd',  'integer', 'read speed',  'bps',  'bytes per second');
1189 $add_throttle_desc->('bps_wr',  'integer', 'write speed', 'bps',  'bytes per second');
1190 $add_throttle_desc->('mbps',    'number',  'r/w speed',   'mbps', 'megabytes per second');
1191 $add_throttle_desc->('mbps_rd', 'number',  'read speed',  'mbps', 'megabytes per second');
1192 $add_throttle_desc->('mbps_wr', 'number',  'write speed', 'mbps', 'megabytes per second');
1193 $add_throttle_desc->('iops',    'integer', 'r/w I/O',     'iops', 'operations per second');
1194 $add_throttle_desc->('iops_rd', 'integer', 'read I/O',    'iops', 'operations per second');
1195 $add_throttle_desc->('iops_wr', 'integer', 'write I/O',   'iops', 'operations per second');
1196
1197 # pools: (pool of IO before throttling starts taking effect)
1198 $add_throttle_desc->('mbps_max',    'number',  'unthrottled r/w pool',       'mbps', 'megabytes per second');
1199 $add_throttle_desc->('mbps_rd_max', 'number',  'unthrottled read pool',      'mbps', 'megabytes per second');
1200 $add_throttle_desc->('mbps_wr_max', 'number',  'unthrottled write pool',     'mbps', 'megabytes per second');
1201 $add_throttle_desc->('iops_max',    'integer', 'unthrottled r/w I/O pool',   'iops', 'operations per second');
1202 $add_throttle_desc->('iops_rd_max', 'integer', 'unthrottled read I/O pool',  'iops', 'operations per second');
1203 $add_throttle_desc->('iops_wr_max', 'integer', 'unthrottled write I/O pool', 'iops', 'operations per second');
1204
1205 # burst lengths
1206 $add_throttle_desc->('bps_max_length',     'integer', 'length of I/O bursts',       'seconds', 'seconds', 1);
1207 $add_throttle_desc->('bps_rd_max_length',  'integer', 'length of read I/O bursts',  'seconds', 'seconds', 1);
1208 $add_throttle_desc->('bps_wr_max_length',  'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
1209 $add_throttle_desc->('iops_max_length',    'integer', 'length of I/O bursts',       'seconds', 'seconds', 1);
1210 $add_throttle_desc->('iops_rd_max_length', 'integer', 'length of read I/O bursts',  'seconds', 'seconds', 1);
1211 $add_throttle_desc->('iops_wr_max_length', 'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
1212
1213 # legacy support
1214 $drivedesc_base{'bps_rd_length'} = { alias => 'bps_rd_max_length' };
1215 $drivedesc_base{'bps_wr_length'} = { alias => 'bps_wr_max_length' };
1216 $drivedesc_base{'iops_rd_length'} = { alias => 'iops_rd_max_length' };
1217 $drivedesc_base{'iops_wr_length'} = { alias => 'iops_wr_max_length' };
1218
1219 my $ide_fmt = {
1220     %drivedesc_base,
1221     %model_fmt,
1222     %ssd_fmt,
1223     %wwn_fmt,
1224 };
1225 PVE::JSONSchema::register_format("pve-qm-ide", $ide_fmt);
1226
1227 my $idedesc = {
1228     optional => 1,
1229     type => 'string', format => $ide_fmt,
1230     description => "Use volume as IDE hard disk or CD-ROM (n is 0 to " .($MAX_IDE_DISKS -1) . ").",
1231 };
1232 PVE::JSONSchema::register_standard_option("pve-qm-ide", $idedesc);
1233
1234 my $scsi_fmt = {
1235     %drivedesc_base,
1236     %iothread_fmt,
1237     %queues_fmt,
1238     %scsiblock_fmt,
1239     %ssd_fmt,
1240     %wwn_fmt,
1241 };
1242 my $scsidesc = {
1243     optional => 1,
1244     type => 'string', format => $scsi_fmt,
1245     description => "Use volume as SCSI hard disk or CD-ROM (n is 0 to " . ($MAX_SCSI_DISKS - 1) . ").",
1246 };
1247 PVE::JSONSchema::register_standard_option("pve-qm-scsi", $scsidesc);
1248
1249 my $sata_fmt = {
1250     %drivedesc_base,
1251     %ssd_fmt,
1252     %wwn_fmt,
1253 };
1254 my $satadesc = {
1255     optional => 1,
1256     type => 'string', format => $sata_fmt,
1257     description => "Use volume as SATA hard disk or CD-ROM (n is 0 to " . ($MAX_SATA_DISKS - 1). ").",
1258 };
1259 PVE::JSONSchema::register_standard_option("pve-qm-sata", $satadesc);
1260
1261 my $virtio_fmt = {
1262     %drivedesc_base,
1263     %iothread_fmt,
1264 };
1265 my $virtiodesc = {
1266     optional => 1,
1267     type => 'string', format => $virtio_fmt,
1268     description => "Use volume as VIRTIO hard disk (n is 0 to " . ($MAX_VIRTIO_DISKS - 1) . ").",
1269 };
1270 PVE::JSONSchema::register_standard_option("pve-qm-virtio", $virtiodesc);
1271
1272 my $alldrive_fmt = {
1273     %drivedesc_base,
1274     %iothread_fmt,
1275     %model_fmt,
1276     %queues_fmt,
1277     %scsiblock_fmt,
1278     %ssd_fmt,
1279     %wwn_fmt,
1280 };
1281
1282 my $efidisk_fmt = {
1283     volume => { alias => 'file' },
1284     file => {
1285         type => 'string',
1286         format => 'pve-volume-id-or-qm-path',
1287         default_key => 1,
1288         format_description => 'volume',
1289         description => "The drive's backing volume.",
1290     },
1291     format => get_standard_option('pve-qm-image-format'),
1292     size => {
1293         type => 'string',
1294         format => 'disk-size',
1295         format_description => 'DiskSize',
1296         description => "Disk size. This is purely informational and has no effect.",
1297         optional => 1,
1298     },
1299 };
1300
1301 my $efidisk_desc = {
1302     optional => 1,
1303     type => 'string', format => $efidisk_fmt,
1304     description => "Configure a Disk for storing EFI vars",
1305 };
1306
1307 PVE::JSONSchema::register_standard_option("pve-qm-efidisk", $efidisk_desc);
1308
1309 my $usb_fmt = {
1310     host => {
1311         default_key => 1,
1312         type => 'string', format => 'pve-qm-usb-device',
1313         format_description => 'HOSTUSBDEVICE|spice',
1314         description => <<EODESCR,
1315 The Host USB device or port or the value 'spice'. HOSTUSBDEVICE syntax is:
1316
1317  'bus-port(.port)*' (decimal numbers) or
1318  'vendor_id:product_id' (hexadeciaml numbers) or
1319  'spice'
1320
1321 You can use the 'lsusb -t' command to list existing usb devices.
1322
1323 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1324
1325 The value 'spice' can be used to add a usb redirection devices for spice.
1326 EODESCR
1327     },
1328     usb3 => {
1329         optional => 1,
1330         type => 'boolean',
1331         description => "Specifies whether if given host option is a USB3 device or port.",
1332         default => 0,
1333     },
1334 };
1335
1336 my $usbdesc = {
1337     optional => 1,
1338     type => 'string', format => $usb_fmt,
1339     description => "Configure an USB device (n is 0 to 4).",
1340 };
1341 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
1342
1343 my $PCIRE = qr/([a-f0-9]{4}:)?[a-f0-9]{2}:[a-f0-9]{2}(?:\.[a-f0-9])?/;
1344 my $hostpci_fmt = {
1345     host => {
1346         default_key => 1,
1347         type => 'string',
1348         pattern => qr/$PCIRE(;$PCIRE)*/,
1349         format_description => 'HOSTPCIID[;HOSTPCIID2...]',
1350         description => <<EODESCR,
1351 Host PCI device pass through. The PCI ID of a host's PCI device or a list
1352 of PCI virtual functions of the host. HOSTPCIID syntax is:
1353
1354 'bus:dev.func' (hexadecimal numbers)
1355
1356 You can us the 'lspci' command to list existing PCI devices.
1357 EODESCR
1358     },
1359     rombar => {
1360         type => 'boolean',
1361         description =>  "Specify whether or not the device's ROM will be visible in the guest's memory map.",
1362         optional => 1,
1363         default => 1,
1364     },
1365     romfile => {
1366         type => 'string',
1367         pattern => '[^,;]+',
1368         format_description => 'string',
1369         description => "Custom pci device rom filename (must be located in /usr/share/kvm/).",
1370         optional => 1,
1371     },
1372     pcie => {
1373         type => 'boolean',
1374         description =>  "Choose the PCI-express bus (needs the 'q35' machine model).",
1375         optional => 1,
1376         default => 0,
1377     },
1378     'x-vga' => {
1379         type => 'boolean',
1380         description =>  "Enable vfio-vga device support.",
1381         optional => 1,
1382         default => 0,
1383     },
1384     'mdev' => {
1385         type => 'string',
1386         format_description => 'string',
1387         pattern => '[^/\.:]+',
1388         optional => 1,
1389         description => <<EODESCR
1390 The type of mediated device to use.
1391 An instance of this type will be created on startup of the VM and
1392 will be cleaned up when the VM stops.
1393 EODESCR
1394     }
1395 };
1396 PVE::JSONSchema::register_format('pve-qm-hostpci', $hostpci_fmt);
1397
1398 my $hostpcidesc = {
1399         optional => 1,
1400         type => 'string', format => 'pve-qm-hostpci',
1401         description => "Map host PCI devices into guest.",
1402         verbose_description =>  <<EODESCR,
1403 Map host PCI devices into guest.
1404
1405 NOTE: This option allows direct access to host hardware. So it is no longer
1406 possible to migrate such machines - use with special care.
1407
1408 CAUTION: Experimental! User reported problems with this option.
1409 EODESCR
1410 };
1411 PVE::JSONSchema::register_standard_option("pve-qm-hostpci", $hostpcidesc);
1412
1413 my $serialdesc = {
1414         optional => 1,
1415         type => 'string',
1416         pattern => '(/dev/.+|socket)',
1417         description =>  "Create a serial device inside the VM (n is 0 to 3)",
1418         verbose_description =>  <<EODESCR,
1419 Create a serial device inside the VM (n is 0 to 3), and pass through a
1420 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1421 host side (use 'qm terminal' to open a terminal connection).
1422
1423 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines - use with special care.
1424
1425 CAUTION: Experimental! User reported problems with this option.
1426 EODESCR
1427 };
1428
1429 my $paralleldesc= {
1430         optional => 1,
1431         type => 'string',
1432         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
1433         description =>  "Map host parallel devices (n is 0 to 2).",
1434         verbose_description =>  <<EODESCR,
1435 Map host parallel devices (n is 0 to 2).
1436
1437 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1438
1439 CAUTION: Experimental! User reported problems with this option.
1440 EODESCR
1441 };
1442
1443 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
1444     $confdesc->{"parallel$i"} = $paralleldesc;
1445 }
1446
1447 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
1448     $confdesc->{"serial$i"} = $serialdesc;
1449 }
1450
1451 for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
1452     $confdesc->{"hostpci$i"} = $hostpcidesc;
1453 }
1454
1455 for (my $i = 0; $i < $MAX_IDE_DISKS; $i++)  {
1456     $drivename_hash->{"ide$i"} = 1;
1457     $confdesc->{"ide$i"} = $idedesc;
1458 }
1459
1460 for (my $i = 0; $i < $MAX_SATA_DISKS; $i++)  {
1461     $drivename_hash->{"sata$i"} = 1;
1462     $confdesc->{"sata$i"} = $satadesc;
1463 }
1464
1465 for (my $i = 0; $i < $MAX_SCSI_DISKS; $i++)  {
1466     $drivename_hash->{"scsi$i"} = 1;
1467     $confdesc->{"scsi$i"} = $scsidesc ;
1468 }
1469
1470 for (my $i = 0; $i < $MAX_VIRTIO_DISKS; $i++)  {
1471     $drivename_hash->{"virtio$i"} = 1;
1472     $confdesc->{"virtio$i"} = $virtiodesc;
1473 }
1474
1475 $drivename_hash->{efidisk0} = 1;
1476 $confdesc->{efidisk0} = $efidisk_desc;
1477
1478 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
1479     $confdesc->{"usb$i"} = $usbdesc;
1480 }
1481
1482 my $unuseddesc = {
1483     optional => 1,
1484     type => 'string', format => 'pve-volume-id',
1485     description => "Reference to unused volumes. This is used internally, and should not be modified manually.",
1486 };
1487
1488 for (my $i = 0; $i < $MAX_UNUSED_DISKS; $i++)  {
1489     $confdesc->{"unused$i"} = $unuseddesc;
1490 }
1491
1492 my $kvm_api_version = 0;
1493
1494 sub kvm_version {
1495     return $kvm_api_version if $kvm_api_version;
1496
1497     open my $fh, '<', '/dev/kvm'
1498         or return undef;
1499
1500     # 0xae00 => KVM_GET_API_VERSION
1501     $kvm_api_version = ioctl($fh, 0xae00, 0);
1502
1503     return $kvm_api_version;
1504 }
1505
1506 my $kvm_user_version = {};
1507 my $kvm_mtime = {};
1508
1509 sub kvm_user_version {
1510     my ($binary) = @_;
1511
1512     $binary //= get_command_for_arch(get_host_arch()); # get the native arch by default
1513     my $st = stat($binary);
1514
1515     my $cachedmtime = $kvm_mtime->{$binary} // -1;
1516     return $kvm_user_version->{$binary} if $kvm_user_version->{$binary} &&
1517         $cachedmtime == $st->mtime;
1518
1519     $kvm_user_version->{$binary} = 'unknown';
1520     $kvm_mtime->{$binary} = $st->mtime;
1521
1522     my $code = sub {
1523         my $line = shift;
1524         if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1525             $kvm_user_version->{$binary} = $2;
1526         }
1527     };
1528
1529     eval { run_command([$binary, '--version'], outfunc => $code); };
1530     warn $@ if $@;
1531
1532     return $kvm_user_version->{$binary};
1533
1534 }
1535
1536 sub kernel_has_vhost_net {
1537     return -c '/dev/vhost-net';
1538 }
1539
1540 sub valid_drive_names {
1541     # order is important - used to autoselect boot disk
1542     return ((map { "ide$_" } (0 .. ($MAX_IDE_DISKS - 1))),
1543             (map { "scsi$_" } (0 .. ($MAX_SCSI_DISKS - 1))),
1544             (map { "virtio$_" } (0 .. ($MAX_VIRTIO_DISKS - 1))),
1545             (map { "sata$_" } (0 .. ($MAX_SATA_DISKS - 1))),
1546             'efidisk0');
1547 }
1548
1549 sub is_valid_drivename {
1550     my $dev = shift;
1551
1552     return defined($drivename_hash->{$dev});
1553 }
1554
1555 sub option_exists {
1556     my $key = shift;
1557     return defined($confdesc->{$key});
1558 }
1559
1560 my $cdrom_path;
1561 sub get_cdrom_path {
1562
1563     return  $cdrom_path if $cdrom_path;
1564
1565     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
1566     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
1567     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
1568 }
1569
1570 sub get_iso_path {
1571     my ($storecfg, $vmid, $cdrom) = @_;
1572
1573     if ($cdrom eq 'cdrom') {
1574         return get_cdrom_path();
1575     } elsif ($cdrom eq 'none') {
1576         return '';
1577     } elsif ($cdrom =~ m|^/|) {
1578         return $cdrom;
1579     } else {
1580         return PVE::Storage::path($storecfg, $cdrom);
1581     }
1582 }
1583
1584 # try to convert old style file names to volume IDs
1585 sub filename_to_volume_id {
1586     my ($vmid, $file, $media) = @_;
1587
1588      if (!($file eq 'none' || $file eq 'cdrom' ||
1589           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1590
1591         return undef if $file =~ m|/|;
1592
1593         if ($media && $media eq 'cdrom') {
1594             $file = "local:iso/$file";
1595         } else {
1596             $file = "local:$vmid/$file";
1597         }
1598     }
1599
1600     return $file;
1601 }
1602
1603 sub verify_media_type {
1604     my ($opt, $vtype, $media) = @_;
1605
1606     return if !$media;
1607
1608     my $etype;
1609     if ($media eq 'disk') {
1610         $etype = 'images';
1611     } elsif ($media eq 'cdrom') {
1612         $etype = 'iso';
1613     } else {
1614         die "internal error";
1615     }
1616
1617     return if ($vtype eq $etype);
1618
1619     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1620 }
1621
1622 sub cleanup_drive_path {
1623     my ($opt, $storecfg, $drive) = @_;
1624
1625     # try to convert filesystem paths to volume IDs
1626
1627     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1628         ($drive->{file} !~ m|^/dev/.+|) &&
1629         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1630         ($drive->{file} !~ m/^\d+$/)) {
1631         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1632         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"}) if !$vtype;
1633         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1634         verify_media_type($opt, $vtype, $drive->{media});
1635         $drive->{file} = $volid;
1636     }
1637
1638     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1639 }
1640
1641 sub parse_hotplug_features {
1642     my ($data) = @_;
1643
1644     my $res = {};
1645
1646     return $res if $data eq '0';
1647
1648     $data = $confdesc->{hotplug}->{default} if $data eq '1';
1649
1650     foreach my $feature (PVE::Tools::split_list($data)) {
1651         if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
1652             $res->{$1} = 1;
1653         } else {
1654             die "invalid hotplug feature '$feature'\n";
1655         }
1656     }
1657     return $res;
1658 }
1659
1660 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1661 sub pve_verify_hotplug_features {
1662     my ($value, $noerr) = @_;
1663
1664     return $value if parse_hotplug_features($value);
1665
1666     return undef if $noerr;
1667
1668     die "unable to parse hotplug option\n";
1669 }
1670
1671 # ideX = [volume=]volume-id[,media=d][,cyls=c,heads=h,secs=s[,trans=t]]
1672 #        [,snapshot=on|off][,cache=on|off][,format=f][,backup=yes|no]
1673 #        [,rerror=ignore|report|stop][,werror=enospc|ignore|report|stop]
1674 #        [,aio=native|threads][,discard=ignore|on][,detect_zeroes=on|off]
1675 #        [,iothread=on][,serial=serial][,model=model]
1676
1677 sub parse_drive {
1678     my ($key, $data) = @_;
1679
1680     my ($interface, $index);
1681
1682     if ($key =~ m/^([^\d]+)(\d+)$/) {
1683         $interface = $1;
1684         $index = $2;
1685     } else {
1686         return undef;
1687     }
1688
1689     my $desc = $key =~ /^unused\d+$/ ? $alldrive_fmt
1690                                      : $confdesc->{$key}->{format};
1691     if (!$desc) {
1692         warn "invalid drive key: $key\n";
1693         return undef;
1694     }
1695     my $res = eval { PVE::JSONSchema::parse_property_string($desc, $data) };
1696     return undef if !$res;
1697     $res->{interface} = $interface;
1698     $res->{index} = $index;
1699
1700     my $error = 0;
1701     foreach my $opt (qw(bps bps_rd bps_wr)) {
1702         if (my $bps = defined(delete $res->{$opt})) {
1703             if (defined($res->{"m$opt"})) {
1704                 warn "both $opt and m$opt specified\n";
1705                 ++$error;
1706                 next;
1707             }
1708             $res->{"m$opt"} = sprintf("%.3f", $bps / (1024*1024.0));
1709         }
1710     }
1711
1712     # can't use the schema's 'requires' because of the mbps* => bps* "transforming aliases"
1713     for my $requirement (
1714         [mbps_max => 'mbps'],
1715         [mbps_rd_max => 'mbps_rd'],
1716         [mbps_wr_max => 'mbps_wr'],
1717         [miops_max => 'miops'],
1718         [miops_rd_max => 'miops_rd'],
1719         [miops_wr_max => 'miops_wr'],
1720         [bps_max_length => 'mbps_max'],
1721         [bps_rd_max_length => 'mbps_rd_max'],
1722         [bps_wr_max_length => 'mbps_wr_max'],
1723         [iops_max_length => 'iops_max'],
1724         [iops_rd_max_length => 'iops_rd_max'],
1725         [iops_wr_max_length => 'iops_wr_max']) {
1726         my ($option, $requires) = @$requirement;
1727         if ($res->{$option} && !$res->{$requires}) {
1728             warn "$option requires $requires\n";
1729             ++$error;
1730         }
1731     }
1732
1733     return undef if $error;
1734
1735     return undef if $res->{mbps_rd} && $res->{mbps};
1736     return undef if $res->{mbps_wr} && $res->{mbps};
1737     return undef if $res->{iops_rd} && $res->{iops};
1738     return undef if $res->{iops_wr} && $res->{iops};
1739
1740     if ($res->{media} && ($res->{media} eq 'cdrom')) {
1741         return undef if $res->{snapshot} || $res->{trans} || $res->{format};
1742         return undef if $res->{heads} || $res->{secs} || $res->{cyls};
1743         return undef if $res->{interface} eq 'virtio';
1744     }
1745
1746     if (my $size = $res->{size}) {
1747         return undef if !defined($res->{size} = PVE::JSONSchema::parse_size($size));
1748     }
1749
1750     return $res;
1751 }
1752
1753 sub print_drive {
1754     my ($vmid, $drive) = @_;
1755     my $data = { %$drive };
1756     delete $data->{$_} for qw(index interface);
1757     return PVE::JSONSchema::print_property_string($data, $alldrive_fmt);
1758 }
1759
1760 sub scsi_inquiry {
1761     my($fh, $noerr) = @_;
1762
1763     my $SG_IO = 0x2285;
1764     my $SG_GET_VERSION_NUM = 0x2282;
1765
1766     my $versionbuf = "\x00" x 8;
1767     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1768     if (!$ret) {
1769         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1770         return undef;
1771     }
1772     my $version = unpack("I", $versionbuf);
1773     if ($version < 30000) {
1774         die "scsi generic interface too old\n"  if !$noerr;
1775         return undef;
1776     }
1777
1778     my $buf = "\x00" x 36;
1779     my $sensebuf = "\x00" x 8;
1780     my $cmd = pack("C x3 C x1", 0x12, 36);
1781
1782     # see /usr/include/scsi/sg.h
1783     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1784
1785     my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1786                       length($sensebuf), 0, length($buf), $buf,
1787                       $cmd, $sensebuf, 6000);
1788
1789     $ret = ioctl($fh, $SG_IO, $packet);
1790     if (!$ret) {
1791         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1792         return undef;
1793     }
1794
1795     my @res = unpack($sg_io_hdr_t, $packet);
1796     if ($res[17] || $res[18]) {
1797         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1798         return undef;
1799     }
1800
1801     my $res = {};
1802     (my $byte0, my $byte1, $res->{vendor},
1803      $res->{product}, $res->{revision}) = unpack("C C x6 A8 A16 A4", $buf);
1804
1805     $res->{removable} = $byte1 & 128 ? 1 : 0;
1806     $res->{type} = $byte0 & 31;
1807
1808     return $res;
1809 }
1810
1811 sub path_is_scsi {
1812     my ($path) = @_;
1813
1814     my $fh = IO::File->new("+<$path") || return undef;
1815     my $res = scsi_inquiry($fh, 1);
1816     close($fh);
1817
1818     return $res;
1819 }
1820
1821 sub print_tabletdevice_full {
1822     my ($conf, $arch) = @_;
1823
1824     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
1825
1826     # we use uhci for old VMs because tablet driver was buggy in older qemu
1827     my $usbbus;
1828     if (PVE::QemuServer::Machine::machine_type_is_q35($conf) || $arch eq 'aarch64') {
1829         $usbbus = 'ehci';
1830     } else {
1831         $usbbus = 'uhci';
1832     }
1833
1834     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1835 }
1836
1837 sub print_keyboarddevice_full {
1838     my ($conf, $arch, $machine) = @_;
1839
1840     return undef if $arch ne 'aarch64';
1841
1842     return "usb-kbd,id=keyboard,bus=ehci.0,port=2";
1843 }
1844
1845 sub print_drivedevice_full {
1846     my ($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type) = @_;
1847
1848     my $device = '';
1849     my $maxdev = 0;
1850
1851     if ($drive->{interface} eq 'virtio') {
1852         my $pciaddr = print_pci_addr("$drive->{interface}$drive->{index}", $bridges, $arch, $machine_type);
1853         $device = "virtio-blk-pci,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}$pciaddr";
1854         $device .= ",iothread=iothread-$drive->{interface}$drive->{index}" if $drive->{iothread};
1855     } elsif ($drive->{interface} eq 'scsi') {
1856
1857         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1858         my $unit = $drive->{index} % $maxdev;
1859         my $devicetype = 'hd';
1860         my $path = '';
1861         if (drive_is_cdrom($drive)) {
1862             $devicetype = 'cd';
1863         } else {
1864             if ($drive->{file} =~ m|^/|) {
1865                 $path = $drive->{file};
1866                 if (my $info = path_is_scsi($path)) {
1867                     if ($info->{type} == 0 && $drive->{scsiblock}) {
1868                         $devicetype = 'block';
1869                     } elsif ($info->{type} == 1) { # tape
1870                         $devicetype = 'generic';
1871                     }
1872                 }
1873             } else {
1874                  $path = PVE::Storage::path($storecfg, $drive->{file});
1875             }
1876
1877             # for compatibility only, we prefer scsi-hd (#2408, #2355, #2380)
1878             my $version = PVE::QemuServer::Machine::extract_version($machine_type) // kvm_user_version();
1879             if ($path =~ m/^iscsi\:\/\// &&
1880                !min_version($version, 4, 1)) {
1881                 $devicetype = 'generic';
1882             }
1883         }
1884
1885         if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)){
1886             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1887         } else {
1888             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0,lun=$drive->{index},drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1889         }
1890
1891         if ($drive->{ssd} && ($devicetype eq 'block' || $devicetype eq 'hd')) {
1892             $device .= ",rotation_rate=1";
1893         }
1894         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1895
1896     } elsif ($drive->{interface} eq 'ide' || $drive->{interface} eq 'sata') {
1897         my $maxdev = ($drive->{interface} eq 'sata') ? $MAX_SATA_DISKS : 2;
1898         my $controller = int($drive->{index} / $maxdev);
1899         my $unit = $drive->{index} % $maxdev;
1900         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1901
1902         $device = "ide-$devicetype";
1903         if ($drive->{interface} eq 'ide') {
1904             $device .= ",bus=ide.$controller,unit=$unit";
1905         } else {
1906             $device .= ",bus=ahci$controller.$unit";
1907         }
1908         $device .= ",drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1909
1910         if ($devicetype eq 'hd') {
1911             if (my $model = $drive->{model}) {
1912                 $model = URI::Escape::uri_unescape($model);
1913                 $device .= ",model=$model";
1914             }
1915             if ($drive->{ssd}) {
1916                 $device .= ",rotation_rate=1";
1917             }
1918         }
1919         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1920     } elsif ($drive->{interface} eq 'usb') {
1921         die "implement me";
1922         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1923     } else {
1924         die "unsupported interface type";
1925     }
1926
1927     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1928
1929     if (my $serial = $drive->{serial}) {
1930         $serial = URI::Escape::uri_unescape($serial);
1931         $device .= ",serial=$serial";
1932     }
1933
1934
1935     return $device;
1936 }
1937
1938 sub get_initiator_name {
1939     my $initiator;
1940
1941     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return undef;
1942     while (defined(my $line = <$fh>)) {
1943         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1944         $initiator = $1;
1945         last;
1946     }
1947     $fh->close();
1948
1949     return $initiator;
1950 }
1951
1952 sub print_drive_full {
1953     my ($storecfg, $vmid, $drive) = @_;
1954
1955     my $path;
1956     my $volid = $drive->{file};
1957     my $format;
1958
1959     if (drive_is_cdrom($drive)) {
1960         $path = get_iso_path($storecfg, $vmid, $volid);
1961     } else {
1962         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1963         if ($storeid) {
1964             $path = PVE::Storage::path($storecfg, $volid);
1965             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
1966             $format = qemu_img_format($scfg, $volname);
1967         } else {
1968             $path = $volid;
1969             $format = "raw";
1970         }
1971    }
1972
1973     my $opts = '';
1974     my @qemu_drive_options = qw(heads secs cyls trans media format cache rerror werror aio discard);
1975     foreach my $o (@qemu_drive_options) {
1976         $opts .= ",$o=$drive->{$o}" if defined($drive->{$o});
1977     }
1978
1979     # snapshot only accepts on|off
1980     if (defined($drive->{snapshot})) {
1981         my $v = $drive->{snapshot} ? 'on' : 'off';
1982         $opts .= ",snapshot=$v";
1983     }
1984
1985     foreach my $type (['', '-total'], [_rd => '-read'], [_wr => '-write']) {
1986         my ($dir, $qmpname) = @$type;
1987         if (my $v = $drive->{"mbps$dir"}) {
1988             $opts .= ",throttling.bps$qmpname=".int($v*1024*1024);
1989         }
1990         if (my $v = $drive->{"mbps${dir}_max"}) {
1991             $opts .= ",throttling.bps$qmpname-max=".int($v*1024*1024);
1992         }
1993         if (my $v = $drive->{"bps${dir}_max_length"}) {
1994             $opts .= ",throttling.bps$qmpname-max-length=$v";
1995         }
1996         if (my $v = $drive->{"iops${dir}"}) {
1997             $opts .= ",throttling.iops$qmpname=$v";
1998         }
1999         if (my $v = $drive->{"iops${dir}_max"}) {
2000             $opts .= ",throttling.iops$qmpname-max=$v";
2001         }
2002         if (my $v = $drive->{"iops${dir}_max_length"}) {
2003             $opts .= ",throttling.iops$qmpname-max-length=$v";
2004         }
2005     }
2006
2007     $opts .= ",format=$format" if $format && !$drive->{format};
2008
2009     my $cache_direct = 0;
2010
2011     if (my $cache = $drive->{cache}) {
2012         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
2013     } elsif (!drive_is_cdrom($drive)) {
2014         $opts .= ",cache=none";
2015         $cache_direct = 1;
2016     }
2017
2018     # aio native works only with O_DIRECT
2019     if (!$drive->{aio}) {
2020         if($cache_direct) {
2021             $opts .= ",aio=native";
2022         } else {
2023             $opts .= ",aio=threads";
2024         }
2025     }
2026
2027     if (!drive_is_cdrom($drive)) {
2028         my $detectzeroes;
2029         if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
2030             $detectzeroes = 'off';
2031         } elsif ($drive->{discard}) {
2032             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
2033         } else {
2034             # This used to be our default with discard not being specified:
2035             $detectzeroes = 'on';
2036         }
2037         $opts .= ",detect-zeroes=$detectzeroes" if $detectzeroes;
2038     }
2039
2040     my $pathinfo = $path ? "file=$path," : '';
2041
2042     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
2043 }
2044
2045 sub print_netdevice_full {
2046     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files, $arch, $machine_type) = @_;
2047
2048     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
2049
2050     my $device = $net->{model};
2051     if ($net->{model} eq 'virtio') {
2052          $device = 'virtio-net-pci';
2053      };
2054
2055     my $pciaddr = print_pci_addr("$netid", $bridges, $arch, $machine_type);
2056     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
2057     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
2058         #Consider we have N queues, the number of vectors needed is 2*N + 2 (plus one config interrupt and control vq)
2059         my $vectors = $net->{queues} * 2 + 2;
2060         $tmpstr .= ",vectors=$vectors,mq=on";
2061     }
2062     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
2063
2064     if ($use_old_bios_files) {
2065         my $romfile;
2066         if ($device eq 'virtio-net-pci') {
2067             $romfile = 'pxe-virtio.rom';
2068         } elsif ($device eq 'e1000') {
2069             $romfile = 'pxe-e1000.rom';
2070         } elsif ($device eq 'ne2k') {
2071             $romfile = 'pxe-ne2k_pci.rom';
2072         } elsif ($device eq 'pcnet') {
2073             $romfile = 'pxe-pcnet.rom';
2074         } elsif ($device eq 'rtl8139') {
2075             $romfile = 'pxe-rtl8139.rom';
2076         }
2077         $tmpstr .= ",romfile=$romfile" if $romfile;
2078     }
2079
2080     return $tmpstr;
2081 }
2082
2083 sub print_netdev_full {
2084     my ($vmid, $conf, $arch, $net, $netid, $hotplug) = @_;
2085
2086     my $i = '';
2087     if ($netid =~ m/^net(\d+)$/) {
2088         $i = int($1);
2089     }
2090
2091     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
2092
2093     my $ifname = "tap${vmid}i$i";
2094
2095     # kvm uses TUNSETIFF ioctl, and that limits ifname length
2096     die "interface name '$ifname' is too long (max 15 character)\n"
2097         if length($ifname) >= 16;
2098
2099     my $vhostparam = '';
2100     if (is_native($arch)) {
2101         $vhostparam = ',vhost=on' if kernel_has_vhost_net() && $net->{model} eq 'virtio';
2102     }
2103
2104     my $vmname = $conf->{name} || "vm$vmid";
2105
2106     my $netdev = "";
2107     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
2108
2109     if ($net->{bridge}) {
2110         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script,downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
2111     } else {
2112         $netdev = "type=user,id=$netid,hostname=$vmname";
2113     }
2114
2115     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
2116
2117     return $netdev;
2118 }
2119
2120
2121 sub print_cpu_device {
2122     my ($conf, $id) = @_;
2123
2124     my $kvm = $conf->{kvm} // 1;
2125     my $cpu = $kvm ? "kvm64" : "qemu64";
2126     if (my $cputype = $conf->{cpu}) {
2127         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
2128             or die "Cannot parse cpu description: $cputype\n";
2129         $cpu = $cpuconf->{cputype};
2130     }
2131
2132     my $cores = $conf->{cores} || 1;
2133
2134     my $current_core = ($id - 1) % $cores;
2135     my $current_socket = int(($id - 1 - $current_core)/$cores);
2136
2137     return "$cpu-x86_64-cpu,id=cpu$id,socket-id=$current_socket,core-id=$current_core,thread-id=0";
2138 }
2139
2140 my $vga_map = {
2141     'cirrus' => 'cirrus-vga',
2142     'std' => 'VGA',
2143     'vmware' => 'vmware-svga',
2144     'virtio' => 'virtio-vga',
2145 };
2146
2147 sub print_vga_device {
2148     my ($conf, $vga, $arch, $machine_version, $machine, $id, $qxlnum, $bridges) = @_;
2149
2150     my $type = $vga_map->{$vga->{type}};
2151     if ($arch eq 'aarch64' && defined($type) && $type eq 'virtio-vga') {
2152         $type = 'virtio-gpu';
2153     }
2154     my $vgamem_mb = $vga->{memory};
2155
2156     my $max_outputs = '';
2157     if ($qxlnum) {
2158         $type = $id ? 'qxl' : 'qxl-vga';
2159
2160         if (!$conf->{ostype} || $conf->{ostype} =~ m/^(?:l\d\d)|(?:other)$/) {
2161             # set max outputs so linux can have up to 4 qxl displays with one device
2162             if (min_version($machine_version, 4, 1)) {
2163                 $max_outputs = ",max_outputs=4";
2164             }
2165         }
2166     }
2167
2168     die "no devicetype for $vga->{type}\n" if !$type;
2169
2170     my $memory = "";
2171     if ($vgamem_mb) {
2172         if ($vga->{type} eq 'virtio') {
2173             my $bytes = PVE::Tools::convert_size($vgamem_mb, "mb" => "b");
2174             $memory = ",max_hostmem=$bytes";
2175         } elsif ($qxlnum) {
2176             # from https://www.spice-space.org/multiple-monitors.html
2177             $memory = ",vgamem_mb=$vga->{memory}";
2178             my $ram = $vgamem_mb * 4;
2179             my $vram = $vgamem_mb * 2;
2180             $memory .= ",ram_size_mb=$ram,vram_size_mb=$vram";
2181         } else {
2182             $memory = ",vgamem_mb=$vga->{memory}";
2183         }
2184     } elsif ($qxlnum && $id) {
2185         $memory = ",ram_size=67108864,vram_size=33554432";
2186     }
2187
2188     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
2189     my $vgaid = "vga" . ($id // '');
2190     my $pciaddr;
2191
2192     if ($q35 && $vgaid eq 'vga') {
2193         # the first display uses pcie.0 bus on q35 machines
2194         $pciaddr = print_pcie_addr($vgaid, $bridges, $arch, $machine);
2195     } else {
2196         $pciaddr = print_pci_addr($vgaid, $bridges, $arch, $machine);
2197     }
2198
2199     return "$type,id=${vgaid}${memory}${max_outputs}${pciaddr}";
2200 }
2201
2202 sub drive_is_cloudinit {
2203     my ($drive) = @_;
2204     return $drive->{file} =~ m@[:/]vm-\d+-cloudinit(?:\.$QEMU_FORMAT_RE)?$@;
2205 }
2206
2207 sub drive_is_cdrom {
2208     my ($drive, $exclude_cloudinit) = @_;
2209
2210     return 0 if $exclude_cloudinit && drive_is_cloudinit($drive);
2211
2212     return $drive && $drive->{media} && ($drive->{media} eq 'cdrom');
2213
2214 }
2215
2216 sub parse_number_sets {
2217     my ($set) = @_;
2218     my $res = [];
2219     foreach my $part (split(/;/, $set)) {
2220         if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
2221             die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
2222             push @$res, [ $1, $2 ];
2223         } else {
2224             die "invalid range: $part\n";
2225         }
2226     }
2227     return $res;
2228 }
2229
2230 sub parse_numa {
2231     my ($data) = @_;
2232
2233     my $res = PVE::JSONSchema::parse_property_string($numa_fmt, $data);
2234     $res->{cpus} = parse_number_sets($res->{cpus}) if defined($res->{cpus});
2235     $res->{hostnodes} = parse_number_sets($res->{hostnodes}) if defined($res->{hostnodes});
2236     return $res;
2237 }
2238
2239 sub parse_hostpci {
2240     my ($value) = @_;
2241
2242     return undef if !$value;
2243
2244     my $res = PVE::JSONSchema::parse_property_string($hostpci_fmt, $value);
2245
2246     my @idlist = split(/;/, $res->{host});
2247     delete $res->{host};
2248     foreach my $id (@idlist) {
2249         if ($id =~ m/\./) { # full id 00:00.1
2250             push @{$res->{pciid}}, {
2251                 id => $id,
2252             };
2253         } else { # partial id 00:00
2254             $res->{pciid} = PVE::SysFSTools::lspci($id);
2255         }
2256     }
2257     return $res;
2258 }
2259
2260 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
2261 sub parse_net {
2262     my ($data) = @_;
2263
2264     my $res = eval { PVE::JSONSchema::parse_property_string($net_fmt, $data) };
2265     if ($@) {
2266         warn $@;
2267         return undef;
2268     }
2269     if (!defined($res->{macaddr})) {
2270         my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
2271         $res->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix});
2272     }
2273     return $res;
2274 }
2275
2276 # ipconfigX ip=cidr,gw=ip,ip6=cidr,gw6=ip
2277 sub parse_ipconfig {
2278     my ($data) = @_;
2279
2280     my $res = eval { PVE::JSONSchema::parse_property_string($ipconfig_fmt, $data) };
2281     if ($@) {
2282         warn $@;
2283         return undef;
2284     }
2285
2286     if ($res->{gw} && !$res->{ip}) {
2287         warn 'gateway specified without specifying an IP address';
2288         return undef;
2289     }
2290     if ($res->{gw6} && !$res->{ip6}) {
2291         warn 'IPv6 gateway specified without specifying an IPv6 address';
2292         return undef;
2293     }
2294     if ($res->{gw} && $res->{ip} eq 'dhcp') {
2295         warn 'gateway specified together with DHCP';
2296         return undef;
2297     }
2298     if ($res->{gw6} && $res->{ip6} !~ /^$IPV6RE/) {
2299         # gw6 + auto/dhcp
2300         warn "IPv6 gateway specified together with $res->{ip6} address";
2301         return undef;
2302     }
2303
2304     if (!$res->{ip} && !$res->{ip6}) {
2305         return { ip => 'dhcp', ip6 => 'dhcp' };
2306     }
2307
2308     return $res;
2309 }
2310
2311 sub print_net {
2312     my $net = shift;
2313
2314     return PVE::JSONSchema::print_property_string($net, $net_fmt);
2315 }
2316
2317 sub add_random_macs {
2318     my ($settings) = @_;
2319
2320     foreach my $opt (keys %$settings) {
2321         next if $opt !~ m/^net(\d+)$/;
2322         my $net = parse_net($settings->{$opt});
2323         next if !$net;
2324         $settings->{$opt} = print_net($net);
2325     }
2326 }
2327
2328 sub vm_is_volid_owner {
2329     my ($storecfg, $vmid, $volid) = @_;
2330
2331     if ($volid !~  m|^/|) {
2332         my ($path, $owner);
2333         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
2334         if ($owner && ($owner == $vmid)) {
2335             return 1;
2336         }
2337     }
2338
2339     return undef;
2340 }
2341
2342 sub vmconfig_register_unused_drive {
2343     my ($storecfg, $vmid, $conf, $drive) = @_;
2344
2345     if (drive_is_cloudinit($drive)) {
2346         eval { PVE::Storage::vdisk_free($storecfg, $drive->{file}) };
2347         warn $@ if $@;
2348     } elsif (!drive_is_cdrom($drive)) {
2349         my $volid = $drive->{file};
2350         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
2351             PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
2352         }
2353     }
2354 }
2355
2356 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str][,base64=bool]
2357 my $smbios1_fmt = {
2358     uuid => {
2359         type => 'string',
2360         pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
2361         format_description => 'UUID',
2362         description => "Set SMBIOS1 UUID.",
2363         optional => 1,
2364     },
2365     version => {
2366         type => 'string',
2367         pattern => '[A-Za-z0-9+\/]+={0,2}',
2368         format_description => 'Base64 encoded string',
2369         description => "Set SMBIOS1 version.",
2370         optional => 1,
2371     },
2372     serial => {
2373         type => 'string',
2374         pattern => '[A-Za-z0-9+\/]+={0,2}',
2375         format_description => 'Base64 encoded string',
2376         description => "Set SMBIOS1 serial number.",
2377         optional => 1,
2378     },
2379     manufacturer => {
2380         type => 'string',
2381         pattern => '[A-Za-z0-9+\/]+={0,2}',
2382         format_description => 'Base64 encoded string',
2383         description => "Set SMBIOS1 manufacturer.",
2384         optional => 1,
2385     },
2386     product => {
2387         type => 'string',
2388         pattern => '[A-Za-z0-9+\/]+={0,2}',
2389         format_description => 'Base64 encoded string',
2390         description => "Set SMBIOS1 product ID.",
2391         optional => 1,
2392     },
2393     sku => {
2394         type => 'string',
2395         pattern => '[A-Za-z0-9+\/]+={0,2}',
2396         format_description => 'Base64 encoded string',
2397         description => "Set SMBIOS1 SKU string.",
2398         optional => 1,
2399     },
2400     family => {
2401         type => 'string',
2402         pattern => '[A-Za-z0-9+\/]+={0,2}',
2403         format_description => 'Base64 encoded string',
2404         description => "Set SMBIOS1 family string.",
2405         optional => 1,
2406     },
2407     base64 => {
2408         type => 'boolean',
2409         description => 'Flag to indicate that the SMBIOS values are base64 encoded',
2410         optional => 1,
2411     },
2412 };
2413
2414 sub parse_smbios1 {
2415     my ($data) = @_;
2416
2417     my $res = eval { PVE::JSONSchema::parse_property_string($smbios1_fmt, $data) };
2418     warn $@ if $@;
2419     return $res;
2420 }
2421
2422 sub print_smbios1 {
2423     my ($smbios1) = @_;
2424     return PVE::JSONSchema::print_property_string($smbios1, $smbios1_fmt);
2425 }
2426
2427 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_fmt);
2428
2429 PVE::JSONSchema::register_format('pve-qm-bootdisk', \&verify_bootdisk);
2430 sub verify_bootdisk {
2431     my ($value, $noerr) = @_;
2432
2433     return $value if is_valid_drivename($value);
2434
2435     return undef if $noerr;
2436
2437     die "invalid boot disk '$value'\n";
2438 }
2439
2440 sub parse_watchdog {
2441     my ($value) = @_;
2442
2443     return undef if !$value;
2444
2445     my $res = eval { PVE::JSONSchema::parse_property_string($watchdog_fmt, $value) };
2446     warn $@ if $@;
2447     return $res;
2448 }
2449
2450 sub parse_guest_agent {
2451     my ($value) = @_;
2452
2453     return {} if !defined($value->{agent});
2454
2455     my $res = eval { PVE::JSONSchema::parse_property_string($agent_fmt, $value->{agent}) };
2456     warn $@ if $@;
2457
2458     # if the agent is disabled ignore the other potentially set properties
2459     return {} if !$res->{enabled};
2460     return $res;
2461 }
2462
2463 sub parse_vga {
2464     my ($value) = @_;
2465
2466     return {} if !$value;
2467     my $res = eval { PVE::JSONSchema::parse_property_string($vga_fmt, $value) };
2468     warn $@ if $@;
2469     return $res;
2470 }
2471
2472 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
2473 sub verify_usb_device {
2474     my ($value, $noerr) = @_;
2475
2476     return $value if parse_usb_device($value);
2477
2478     return undef if $noerr;
2479
2480     die "unable to parse usb device\n";
2481 }
2482
2483 # add JSON properties for create and set function
2484 sub json_config_properties {
2485     my $prop = shift;
2486
2487     foreach my $opt (keys %$confdesc) {
2488         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate' || $opt eq 'runningmachine';
2489         $prop->{$opt} = $confdesc->{$opt};
2490     }
2491
2492     return $prop;
2493 }
2494
2495 # return copy of $confdesc_cloudinit to generate documentation
2496 sub cloudinit_config_properties {
2497
2498     return dclone($confdesc_cloudinit);
2499 }
2500
2501 sub check_type {
2502     my ($key, $value) = @_;
2503
2504     die "unknown setting '$key'\n" if !$confdesc->{$key};
2505
2506     my $type = $confdesc->{$key}->{type};
2507
2508     if (!defined($value)) {
2509         die "got undefined value\n";
2510     }
2511
2512     if ($value =~ m/[\n\r]/) {
2513         die "property contains a line feed\n";
2514     }
2515
2516     if ($type eq 'boolean') {
2517         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
2518         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
2519         die "type check ('boolean') failed - got '$value'\n";
2520     } elsif ($type eq 'integer') {
2521         return int($1) if $value =~ m/^(\d+)$/;
2522         die "type check ('integer') failed - got '$value'\n";
2523     } elsif ($type eq 'number') {
2524         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
2525         die "type check ('number') failed - got '$value'\n";
2526     } elsif ($type eq 'string') {
2527         if (my $fmt = $confdesc->{$key}->{format}) {
2528             PVE::JSONSchema::check_format($fmt, $value);
2529             return $value;
2530         }
2531         $value =~ s/^\"(.*)\"$/$1/;
2532         return $value;
2533     } else {
2534         die "internal error"
2535     }
2536 }
2537
2538 sub destroy_vm {
2539     my ($storecfg, $vmid, $skiplock, $replacement_conf) = @_;
2540
2541     my $conf = PVE::QemuConfig->load_config($vmid);
2542
2543     PVE::QemuConfig->check_lock($conf) if !$skiplock;
2544
2545     if ($conf->{template}) {
2546         # check if any base image is still used by a linked clone
2547         foreach_drive($conf, sub {
2548                 my ($ds, $drive) = @_;
2549                 return if drive_is_cdrom($drive);
2550
2551                 my $volid = $drive->{file};
2552                 return if !$volid || $volid =~ m|^/|;
2553
2554                 die "base volume '$volid' is still in use by linked cloned\n"
2555                     if PVE::Storage::volume_is_base_and_used($storecfg, $volid);
2556
2557         });
2558     }
2559
2560     # only remove disks owned by this VM
2561     foreach_drive($conf, sub {
2562         my ($ds, $drive) = @_;
2563         return if drive_is_cdrom($drive, 1);
2564
2565         my $volid = $drive->{file};
2566         return if !$volid || $volid =~ m|^/|;
2567
2568         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
2569         return if !$path || !$owner || ($owner != $vmid);
2570
2571         eval { PVE::Storage::vdisk_free($storecfg, $volid) };
2572         warn "Could not remove disk '$volid', check manually: $@" if $@;
2573     });
2574
2575     # also remove unused disk
2576     my $vmdisks = PVE::Storage::vdisk_list($storecfg, undef, $vmid);
2577     PVE::Storage::foreach_volid($vmdisks, sub {
2578         my ($volid, $sid, $volname, $d) = @_;
2579         eval { PVE::Storage::vdisk_free($storecfg, $volid) };
2580         warn $@ if $@;
2581     });
2582
2583     if (defined $replacement_conf) {
2584         PVE::QemuConfig->write_config($vmid, $replacement_conf);
2585     } else {
2586         PVE::QemuConfig->destroy_config($vmid);
2587     }
2588 }
2589
2590 sub parse_vm_config {
2591     my ($filename, $raw) = @_;
2592
2593     return undef if !defined($raw);
2594
2595     my $res = {
2596         digest => Digest::SHA::sha1_hex($raw),
2597         snapshots => {},
2598         pending => {},
2599     };
2600
2601     $filename =~ m|/qemu-server/(\d+)\.conf$|
2602         || die "got strange filename '$filename'";
2603
2604     my $vmid = $1;
2605
2606     my $conf = $res;
2607     my $descr;
2608     my $section = '';
2609
2610     my @lines = split(/\n/, $raw);
2611     foreach my $line (@lines) {
2612         next if $line =~ m/^\s*$/;
2613
2614         if ($line =~ m/^\[PENDING\]\s*$/i) {
2615             $section = 'pending';
2616             if (defined($descr)) {
2617                 $descr =~ s/\s+$//;
2618                 $conf->{description} = $descr;
2619             }
2620             $descr = undef;
2621             $conf = $res->{$section} = {};
2622             next;
2623
2624         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2625             $section = $1;
2626             if (defined($descr)) {
2627                 $descr =~ s/\s+$//;
2628                 $conf->{description} = $descr;
2629             }
2630             $descr = undef;
2631             $conf = $res->{snapshots}->{$section} = {};
2632             next;
2633         }
2634
2635         if ($line =~ m/^\#(.*)\s*$/) {
2636             $descr = '' if !defined($descr);
2637             $descr .= PVE::Tools::decode_text($1) . "\n";
2638             next;
2639         }
2640
2641         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2642             $descr = '' if !defined($descr);
2643             $descr .= PVE::Tools::decode_text($2);
2644         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2645             $conf->{snapstate} = $1;
2646         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2647             my $key = $1;
2648             my $value = $2;
2649             $conf->{$key} = $value;
2650         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2651             my $value = $1;
2652             if ($section eq 'pending') {
2653                 $conf->{delete} = $value; # we parse this later
2654             } else {
2655                 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
2656             }
2657         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(.+?)\s*$/) {
2658             my $key = $1;
2659             my $value = $2;
2660             eval { $value = check_type($key, $value); };
2661             if ($@) {
2662                 warn "vm $vmid - unable to parse value of '$key' - $@";
2663             } else {
2664                 $key = 'ide2' if $key eq 'cdrom';
2665                 my $fmt = $confdesc->{$key}->{format};
2666                 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2667                     my $v = parse_drive($key, $value);
2668                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2669                         $v->{file} = $volid;
2670                         $value = print_drive($vmid, $v);
2671                     } else {
2672                         warn "vm $vmid - unable to parse value of '$key'\n";
2673                         next;
2674                     }
2675                 }
2676
2677                 $conf->{$key} = $value;
2678             }
2679         }
2680     }
2681
2682     if (defined($descr)) {
2683         $descr =~ s/\s+$//;
2684         $conf->{description} = $descr;
2685     }
2686     delete $res->{snapstate}; # just to be sure
2687
2688     return $res;
2689 }
2690
2691 sub write_vm_config {
2692     my ($filename, $conf) = @_;
2693
2694     delete $conf->{snapstate}; # just to be sure
2695
2696     if ($conf->{cdrom}) {
2697         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2698         $conf->{ide2} = $conf->{cdrom};
2699         delete $conf->{cdrom};
2700     }
2701
2702     # we do not use 'smp' any longer
2703     if ($conf->{sockets}) {
2704         delete $conf->{smp};
2705     } elsif ($conf->{smp}) {
2706         $conf->{sockets} = $conf->{smp};
2707         delete $conf->{cores};
2708         delete $conf->{smp};
2709     }
2710
2711     my $used_volids = {};
2712
2713     my $cleanup_config = sub {
2714         my ($cref, $pending, $snapname) = @_;
2715
2716         foreach my $key (keys %$cref) {
2717             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2718                 $key eq 'snapstate' || $key eq 'pending';
2719             my $value = $cref->{$key};
2720             if ($key eq 'delete') {
2721                 die "propertry 'delete' is only allowed in [PENDING]\n"
2722                     if !$pending;
2723                 # fixme: check syntax?
2724                 next;
2725             }
2726             eval { $value = check_type($key, $value); };
2727             die "unable to parse value of '$key' - $@" if $@;
2728
2729             $cref->{$key} = $value;
2730
2731             if (!$snapname && is_valid_drivename($key)) {
2732                 my $drive = parse_drive($key, $value);
2733                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2734             }
2735         }
2736     };
2737
2738     &$cleanup_config($conf);
2739
2740     &$cleanup_config($conf->{pending}, 1);
2741
2742     foreach my $snapname (keys %{$conf->{snapshots}}) {
2743         die "internal error" if $snapname eq 'pending';
2744         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2745     }
2746
2747     # remove 'unusedX' settings if we re-add a volume
2748     foreach my $key (keys %$conf) {
2749         my $value = $conf->{$key};
2750         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2751             delete $conf->{$key};
2752         }
2753     }
2754
2755     my $generate_raw_config = sub {
2756         my ($conf, $pending) = @_;
2757
2758         my $raw = '';
2759
2760         # add description as comment to top of file
2761         if (defined(my $descr = $conf->{description})) {
2762             if ($descr) {
2763                 foreach my $cl (split(/\n/, $descr)) {
2764                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2765                 }
2766             } else {
2767                 $raw .= "#\n" if $pending;
2768             }
2769         }
2770
2771         foreach my $key (sort keys %$conf) {
2772             next if $key eq 'digest' || $key eq 'description' || $key eq 'pending' || $key eq 'snapshots';
2773             $raw .= "$key: $conf->{$key}\n";
2774         }
2775         return $raw;
2776     };
2777
2778     my $raw = &$generate_raw_config($conf);
2779
2780     if (scalar(keys %{$conf->{pending}})){
2781         $raw .= "\n[PENDING]\n";
2782         $raw .= &$generate_raw_config($conf->{pending}, 1);
2783     }
2784
2785     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2786         $raw .= "\n[$snapname]\n";
2787         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2788     }
2789
2790     return $raw;
2791 }
2792
2793 sub load_defaults {
2794
2795     my $res = {};
2796
2797     # we use static defaults from our JSON schema configuration
2798     foreach my $key (keys %$confdesc) {
2799         if (defined(my $default = $confdesc->{$key}->{default})) {
2800             $res->{$key} = $default;
2801         }
2802     }
2803
2804     return $res;
2805 }
2806
2807 sub config_list {
2808     my $vmlist = PVE::Cluster::get_vmlist();
2809     my $res = {};
2810     return $res if !$vmlist || !$vmlist->{ids};
2811     my $ids = $vmlist->{ids};
2812
2813     foreach my $vmid (keys %$ids) {
2814         my $d = $ids->{$vmid};
2815         next if !$d->{node} || $d->{node} ne $nodename;
2816         next if !$d->{type} || $d->{type} ne 'qemu';
2817         $res->{$vmid}->{exists} = 1;
2818     }
2819     return $res;
2820 }
2821
2822 # test if VM uses local resources (to prevent migration)
2823 sub check_local_resources {
2824     my ($conf, $noerr) = @_;
2825
2826     my @loc_res = ();
2827
2828     push @loc_res, "hostusb" if $conf->{hostusb}; # old syntax
2829     push @loc_res, "hostpci" if $conf->{hostpci}; # old syntax
2830
2831     push @loc_res, "ivshmem" if $conf->{ivshmem};
2832
2833     foreach my $k (keys %$conf) {
2834         next if $k =~ m/^usb/ && ($conf->{$k} =~ m/^spice(?![^,])/);
2835         # sockets are safe: they will recreated be on the target side post-migrate
2836         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2837         push @loc_res, $k if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2838     }
2839
2840     die "VM uses local resources\n" if scalar @loc_res && !$noerr;
2841
2842     return \@loc_res;
2843 }
2844
2845 # check if used storages are available on all nodes (use by migrate)
2846 sub check_storage_availability {
2847     my ($storecfg, $conf, $node) = @_;
2848
2849     foreach_drive($conf, sub {
2850         my ($ds, $drive) = @_;
2851
2852         my $volid = $drive->{file};
2853         return if !$volid;
2854
2855         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2856         return if !$sid;
2857
2858         # check if storage is available on both nodes
2859         my $scfg = PVE::Storage::storage_check_node($storecfg, $sid);
2860         PVE::Storage::storage_check_node($storecfg, $sid, $node);
2861    });
2862 }
2863
2864 # list nodes where all VM images are available (used by has_feature API)
2865 sub shared_nodes {
2866     my ($conf, $storecfg) = @_;
2867
2868     my $nodelist = PVE::Cluster::get_nodelist();
2869     my $nodehash = { map { $_ => 1 } @$nodelist };
2870     my $nodename = PVE::INotify::nodename();
2871
2872     foreach_drive($conf, sub {
2873         my ($ds, $drive) = @_;
2874
2875         my $volid = $drive->{file};
2876         return if !$volid;
2877
2878         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2879         if ($storeid) {
2880             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2881             if ($scfg->{disable}) {
2882                 $nodehash = {};
2883             } elsif (my $avail = $scfg->{nodes}) {
2884                 foreach my $node (keys %$nodehash) {
2885                     delete $nodehash->{$node} if !$avail->{$node};
2886                 }
2887             } elsif (!$scfg->{shared}) {
2888                 foreach my $node (keys %$nodehash) {
2889                     delete $nodehash->{$node} if $node ne $nodename
2890                 }
2891             }
2892         }
2893     });
2894
2895     return $nodehash
2896 }
2897
2898 sub check_local_storage_availability {
2899     my ($conf, $storecfg) = @_;
2900
2901     my $nodelist = PVE::Cluster::get_nodelist();
2902     my $nodehash = { map { $_ => {} } @$nodelist };
2903
2904     foreach_drive($conf, sub {
2905         my ($ds, $drive) = @_;
2906
2907         my $volid = $drive->{file};
2908         return if !$volid;
2909
2910         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2911         if ($storeid) {
2912             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2913
2914             if ($scfg->{disable}) {
2915                 foreach my $node (keys %$nodehash) {
2916                     $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2917                 }
2918             } elsif (my $avail = $scfg->{nodes}) {
2919                 foreach my $node (keys %$nodehash) {
2920                     if (!$avail->{$node}) {
2921                         $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2922                     }
2923                 }
2924             }
2925         }
2926     });
2927
2928     foreach my $node (values %$nodehash) {
2929         if (my $unavail = $node->{unavailable_storages}) {
2930             $node->{unavailable_storages} = [ sort keys %$unavail ];
2931         }
2932     }
2933
2934     return $nodehash
2935 }
2936
2937 # Compat only, use assert_config_exists_on_node and vm_running_locally where possible
2938 sub check_running {
2939     my ($vmid, $nocheck, $node) = @_;
2940
2941     PVE::QemuConfig::assert_config_exists_on_node($vmid, $node) if !$nocheck;
2942     return PVE::QemuServer::Helpers::vm_running_locally($vmid);
2943 }
2944
2945 sub vzlist {
2946
2947     my $vzlist = config_list();
2948
2949     my $fd = IO::Dir->new($PVE::QemuServer::Helpers::var_run_tmpdir) || return $vzlist;
2950
2951     while (defined(my $de = $fd->read)) {
2952         next if $de !~ m/^(\d+)\.pid$/;
2953         my $vmid = $1;
2954         next if !defined($vzlist->{$vmid});
2955         if (my $pid = check_running($vmid)) {
2956             $vzlist->{$vmid}->{pid} = $pid;
2957         }
2958     }
2959
2960     return $vzlist;
2961 }
2962
2963 sub disksize {
2964     my ($storecfg, $conf) = @_;
2965
2966     my $bootdisk = $conf->{bootdisk};
2967     return undef if !$bootdisk;
2968     return undef if !is_valid_drivename($bootdisk);
2969
2970     return undef if !$conf->{$bootdisk};
2971
2972     my $drive = parse_drive($bootdisk, $conf->{$bootdisk});
2973     return undef if !defined($drive);
2974
2975     return undef if drive_is_cdrom($drive);
2976
2977     my $volid = $drive->{file};
2978     return undef if !$volid;
2979
2980     return $drive->{size};
2981 }
2982
2983 our $vmstatus_return_properties = {
2984     vmid => get_standard_option('pve-vmid'),
2985     status => {
2986         description => "Qemu process status.",
2987         type => 'string',
2988         enum => ['stopped', 'running'],
2989     },
2990     maxmem => {
2991         description => "Maximum memory in bytes.",
2992         type => 'integer',
2993         optional => 1,
2994         renderer => 'bytes',
2995     },
2996     maxdisk => {
2997         description => "Root disk size in bytes.",
2998         type => 'integer',
2999         optional => 1,
3000         renderer => 'bytes',
3001     },
3002     name => {
3003         description => "VM name.",
3004         type => 'string',
3005         optional => 1,
3006     },
3007     qmpstatus => {
3008         description => "Qemu QMP agent status.",
3009         type => 'string',
3010         optional => 1,
3011     },
3012     pid => {
3013         description => "PID of running qemu process.",
3014         type => 'integer',
3015         optional => 1,
3016     },
3017     uptime => {
3018         description => "Uptime.",
3019         type => 'integer',
3020         optional => 1,
3021         renderer => 'duration',
3022     },
3023     cpus => {
3024         description => "Maximum usable CPUs.",
3025         type => 'number',
3026         optional => 1,
3027     },
3028     lock => {
3029         description => "The current config lock, if any.",
3030         type => 'string',
3031         optional => 1,
3032     },
3033     tags => {
3034         description  => "The current configured tags, if any",
3035         type => 'string',
3036         optional => 1,
3037     },
3038 };
3039
3040 my $last_proc_pid_stat;
3041
3042 # get VM status information
3043 # This must be fast and should not block ($full == false)
3044 # We only query KVM using QMP if $full == true (this can be slow)
3045 sub vmstatus {
3046     my ($opt_vmid, $full) = @_;
3047
3048     my $res = {};
3049
3050     my $storecfg = PVE::Storage::config();
3051
3052     my $list = vzlist();
3053     my $defaults = load_defaults();
3054
3055     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
3056
3057     my $cpucount = $cpuinfo->{cpus} || 1;
3058
3059     foreach my $vmid (keys %$list) {
3060         next if $opt_vmid && ($vmid ne $opt_vmid);
3061
3062         my $conf = PVE::QemuConfig->load_config($vmid);
3063
3064         my $d = { vmid => $vmid };
3065         $d->{pid} = $list->{$vmid}->{pid};
3066
3067         # fixme: better status?
3068         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
3069
3070         my $size = disksize($storecfg, $conf);
3071         if (defined($size)) {
3072             $d->{disk} = 0; # no info available
3073             $d->{maxdisk} = $size;
3074         } else {
3075             $d->{disk} = 0;
3076             $d->{maxdisk} = 0;
3077         }
3078
3079         $d->{cpus} = ($conf->{sockets} || $defaults->{sockets})
3080             * ($conf->{cores} || $defaults->{cores});
3081         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
3082         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
3083
3084         $d->{name} = $conf->{name} || "VM $vmid";
3085         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024)
3086             : $defaults->{memory}*(1024*1024);
3087
3088         if ($conf->{balloon}) {
3089             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
3090             $d->{shares} = defined($conf->{shares}) ? $conf->{shares}
3091                 : $defaults->{shares};
3092         }
3093
3094         $d->{uptime} = 0;
3095         $d->{cpu} = 0;
3096         $d->{mem} = 0;
3097
3098         $d->{netout} = 0;
3099         $d->{netin} = 0;
3100
3101         $d->{diskread} = 0;
3102         $d->{diskwrite} = 0;
3103
3104         $d->{template} = PVE::QemuConfig->is_template($conf);
3105
3106         $d->{serial} = 1 if conf_has_serial($conf);
3107         $d->{lock} = $conf->{lock} if $conf->{lock};
3108         $d->{tags} = $conf->{tags} if defined($conf->{tags});
3109
3110         $res->{$vmid} = $d;
3111     }
3112
3113     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
3114     foreach my $dev (keys %$netdev) {
3115         next if $dev !~ m/^tap([1-9]\d*)i/;
3116         my $vmid = $1;
3117         my $d = $res->{$vmid};
3118         next if !$d;
3119
3120         $d->{netout} += $netdev->{$dev}->{receive};
3121         $d->{netin} += $netdev->{$dev}->{transmit};
3122
3123         if ($full) {
3124             $d->{nics}->{$dev}->{netout} = $netdev->{$dev}->{receive};
3125             $d->{nics}->{$dev}->{netin} = $netdev->{$dev}->{transmit};
3126         }
3127
3128     }
3129
3130     my $ctime = gettimeofday;
3131
3132     foreach my $vmid (keys %$list) {
3133
3134         my $d = $res->{$vmid};
3135         my $pid = $d->{pid};
3136         next if !$pid;
3137
3138         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
3139         next if !$pstat; # not running
3140
3141         my $used = $pstat->{utime} + $pstat->{stime};
3142
3143         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
3144
3145         if ($pstat->{vsize}) {
3146             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
3147         }
3148
3149         my $old = $last_proc_pid_stat->{$pid};
3150         if (!$old) {
3151             $last_proc_pid_stat->{$pid} = {
3152                 time => $ctime,
3153                 used => $used,
3154                 cpu => 0,
3155             };
3156             next;
3157         }
3158
3159         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
3160
3161         if ($dtime > 1000) {
3162             my $dutime = $used -  $old->{used};
3163
3164             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
3165             $last_proc_pid_stat->{$pid} = {
3166                 time => $ctime,
3167                 used => $used,
3168                 cpu => $d->{cpu},
3169             };
3170         } else {
3171             $d->{cpu} = $old->{cpu};
3172         }
3173     }
3174
3175     return $res if !$full;
3176
3177     my $qmpclient = PVE::QMPClient->new();
3178
3179     my $ballooncb = sub {
3180         my ($vmid, $resp) = @_;
3181
3182         my $info = $resp->{'return'};
3183         return if !$info->{max_mem};
3184
3185         my $d = $res->{$vmid};
3186
3187         # use memory assigned to VM
3188         $d->{maxmem} = $info->{max_mem};
3189         $d->{balloon} = $info->{actual};
3190
3191         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
3192             $d->{mem} = $info->{total_mem} - $info->{free_mem};
3193             $d->{freemem} = $info->{free_mem};
3194         }
3195
3196         $d->{ballooninfo} = $info;
3197     };
3198
3199     my $blockstatscb = sub {
3200         my ($vmid, $resp) = @_;
3201         my $data = $resp->{'return'} || [];
3202         my $totalrdbytes = 0;
3203         my $totalwrbytes = 0;
3204
3205         for my $blockstat (@$data) {
3206             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
3207             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
3208
3209             $blockstat->{device} =~ s/drive-//;
3210             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
3211         }
3212         $res->{$vmid}->{diskread} = $totalrdbytes;
3213         $res->{$vmid}->{diskwrite} = $totalwrbytes;
3214     };
3215
3216     my $statuscb = sub {
3217         my ($vmid, $resp) = @_;
3218
3219         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
3220         # this fails if ballon driver is not loaded, so this must be
3221         # the last commnand (following command are aborted if this fails).
3222         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
3223
3224         my $status = 'unknown';
3225         if (!defined($status = $resp->{'return'}->{status})) {
3226             warn "unable to get VM status\n";
3227             return;
3228         }
3229
3230         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
3231     };
3232
3233     foreach my $vmid (keys %$list) {
3234         next if $opt_vmid && ($vmid ne $opt_vmid);
3235         next if !$res->{$vmid}->{pid}; # not running
3236         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
3237     }
3238
3239     $qmpclient->queue_execute(undef, 2);
3240
3241     foreach my $vmid (keys %$list) {
3242         next if $opt_vmid && ($vmid ne $opt_vmid);
3243         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
3244     }
3245
3246     return $res;
3247 }
3248
3249 sub foreach_drive {
3250     my ($conf, $func, @param) = @_;
3251
3252     foreach my $ds (valid_drive_names()) {
3253         next if !defined($conf->{$ds});
3254
3255         my $drive = parse_drive($ds, $conf->{$ds});
3256         next if !$drive;
3257
3258         &$func($ds, $drive, @param);
3259     }
3260 }
3261
3262 sub foreach_volid {
3263     my ($conf, $func, @param) = @_;
3264
3265     my $volhash = {};
3266
3267     my $test_volid = sub {
3268         my ($volid, $is_cdrom, $replicate, $shared, $snapname, $size) = @_;
3269
3270         return if !$volid;
3271
3272         $volhash->{$volid}->{cdrom} //= 1;
3273         $volhash->{$volid}->{cdrom} = 0 if !$is_cdrom;
3274
3275         $volhash->{$volid}->{replicate} //= 0;
3276         $volhash->{$volid}->{replicate} = 1 if $replicate;
3277
3278         $volhash->{$volid}->{shared} //= 0;
3279         $volhash->{$volid}->{shared} = 1 if $shared;
3280
3281         $volhash->{$volid}->{referenced_in_config} //= 0;
3282         $volhash->{$volid}->{referenced_in_config} = 1 if !defined($snapname);
3283
3284         $volhash->{$volid}->{referenced_in_snapshot}->{$snapname} = 1
3285             if defined($snapname);
3286         $volhash->{$volid}->{size} = $size if $size;
3287     };
3288
3289     foreach_drive($conf, sub {
3290         my ($ds, $drive) = @_;
3291         $test_volid->($drive->{file}, drive_is_cdrom($drive), $drive->{replicate} // 1, $drive->{shared}, undef, $drive->{size});
3292     });
3293
3294     foreach my $snapname (keys %{$conf->{snapshots}}) {
3295         my $snap = $conf->{snapshots}->{$snapname};
3296         $test_volid->($snap->{vmstate}, 0, 1, $snapname);
3297         foreach_drive($snap, sub {
3298             my ($ds, $drive) = @_;
3299             $test_volid->($drive->{file}, drive_is_cdrom($drive), $drive->{replicate} // 1, $drive->{shared}, $snapname);
3300         });
3301     }
3302
3303     foreach my $volid (keys %$volhash) {
3304         &$func($volid, $volhash->{$volid}, @param);
3305     }
3306 }
3307
3308 sub conf_has_serial {
3309     my ($conf) = @_;
3310
3311     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3312         if ($conf->{"serial$i"}) {
3313             return 1;
3314         }
3315     }
3316
3317     return 0;
3318 }
3319
3320 sub conf_has_audio {
3321     my ($conf, $id) = @_;
3322
3323     $id //= 0;
3324     my $audio = $conf->{"audio$id"};
3325     return undef if !defined($audio);
3326
3327     my $audioproperties = PVE::JSONSchema::parse_property_string($audio_fmt, $audio);
3328     my $audiodriver = $audioproperties->{driver} // 'spice';
3329
3330     return {
3331         dev => $audioproperties->{device},
3332         dev_id => "audiodev$id",
3333         backend => $audiodriver,
3334         backend_id => "$audiodriver-backend${id}",
3335     };
3336 }
3337
3338 sub vga_conf_has_spice {
3339     my ($vga) = @_;
3340
3341     my $vgaconf = parse_vga($vga);
3342     my $vgatype = $vgaconf->{type};
3343     return 0 if !$vgatype || $vgatype !~ m/^qxl([234])?$/;
3344
3345     return $1 || 1;
3346 }
3347
3348 sub is_native($) {
3349     my ($arch) = @_;
3350     return get_host_arch() eq $arch;
3351 }
3352
3353 my $default_machines = {
3354     x86_64 => 'pc',
3355     aarch64 => 'virt',
3356 };
3357
3358 sub get_basic_machine_info {
3359     my ($conf, $forcemachine) = @_;
3360
3361     my $arch = $conf->{arch} // get_host_arch();
3362     my $machine = $forcemachine || $conf->{machine} || $default_machines->{$arch};
3363     return ($arch, $machine);
3364 }
3365
3366 sub get_ovmf_files($) {
3367     my ($arch) = @_;
3368
3369     my $ovmf = $OVMF->{$arch}
3370         or die "no OVMF images known for architecture '$arch'\n";
3371
3372     return @$ovmf;
3373 }
3374
3375 my $Arch2Qemu = {
3376     aarch64 => '/usr/bin/qemu-system-aarch64',
3377     x86_64 => '/usr/bin/qemu-system-x86_64',
3378 };
3379 sub get_command_for_arch($) {
3380     my ($arch) = @_;
3381     return '/usr/bin/kvm' if is_native($arch);
3382
3383     my $cmd = $Arch2Qemu->{$arch}
3384         or die "don't know how to emulate architecture '$arch'\n";
3385     return $cmd;
3386 }
3387
3388 sub get_cpu_options {
3389     my ($conf, $arch, $kvm, $kvm_off, $machine_version, $winversion, $gpu_passthrough) = @_;
3390
3391     my $cpuFlags = [];
3392     my $ostype = $conf->{ostype};
3393
3394     my $cpu = $kvm ? "kvm64" : "qemu64";
3395     if ($arch eq 'aarch64') {
3396         $cpu = 'cortex-a57';
3397     }
3398     my $hv_vendor_id;
3399     if (my $cputype = $conf->{cpu}) {
3400         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
3401             or die "Cannot parse cpu description: $cputype\n";
3402         $cpu = $cpuconf->{cputype};
3403         $kvm_off = 1 if $cpuconf->{hidden};
3404         $hv_vendor_id = $cpuconf->{'hv-vendor-id'};
3405
3406         if (defined(my $flags = $cpuconf->{flags})) {
3407             push @$cpuFlags, split(";", $flags);
3408         }
3409     }
3410
3411     push @$cpuFlags , '+lahf_lm' if $cpu eq 'kvm64' && $arch eq 'x86_64';
3412
3413     push @$cpuFlags , '-x2apic' if $ostype && $ostype eq 'solaris';
3414
3415     push @$cpuFlags, '+sep' if $cpu eq 'kvm64' || $cpu eq 'kvm32';
3416
3417     push @$cpuFlags, '-rdtscp' if $cpu =~ m/^Opteron/;
3418
3419     if (min_version($machine_version, 2, 3) && $arch eq 'x86_64') {
3420
3421         push @$cpuFlags , '+kvm_pv_unhalt' if $kvm;
3422         push @$cpuFlags , '+kvm_pv_eoi' if $kvm;
3423     }
3424
3425     add_hyperv_enlightenments($cpuFlags, $winversion, $machine_version, $conf->{bios}, $gpu_passthrough, $hv_vendor_id) if $kvm;
3426
3427     push @$cpuFlags, 'enforce' if $cpu ne 'host' && $kvm && $arch eq 'x86_64';
3428
3429     push @$cpuFlags, 'kvm=off' if $kvm_off;
3430
3431     if (my $cpu_vendor = $cpu_vendor_list->{$cpu}) {
3432         push @$cpuFlags, "vendor=${cpu_vendor}"
3433             if $cpu_vendor ne 'default';
3434     } elsif ($arch ne 'aarch64') {
3435         die "internal error"; # should not happen
3436     }
3437
3438     $cpu .= "," . join(',', @$cpuFlags) if scalar(@$cpuFlags);
3439
3440     return ('-cpu', $cpu);
3441 }
3442
3443 sub config_to_command {
3444     my ($storecfg, $vmid, $conf, $defaults, $forcemachine) = @_;
3445
3446     my $cmd = [];
3447     my $globalFlags = [];
3448     my $machineFlags = [];
3449     my $rtcFlags = [];
3450     my $devices = [];
3451     my $pciaddr = '';
3452     my $bridges = {};
3453     my $vernum = 0; # unknown
3454     my $ostype = $conf->{ostype};
3455     my $winversion = windows_version($ostype);
3456     my $kvm = $conf->{kvm};
3457
3458     my ($arch, $machine_type) = get_basic_machine_info($conf, $forcemachine);
3459     my $kvm_binary = get_command_for_arch($arch);
3460     my $kvmver = kvm_user_version($kvm_binary);
3461     my $machine_version = PVE::QemuServer::Machine::extract_version($machine_type) // $kvmver;
3462     $kvm //= 1 if is_native($arch);
3463
3464     if ($kvm) {
3465         die "KVM virtualisation configured, but not available. Either disable in VM configuration or enable in BIOS.\n"
3466             if !defined kvm_version();
3467     }
3468
3469     if ($kvmver =~ m/^(\d+)\.(\d+)$/) {
3470         $vernum = $1*1000000+$2*1000;
3471     } elsif ($kvmver =~ m/^(\d+)\.(\d+)\.(\d+)$/) {
3472         $vernum = $1*1000000+$2*1000+$3;
3473     }
3474
3475     die "detected old qemu-kvm binary ($kvmver)\n" if $vernum < 15000;
3476
3477     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
3478     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
3479     my $use_old_bios_files = undef;
3480     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3481
3482     my $cpuunits = defined($conf->{cpuunits}) ?
3483             $conf->{cpuunits} : $defaults->{cpuunits};
3484
3485     push @$cmd, $kvm_binary;
3486
3487     push @$cmd, '-id', $vmid;
3488
3489     my $vmname = $conf->{name} || "vm$vmid";
3490
3491     push @$cmd, '-name', $vmname;
3492
3493     my $use_virtio = 0;
3494
3495     my $qmpsocket = PVE::QemuServer::Helpers::qmp_socket($vmid);
3496     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server,nowait";
3497     push @$cmd, '-mon', "chardev=qmp,mode=control";
3498
3499     if (min_version($machine_version, 2, 12)) {
3500         push @$cmd, '-chardev', "socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5";
3501         push @$cmd, '-mon', "chardev=qmp-event,mode=control";
3502     }
3503
3504     push @$cmd, '-pidfile' , PVE::QemuServer::Helpers::pidfile_name($vmid);
3505
3506     push @$cmd, '-daemonize';
3507
3508     if ($conf->{smbios1}) {
3509         my $smbios_conf = parse_smbios1($conf->{smbios1});
3510         if ($smbios_conf->{base64}) {
3511             # Do not pass base64 flag to qemu
3512             delete $smbios_conf->{base64};
3513             my $smbios_string = "";
3514             foreach my $key (keys %$smbios_conf) {
3515                 my $value;
3516                 if ($key eq "uuid") {
3517                     $value = $smbios_conf->{uuid}
3518                 } else {
3519                     $value = decode_base64($smbios_conf->{$key});
3520                 }
3521                 # qemu accepts any binary data, only commas need escaping by double comma
3522                 $value =~ s/,/,,/g;
3523                 $smbios_string .= "," . $key . "=" . $value if $value;
3524             }
3525             push @$cmd, '-smbios', "type=1" . $smbios_string;
3526         } else {
3527             push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
3528         }
3529     }
3530
3531     if ($conf->{vmgenid}) {
3532         push @$devices, '-device', 'vmgenid,guid='.$conf->{vmgenid};
3533     }
3534
3535     my ($ovmf_code, $ovmf_vars) = get_ovmf_files($arch);
3536     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3537         die "uefi base image not found\n" if ! -f $ovmf_code;
3538
3539         my $path;
3540         my $format;
3541         if (my $efidisk = $conf->{efidisk0}) {
3542             my $d = PVE::JSONSchema::parse_property_string($efidisk_fmt, $efidisk);
3543             my ($storeid, $volname) = PVE::Storage::parse_volume_id($d->{file}, 1);
3544             $format = $d->{format};
3545             if ($storeid) {
3546                 $path = PVE::Storage::path($storecfg, $d->{file});
3547                 if (!defined($format)) {
3548                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
3549                     $format = qemu_img_format($scfg, $volname);
3550                 }
3551             } else {
3552                 $path = $d->{file};
3553                 die "efidisk format must be specified\n"
3554                     if !defined($format);
3555             }
3556         } else {
3557             warn "no efidisk configured! Using temporary efivars disk.\n";
3558             $path = "/tmp/$vmid-ovmf.fd";
3559             PVE::Tools::file_copy($ovmf_vars, $path, -s $ovmf_vars);
3560             $format = 'raw';
3561         }
3562
3563         push @$cmd, '-drive', "if=pflash,unit=0,format=raw,readonly,file=$ovmf_code";
3564         push @$cmd, '-drive', "if=pflash,unit=1,format=$format,id=drive-efidisk0,file=$path";
3565     }
3566
3567     # load q35 config
3568     if ($q35) {
3569         # we use different pcie-port hardware for qemu >= 4.0 for passthrough
3570         if (min_version($machine_version, 4, 0)) {
3571             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35-4.0.cfg';
3572         } else {
3573             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35.cfg';
3574         }
3575     }
3576
3577     # add usb controllers
3578     my @usbcontrollers = PVE::QemuServer::USB::get_usb_controllers($conf, $bridges, $arch, $machine_type, $usbdesc->{format}, $MAX_USB_DEVICES);
3579     push @$devices, @usbcontrollers if @usbcontrollers;
3580     my $vga = parse_vga($conf->{vga});
3581
3582     my $qxlnum = vga_conf_has_spice($conf->{vga});
3583     $vga->{type} = 'qxl' if $qxlnum;
3584
3585     if (!$vga->{type}) {
3586         if ($arch eq 'aarch64') {
3587             $vga->{type} = 'virtio';
3588         } elsif (min_version($machine_version, 2, 9)) {
3589             $vga->{type} = (!$winversion || $winversion >= 6) ? 'std' : 'cirrus';
3590         } else {
3591             $vga->{type} = ($winversion >= 6) ? 'std' : 'cirrus';
3592         }
3593     }
3594
3595     # enable absolute mouse coordinates (needed by vnc)
3596     my $tablet;
3597     if (defined($conf->{tablet})) {
3598         $tablet = $conf->{tablet};
3599     } else {
3600         $tablet = $defaults->{tablet};
3601         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
3602         $tablet = 0 if $vga->{type} =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
3603     }
3604
3605     if ($tablet) {
3606         push @$devices, '-device', print_tabletdevice_full($conf, $arch) if $tablet;
3607         my $kbd = print_keyboarddevice_full($conf, $arch);
3608         push @$devices, '-device', $kbd if defined($kbd);
3609     }
3610
3611     my $kvm_off = 0;
3612     my $gpu_passthrough;
3613
3614     # host pci devices
3615     for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
3616         my $id = "hostpci$i";
3617         my $d = parse_hostpci($conf->{$id});
3618         next if !$d;
3619
3620         if (my $pcie = $d->{pcie}) {
3621             die "q35 machine model is not enabled" if !$q35;
3622             # win7 wants to have the pcie devices directly on the pcie bus
3623             # instead of in the root port
3624             if ($winversion == 7) {
3625                 $pciaddr = print_pcie_addr("${id}bus0");
3626             } else {
3627                 # add more root ports if needed, 4 are present by default
3628                 # by pve-q35 cfgs, rest added here on demand.
3629                 if ($i > 3) {
3630                     push @$devices, '-device', print_pcie_root_port($i);
3631                 }
3632                 $pciaddr = print_pcie_addr($id);
3633             }
3634         } else {
3635             $pciaddr = print_pci_addr($id, $bridges, $arch, $machine_type);
3636         }
3637
3638         my $xvga = '';
3639         if ($d->{'x-vga'}) {
3640             $xvga = ',x-vga=on' if !($conf->{bios} && $conf->{bios} eq 'ovmf');
3641             $kvm_off = 1;
3642             $vga->{type} = 'none' if !defined($conf->{vga});
3643             $gpu_passthrough = 1;
3644         }
3645
3646         my $pcidevices = $d->{pciid};
3647         my $multifunction = 1 if @$pcidevices > 1;
3648
3649         my $sysfspath;
3650         if ($d->{mdev} && scalar(@$pcidevices) == 1) {
3651             my $pci_id = $pcidevices->[0]->{id};
3652             my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $i);
3653             $sysfspath = "/sys/bus/pci/devices/$pci_id/$uuid";
3654         } elsif ($d->{mdev}) {
3655             warn "ignoring mediated device '$id' with multifunction device\n";
3656         }
3657
3658         my $j=0;
3659         foreach my $pcidevice (@$pcidevices) {
3660             my $devicestr = "vfio-pci";
3661
3662             if ($sysfspath) {
3663                 $devicestr .= ",sysfsdev=$sysfspath";
3664             } else {
3665                 $devicestr .= ",host=$pcidevice->{id}";
3666             }
3667
3668             my $mf_addr = $multifunction ? ".$j" : '';
3669             $devicestr .= ",id=${id}${mf_addr}${pciaddr}${mf_addr}";
3670
3671             if ($j == 0) {
3672                 $devicestr .= ',rombar=0' if defined($d->{rombar}) && !$d->{rombar};
3673                 $devicestr .= "$xvga";
3674                 $devicestr .= ",multifunction=on" if $multifunction;
3675                 $devicestr .= ",romfile=/usr/share/kvm/$d->{romfile}" if $d->{romfile};
3676             }
3677
3678             push @$devices, '-device', $devicestr;
3679             $j++;
3680         }
3681     }
3682
3683     # usb devices
3684     my $usb_dev_features = {};
3685     $usb_dev_features->{spice_usb3} = 1 if min_version($machine_version, 4, 0);
3686
3687     my @usbdevices = PVE::QemuServer::USB::get_usb_devices($conf, $usbdesc->{format}, $MAX_USB_DEVICES, $usb_dev_features);
3688     push @$devices, @usbdevices if @usbdevices;
3689     # serial devices
3690     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3691         if (my $path = $conf->{"serial$i"}) {
3692             if ($path eq 'socket') {
3693                 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
3694                 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server,nowait";
3695                 # On aarch64, serial0 is the UART device. Qemu only allows
3696                 # connecting UART devices via the '-serial' command line, as
3697                 # the device has a fixed slot on the hardware...
3698                 if ($arch eq 'aarch64' && $i == 0) {
3699                     push @$devices, '-serial', "chardev:serial$i";
3700                 } else {
3701                     push @$devices, '-device', "isa-serial,chardev=serial$i";
3702                 }
3703             } else {
3704                 die "no such serial device\n" if ! -c $path;
3705                 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
3706                 push @$devices, '-device', "isa-serial,chardev=serial$i";
3707             }
3708         }
3709     }
3710
3711     # parallel devices
3712     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
3713         if (my $path = $conf->{"parallel$i"}) {
3714             die "no such parallel device\n" if ! -c $path;
3715             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
3716             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
3717             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
3718         }
3719     }
3720
3721     if (my $audio = conf_has_audio($conf)) {
3722
3723         my $audiopciaddr = print_pci_addr("audio0", $bridges, $arch, $machine_type);
3724
3725         my $id = $audio->{dev_id};
3726         if ($audio->{dev} eq 'AC97') {
3727             push @$devices, '-device', "AC97,id=${id}${audiopciaddr}";
3728         } elsif ($audio->{dev} =~ /intel\-hda$/) {
3729             push @$devices, '-device', "$audio->{dev},id=${id}${audiopciaddr}";
3730             push @$devices, '-device', "hda-micro,id=${id}-codec0,bus=${id}.0,cad=0";
3731             push @$devices, '-device', "hda-duplex,id=${id}-codec1,bus=${id}.0,cad=1";
3732         } else {
3733             die "unkown audio device '$audio->{dev}', implement me!";
3734         }
3735
3736         push @$devices, '-audiodev', "$audio->{backend},id=$audio->{backend_id}";
3737     }
3738
3739     my $sockets = 1;
3740     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3741     $sockets = $conf->{sockets} if  $conf->{sockets};
3742
3743     my $cores = $conf->{cores} || 1;
3744
3745     my $maxcpus = $sockets * $cores;
3746
3747     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
3748
3749     my $allowed_vcpus = $cpuinfo->{cpus};
3750
3751     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n"
3752         if ($allowed_vcpus < $maxcpus);
3753
3754     if($hotplug_features->{cpu} && min_version($machine_version, 2, 7)) {
3755
3756         push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3757         for (my $i = 2; $i <= $vcpus; $i++)  {
3758             my $cpustr = print_cpu_device($conf,$i);
3759             push @$cmd, '-device', $cpustr;
3760         }
3761
3762     } else {
3763
3764         push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3765     }
3766     push @$cmd, '-nodefaults';
3767
3768     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
3769
3770     my $bootindex_hash = {};
3771     my $i = 1;
3772     foreach my $o (split(//, $bootorder)) {
3773         $bootindex_hash->{$o} = $i*100;
3774         $i++;
3775     }
3776
3777     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
3778
3779     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
3780
3781     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
3782
3783     if ($vga->{type} && $vga->{type} !~ m/^serial\d+$/ && $vga->{type} ne 'none'){
3784         push @$devices, '-device', print_vga_device($conf, $vga, $arch, $machine_version, $machine_type, undef, $qxlnum, $bridges);
3785         my $socket = PVE::QemuServer::Helpers::vnc_socket($vmid);
3786         push @$cmd,  '-vnc', "unix:$socket,password";
3787     } else {
3788         push @$cmd, '-vga', 'none' if $vga->{type} eq 'none';
3789         push @$cmd, '-nographic';
3790     }
3791
3792     # time drift fix
3793     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
3794
3795     my $useLocaltime = $conf->{localtime};
3796
3797     if ($winversion >= 5) { # windows
3798         $useLocaltime = 1 if !defined($conf->{localtime});
3799
3800         # use time drift fix when acpi is enabled
3801         if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
3802             $tdf = 1 if !defined($conf->{tdf});
3803         }
3804     }
3805
3806     if ($winversion >= 6) {
3807         push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3808         push @$cmd, '-no-hpet';
3809     }
3810
3811     push @$rtcFlags, 'driftfix=slew' if $tdf;
3812
3813     if (!$kvm) {
3814         push @$machineFlags, 'accel=tcg';
3815     }
3816
3817     if ($machine_type) {
3818         push @$machineFlags, "type=${machine_type}";
3819     }
3820
3821     if (($conf->{startdate}) && ($conf->{startdate} ne 'now')) {
3822         push @$rtcFlags, "base=$conf->{startdate}";
3823     } elsif ($useLocaltime) {
3824         push @$rtcFlags, 'base=localtime';
3825     }
3826
3827     push @$cmd, get_cpu_options($conf, $arch, $kvm, $kvm_off, $machine_version, $winversion, $gpu_passthrough);
3828
3829     PVE::QemuServer::Memory::config($conf, $vmid, $sockets, $cores, $defaults, $hotplug_features, $cmd);
3830
3831     push @$cmd, '-S' if $conf->{freeze};
3832
3833     push @$cmd, '-k', $conf->{keyboard} if defined($conf->{keyboard});
3834
3835     my $guest_agent = parse_guest_agent($conf);
3836
3837     if ($guest_agent->{enabled}) {
3838         my $qgasocket = PVE::QemuServer::Helpers::qmp_socket($vmid, 1);
3839         push @$devices, '-chardev', "socket,path=$qgasocket,server,nowait,id=qga0";
3840
3841         if (!$guest_agent->{type} || $guest_agent->{type} eq 'virtio') {
3842             my $pciaddr = print_pci_addr("qga0", $bridges, $arch, $machine_type);
3843             push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3844             push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3845         } elsif ($guest_agent->{type} eq 'isa') {
3846             push @$devices, '-device', "isa-serial,chardev=qga0";
3847         }
3848     }
3849
3850     my $spice_port;
3851
3852     if ($qxlnum) {
3853         if ($qxlnum > 1) {
3854             if ($winversion){
3855                 for(my $i = 1; $i < $qxlnum; $i++){
3856                     push @$devices, '-device', print_vga_device($conf, $vga, $arch, $machine_version, $machine_type, $i, $qxlnum, $bridges);
3857                 }
3858             } else {
3859                 # assume other OS works like Linux
3860                 my ($ram, $vram) = ("134217728", "67108864");
3861                 if ($vga->{memory}) {
3862                     $ram = PVE::Tools::convert_size($qxlnum*4*$vga->{memory}, 'mb' => 'b');
3863                     $vram = PVE::Tools::convert_size($qxlnum*2*$vga->{memory}, 'mb' => 'b');
3864                 }
3865                 push @$cmd, '-global', "qxl-vga.ram_size=$ram";
3866                 push @$cmd, '-global', "qxl-vga.vram_size=$vram";
3867             }
3868         }
3869
3870         my $pciaddr = print_pci_addr("spice", $bridges, $arch, $machine_type);
3871
3872         my $nodename = PVE::INotify::nodename();
3873         my $pfamily = PVE::Tools::get_host_address_family($nodename);
3874         my @nodeaddrs = PVE::Tools::getaddrinfo_all('localhost', family => $pfamily);
3875         die "failed to get an ip address of type $pfamily for 'localhost'\n" if !@nodeaddrs;
3876
3877         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3878         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3879         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3880
3881         my $localhost = PVE::Network::addr_to_ip($nodeaddrs[0]->{addr});
3882         $spice_port = PVE::Tools::next_spice_port($pfamily, $localhost);
3883
3884         my $spice_enhancement = PVE::JSONSchema::parse_property_string($spice_enhancements_fmt, $conf->{spice_enhancements} // '');
3885         if ($spice_enhancement->{foldersharing}) {
3886             push @$devices, '-chardev', "spiceport,id=foldershare,name=org.spice-space.webdav.0";
3887             push @$devices, '-device', "virtserialport,chardev=foldershare,name=org.spice-space.webdav.0";
3888         }
3889
3890         my $spice_opts = "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";
3891         $spice_opts .= ",streaming-video=$spice_enhancement->{videostreaming}" if $spice_enhancement->{videostreaming};
3892         push @$devices, '-spice', "$spice_opts";
3893     }
3894
3895     # enable balloon by default, unless explicitly disabled
3896     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3897         $pciaddr = print_pci_addr("balloon0", $bridges, $arch, $machine_type);
3898         push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
3899     }
3900
3901     if ($conf->{watchdog}) {
3902         my $wdopts = parse_watchdog($conf->{watchdog});
3903         $pciaddr = print_pci_addr("watchdog", $bridges, $arch, $machine_type);
3904         my $watchdog = $wdopts->{model} || 'i6300esb';
3905         push @$devices, '-device', "$watchdog$pciaddr";
3906         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3907     }
3908
3909     my $vollist = [];
3910     my $scsicontroller = {};
3911     my $ahcicontroller = {};
3912     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3913
3914     # Add iscsi initiator name if available
3915     if (my $initiator = get_initiator_name()) {
3916         push @$devices, '-iscsi', "initiator-name=$initiator";
3917     }
3918
3919     foreach_drive($conf, sub {
3920         my ($ds, $drive) = @_;
3921
3922         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3923             push @$vollist, $drive->{file};
3924         }
3925
3926         # ignore efidisk here, already added in bios/fw handling code above
3927         return if $drive->{interface} eq 'efidisk';
3928
3929         $use_virtio = 1 if $ds =~ m/^virtio/;
3930
3931         if (drive_is_cdrom ($drive)) {
3932             if ($bootindex_hash->{d}) {
3933                 $drive->{bootindex} = $bootindex_hash->{d};
3934                 $bootindex_hash->{d} += 1;
3935             }
3936         } else {
3937             if ($bootindex_hash->{c}) {
3938                 $drive->{bootindex} = $bootindex_hash->{c} if $conf->{bootdisk} && ($conf->{bootdisk} eq $ds);
3939                 $bootindex_hash->{c} += 1;
3940             }
3941         }
3942
3943         if($drive->{interface} eq 'virtio'){
3944            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3945         }
3946
3947         if ($drive->{interface} eq 'scsi') {
3948
3949             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
3950
3951             $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges, $arch, $machine_type);
3952             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
3953
3954             my $iothread = '';
3955             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
3956                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3957                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3958             } elsif ($drive->{iothread}) {
3959                 warn "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n";
3960             }
3961
3962             my $queues = '';
3963             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
3964                 $queues = ",num_queues=$drive->{queues}";
3965             }
3966
3967             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues" if !$scsicontroller->{$controller};
3968             $scsicontroller->{$controller}=1;
3969         }
3970
3971         if ($drive->{interface} eq 'sata') {
3972            my $controller = int($drive->{index} / $MAX_SATA_DISKS);
3973            $pciaddr = print_pci_addr("ahci$controller", $bridges, $arch, $machine_type);
3974            push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr" if !$ahcicontroller->{$controller};
3975            $ahcicontroller->{$controller}=1;
3976         }
3977
3978         my $drive_cmd = print_drive_full($storecfg, $vmid, $drive);
3979         push @$devices, '-drive',$drive_cmd;
3980         push @$devices, '-device', print_drivedevice_full($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type);
3981     });
3982
3983     for (my $i = 0; $i < $MAX_NETS; $i++) {
3984          next if !$conf->{"net$i"};
3985          my $d = parse_net($conf->{"net$i"});
3986          next if !$d;
3987
3988          $use_virtio = 1 if $d->{model} eq 'virtio';
3989
3990          if ($bootindex_hash->{n}) {
3991             $d->{bootindex} = $bootindex_hash->{n};
3992             $bootindex_hash->{n} += 1;
3993          }
3994
3995          my $netdevfull = print_netdev_full($vmid, $conf, $arch, $d, "net$i");
3996          push @$devices, '-netdev', $netdevfull;
3997
3998          my $netdevicefull = print_netdevice_full($vmid, $conf, $d, "net$i", $bridges, $use_old_bios_files, $arch, $machine_type);
3999          push @$devices, '-device', $netdevicefull;
4000     }
4001
4002     if ($conf->{ivshmem}) {
4003         my $ivshmem = PVE::JSONSchema::parse_property_string($ivshmem_fmt, $conf->{ivshmem});
4004
4005         my $bus;
4006         if ($q35) {
4007             $bus = print_pcie_addr("ivshmem");
4008         } else {
4009             $bus = print_pci_addr("ivshmem", $bridges, $arch, $machine_type);
4010         }
4011
4012         my $ivshmem_name = $ivshmem->{name} // $vmid;
4013         my $path = '/dev/shm/pve-shm-' . $ivshmem_name;
4014
4015         push @$devices, '-device', "ivshmem-plain,memdev=ivshmem$bus,";
4016         push @$devices, '-object', "memory-backend-file,id=ivshmem,share=on,mem-path=$path,size=$ivshmem->{size}M";
4017     }
4018
4019     if (!$q35) {
4020         # add pci bridges
4021         if (min_version($machine_version, 2, 3)) {
4022            $bridges->{1} = 1;
4023            $bridges->{2} = 1;
4024         }
4025
4026         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
4027
4028         for my $k (sort {$b cmp $a} keys %$bridges) {
4029             $pciaddr = print_pci_addr("pci.$k", undef, $arch, $machine_type);
4030             unshift @$devices, '-device', "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr" if $k > 0;
4031         }
4032     }
4033
4034     push @$cmd, @$devices;
4035     push @$cmd, '-rtc', join(',', @$rtcFlags)
4036         if scalar(@$rtcFlags);
4037     push @$cmd, '-machine', join(',', @$machineFlags)
4038         if scalar(@$machineFlags);
4039     push @$cmd, '-global', join(',', @$globalFlags)
4040         if scalar(@$globalFlags);
4041
4042     if (my $vmstate = $conf->{vmstate}) {
4043         my $statepath = PVE::Storage::path($storecfg, $vmstate);
4044         push @$vollist, $vmstate;
4045         push @$cmd, '-loadstate', $statepath;
4046     }
4047
4048     # add custom args
4049     if ($conf->{args}) {
4050         my $aa = PVE::Tools::split_args($conf->{args});
4051         push @$cmd, @$aa;
4052     }
4053
4054     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
4055 }
4056
4057 sub spice_port {
4058     my ($vmid) = @_;
4059
4060     my $res = mon_cmd($vmid, 'query-spice');
4061
4062     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
4063 }
4064
4065 sub vm_devices_list {
4066     my ($vmid) = @_;
4067
4068     my $res = mon_cmd($vmid, 'query-pci');
4069     my $devices_to_check = [];
4070     my $devices = {};
4071     foreach my $pcibus (@$res) {
4072         push @$devices_to_check, @{$pcibus->{devices}},
4073     }
4074
4075     while (@$devices_to_check) {
4076         my $to_check = [];
4077         for my $d (@$devices_to_check) {
4078             $devices->{$d->{'qdev_id'}} = 1 if $d->{'qdev_id'};
4079             next if !$d->{'pci_bridge'};
4080
4081             $devices->{$d->{'qdev_id'}} += scalar(@{$d->{'pci_bridge'}->{devices}});
4082             push @$to_check, @{$d->{'pci_bridge'}->{devices}};
4083         }
4084         $devices_to_check = $to_check;
4085     }
4086
4087     my $resblock = mon_cmd($vmid, 'query-block');
4088     foreach my $block (@$resblock) {
4089         if($block->{device} =~ m/^drive-(\S+)/){
4090                 $devices->{$1} = 1;
4091         }
4092     }
4093
4094     my $resmice = mon_cmd($vmid, 'query-mice');
4095     foreach my $mice (@$resmice) {
4096         if ($mice->{name} eq 'QEMU HID Tablet') {
4097             $devices->{tablet} = 1;
4098             last;
4099         }
4100     }
4101
4102     # for usb devices there is no query-usb
4103     # but we can iterate over the entries in
4104     # qom-list path=/machine/peripheral
4105     my $resperipheral = mon_cmd($vmid, 'qom-list', path => '/machine/peripheral');
4106     foreach my $per (@$resperipheral) {
4107         if ($per->{name} =~ m/^usb\d+$/) {
4108             $devices->{$per->{name}} = 1;
4109         }
4110     }
4111
4112     return $devices;
4113 }
4114
4115 sub vm_deviceplug {
4116     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4117
4118     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
4119
4120     my $devices_list = vm_devices_list($vmid);
4121     return 1 if defined($devices_list->{$deviceid});
4122
4123     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid, $arch, $machine_type); # add PCI bridge if we need it for the device
4124
4125     if ($deviceid eq 'tablet') {
4126
4127         qemu_deviceadd($vmid, print_tabletdevice_full($conf, $arch));
4128
4129     } elsif ($deviceid eq 'keyboard') {
4130
4131         qemu_deviceadd($vmid, print_keyboarddevice_full($conf, $arch));
4132
4133     } elsif ($deviceid =~ m/^usb(\d+)$/) {
4134
4135         die "usb hotplug currently not reliable\n";
4136         # since we can't reliably hot unplug all added usb devices
4137         # and usb passthrough disables live migration
4138         # we disable usb hotplugging for now
4139         qemu_deviceadd($vmid, PVE::QemuServer::USB::print_usbdevice_full($conf, $deviceid, $device));
4140
4141     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4142
4143         qemu_iothread_add($vmid, $deviceid, $device);
4144
4145         qemu_driveadd($storecfg, $vmid, $device);
4146         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, $arch, $machine_type);
4147
4148         qemu_deviceadd($vmid, $devicefull);
4149         eval { qemu_deviceaddverify($vmid, $deviceid); };
4150         if (my $err = $@) {
4151             eval { qemu_drivedel($vmid, $deviceid); };
4152             warn $@ if $@;
4153             die $err;
4154         }
4155
4156     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4157
4158
4159         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
4160         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
4161         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
4162
4163         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
4164
4165         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
4166             qemu_iothread_add($vmid, $deviceid, $device);
4167             $devicefull .= ",iothread=iothread-$deviceid";
4168         }
4169
4170         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
4171             $devicefull .= ",num_queues=$device->{queues}";
4172         }
4173
4174         qemu_deviceadd($vmid, $devicefull);
4175         qemu_deviceaddverify($vmid, $deviceid);
4176
4177     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4178
4179         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device, $arch, $machine_type);
4180         qemu_driveadd($storecfg, $vmid, $device);
4181
4182         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, $arch, $machine_type);
4183         eval { qemu_deviceadd($vmid, $devicefull); };
4184         if (my $err = $@) {
4185             eval { qemu_drivedel($vmid, $deviceid); };
4186             warn $@ if $@;
4187             die $err;
4188         }
4189
4190     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4191
4192         return undef if !qemu_netdevadd($vmid, $conf, $arch, $device, $deviceid);
4193
4194         my $machine_type = PVE::QemuServer::Machine::qemu_machine_pxe($vmid, $conf);
4195         my $use_old_bios_files = undef;
4196         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
4197
4198         my $netdevicefull = print_netdevice_full($vmid, $conf, $device, $deviceid, undef, $use_old_bios_files, $arch, $machine_type);
4199         qemu_deviceadd($vmid, $netdevicefull);
4200         eval {
4201             qemu_deviceaddverify($vmid, $deviceid);
4202             qemu_set_link_status($vmid, $deviceid, !$device->{link_down});
4203         };
4204         if (my $err = $@) {
4205             eval { qemu_netdevdel($vmid, $deviceid); };
4206             warn $@ if $@;
4207             die $err;
4208         }
4209
4210     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
4211
4212         my $bridgeid = $2;
4213         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
4214         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
4215
4216         qemu_deviceadd($vmid, $devicefull);
4217         qemu_deviceaddverify($vmid, $deviceid);
4218
4219     } else {
4220         die "can't hotplug device '$deviceid'\n";
4221     }
4222
4223     return 1;
4224 }
4225
4226 # fixme: this should raise exceptions on error!
4227 sub vm_deviceunplug {
4228     my ($vmid, $conf, $deviceid) = @_;
4229
4230     my $devices_list = vm_devices_list($vmid);
4231     return 1 if !defined($devices_list->{$deviceid});
4232
4233     die "can't unplug bootdisk" if $conf->{bootdisk} && $conf->{bootdisk} eq $deviceid;
4234
4235     if ($deviceid eq 'tablet' || $deviceid eq 'keyboard') {
4236
4237         qemu_devicedel($vmid, $deviceid);
4238
4239     } elsif ($deviceid =~ m/^usb\d+$/) {
4240
4241         die "usb hotplug currently not reliable\n";
4242         # when unplugging usb devices this way,
4243         # there may be remaining usb controllers/hubs
4244         # so we disable it for now
4245         qemu_devicedel($vmid, $deviceid);
4246         qemu_devicedelverify($vmid, $deviceid);
4247
4248     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4249
4250         qemu_devicedel($vmid, $deviceid);
4251         qemu_devicedelverify($vmid, $deviceid);
4252         qemu_drivedel($vmid, $deviceid);
4253         qemu_iothread_del($conf, $vmid, $deviceid);
4254
4255     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4256
4257         qemu_devicedel($vmid, $deviceid);
4258         qemu_devicedelverify($vmid, $deviceid);
4259         qemu_iothread_del($conf, $vmid, $deviceid);
4260
4261     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4262
4263         qemu_devicedel($vmid, $deviceid);
4264         qemu_drivedel($vmid, $deviceid);
4265         qemu_deletescsihw($conf, $vmid, $deviceid);
4266
4267     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4268
4269         qemu_devicedel($vmid, $deviceid);
4270         qemu_devicedelverify($vmid, $deviceid);
4271         qemu_netdevdel($vmid, $deviceid);
4272
4273     } else {
4274         die "can't unplug device '$deviceid'\n";
4275     }
4276
4277     return 1;
4278 }
4279
4280 sub qemu_deviceadd {
4281     my ($vmid, $devicefull) = @_;
4282
4283     $devicefull = "driver=".$devicefull;
4284     my %options =  split(/[=,]/, $devicefull);
4285
4286     mon_cmd($vmid, "device_add" , %options);
4287 }
4288
4289 sub qemu_devicedel {
4290     my ($vmid, $deviceid) = @_;
4291
4292     my $ret = mon_cmd($vmid, "device_del", id => $deviceid);
4293 }
4294
4295 sub qemu_iothread_add {
4296     my($vmid, $deviceid, $device) = @_;
4297
4298     if ($device->{iothread}) {
4299         my $iothreads = vm_iothreads_list($vmid);
4300         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
4301     }
4302 }
4303
4304 sub qemu_iothread_del {
4305     my($conf, $vmid, $deviceid) = @_;
4306
4307     my $confid = $deviceid;
4308     if ($deviceid =~ m/^(?:virtioscsi|scsihw)(\d+)$/) {
4309         $confid = 'scsi' . $1;
4310     }
4311     my $device = parse_drive($confid, $conf->{$confid});
4312     if ($device->{iothread}) {
4313         my $iothreads = vm_iothreads_list($vmid);
4314         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
4315     }
4316 }
4317
4318 sub qemu_objectadd {
4319     my($vmid, $objectid, $qomtype) = @_;
4320
4321     mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
4322
4323     return 1;
4324 }
4325
4326 sub qemu_objectdel {
4327     my($vmid, $objectid) = @_;
4328
4329     mon_cmd($vmid, "object-del", id => $objectid);
4330
4331     return 1;
4332 }
4333
4334 sub qemu_driveadd {
4335     my ($storecfg, $vmid, $device) = @_;
4336
4337     my $drive = print_drive_full($storecfg, $vmid, $device);
4338     $drive =~ s/\\/\\\\/g;
4339     my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_add auto \"$drive\"");
4340
4341     # If the command succeeds qemu prints: "OK"
4342     return 1 if $ret =~ m/OK/s;
4343
4344     die "adding drive failed: $ret\n";
4345 }
4346
4347 sub qemu_drivedel {
4348     my($vmid, $deviceid) = @_;
4349
4350     my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_del drive-$deviceid");
4351     $ret =~ s/^\s+//;
4352
4353     return 1 if $ret eq "";
4354
4355     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
4356     return 1 if $ret =~ m/Device \'.*?\' not found/s;
4357
4358     die "deleting drive $deviceid failed : $ret\n";
4359 }
4360
4361 sub qemu_deviceaddverify {
4362     my ($vmid, $deviceid) = @_;
4363
4364     for (my $i = 0; $i <= 5; $i++) {
4365          my $devices_list = vm_devices_list($vmid);
4366          return 1 if defined($devices_list->{$deviceid});
4367          sleep 1;
4368     }
4369
4370     die "error on hotplug device '$deviceid'\n";
4371 }
4372
4373
4374 sub qemu_devicedelverify {
4375     my ($vmid, $deviceid) = @_;
4376
4377     # need to verify that the device is correctly removed as device_del
4378     # is async and empty return is not reliable
4379
4380     for (my $i = 0; $i <= 5; $i++) {
4381          my $devices_list = vm_devices_list($vmid);
4382          return 1 if !defined($devices_list->{$deviceid});
4383          sleep 1;
4384     }
4385
4386     die "error on hot-unplugging device '$deviceid'\n";
4387 }
4388
4389 sub qemu_findorcreatescsihw {
4390     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4391
4392     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4393
4394     my $scsihwid="$controller_prefix$controller";
4395     my $devices_list = vm_devices_list($vmid);
4396
4397     if(!defined($devices_list->{$scsihwid})) {
4398         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device, $arch, $machine_type);
4399     }
4400
4401     return 1;
4402 }
4403
4404 sub qemu_deletescsihw {
4405     my ($conf, $vmid, $opt) = @_;
4406
4407     my $device = parse_drive($opt, $conf->{$opt});
4408
4409     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
4410         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
4411         return 1;
4412     }
4413
4414     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4415
4416     my $devices_list = vm_devices_list($vmid);
4417     foreach my $opt (keys %{$devices_list}) {
4418         if (PVE::QemuServer::is_valid_drivename($opt)) {
4419             my $drive = PVE::QemuServer::parse_drive($opt, $conf->{$opt});
4420             if($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
4421                 return 1;
4422             }
4423         }
4424     }
4425
4426     my $scsihwid="scsihw$controller";
4427
4428     vm_deviceunplug($vmid, $conf, $scsihwid);
4429
4430     return 1;
4431 }
4432
4433 sub qemu_add_pci_bridge {
4434     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4435
4436     my $bridges = {};
4437
4438     my $bridgeid;
4439
4440     print_pci_addr($device, $bridges, $arch, $machine_type);
4441
4442     while (my ($k, $v) = each %$bridges) {
4443         $bridgeid = $k;
4444     }
4445     return 1 if !defined($bridgeid) || $bridgeid < 1;
4446
4447     my $bridge = "pci.$bridgeid";
4448     my $devices_list = vm_devices_list($vmid);
4449
4450     if (!defined($devices_list->{$bridge})) {
4451         vm_deviceplug($storecfg, $conf, $vmid, $bridge, $arch, $machine_type);
4452     }
4453
4454     return 1;
4455 }
4456
4457 sub qemu_set_link_status {
4458     my ($vmid, $device, $up) = @_;
4459
4460     mon_cmd($vmid, "set_link", name => $device,
4461                up => $up ? JSON::true : JSON::false);
4462 }
4463
4464 sub qemu_netdevadd {
4465     my ($vmid, $conf, $arch, $device, $deviceid) = @_;
4466
4467     my $netdev = print_netdev_full($vmid, $conf, $arch, $device, $deviceid, 1);
4468     my %options =  split(/[=,]/, $netdev);
4469
4470     mon_cmd($vmid, "netdev_add",  %options);
4471     return 1;
4472 }
4473
4474 sub qemu_netdevdel {
4475     my ($vmid, $deviceid) = @_;
4476
4477     mon_cmd($vmid, "netdev_del", id => $deviceid);
4478 }
4479
4480 sub qemu_usb_hotplug {
4481     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4482
4483     return if !$device;
4484
4485     # remove the old one first
4486     vm_deviceunplug($vmid, $conf, $deviceid);
4487
4488     # check if xhci controller is necessary and available
4489     if ($device->{usb3}) {
4490
4491         my $devicelist = vm_devices_list($vmid);
4492
4493         if (!$devicelist->{xhci}) {
4494             my $pciaddr = print_pci_addr("xhci", undef, $arch, $machine_type);
4495             qemu_deviceadd($vmid, "nec-usb-xhci,id=xhci$pciaddr");
4496         }
4497     }
4498     my $d = parse_usb_device($device->{host});
4499     $d->{usb3} = $device->{usb3};
4500
4501     # add the new one
4502     vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $d, $arch, $machine_type);
4503 }
4504
4505 sub qemu_cpu_hotplug {
4506     my ($vmid, $conf, $vcpus) = @_;
4507
4508     my $machine_type = PVE::QemuServer::Machine::get_current_qemu_machine($vmid);
4509
4510     my $sockets = 1;
4511     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
4512     $sockets = $conf->{sockets} if  $conf->{sockets};
4513     my $cores = $conf->{cores} || 1;
4514     my $maxcpus = $sockets * $cores;
4515
4516     $vcpus = $maxcpus if !$vcpus;
4517
4518     die "you can't add more vcpus than maxcpus\n"
4519         if $vcpus > $maxcpus;
4520
4521     my $currentvcpus = $conf->{vcpus} || $maxcpus;
4522
4523     if ($vcpus < $currentvcpus) {
4524
4525         if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4526
4527             for (my $i = $currentvcpus; $i > $vcpus; $i--) {
4528                 qemu_devicedel($vmid, "cpu$i");
4529                 my $retry = 0;
4530                 my $currentrunningvcpus = undef;
4531                 while (1) {
4532                     $currentrunningvcpus = mon_cmd($vmid, "query-cpus");
4533                     last if scalar(@{$currentrunningvcpus}) == $i-1;
4534                     raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
4535                     $retry++;
4536                     sleep 1;
4537                 }
4538                 #update conf after each succesfull cpu unplug
4539                 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4540                 PVE::QemuConfig->write_config($vmid, $conf);
4541             }
4542         } else {
4543             die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
4544         }
4545
4546         return;
4547     }
4548
4549     my $currentrunningvcpus = mon_cmd($vmid, "query-cpus");
4550     die "vcpus in running vm does not match its configuration\n"
4551         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
4552
4553     if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4554
4555         for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
4556             my $cpustr = print_cpu_device($conf, $i);
4557             qemu_deviceadd($vmid, $cpustr);
4558
4559             my $retry = 0;
4560             my $currentrunningvcpus = undef;
4561             while (1) {
4562                 $currentrunningvcpus = mon_cmd($vmid, "query-cpus");
4563                 last if scalar(@{$currentrunningvcpus}) == $i;
4564                 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
4565                 sleep 1;
4566                 $retry++;
4567             }
4568             #update conf after each succesfull cpu hotplug
4569             $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4570             PVE::QemuConfig->write_config($vmid, $conf);
4571         }
4572     } else {
4573
4574         for (my $i = $currentvcpus; $i < $vcpus; $i++) {
4575             mon_cmd($vmid, "cpu-add", id => int($i));
4576         }
4577     }
4578 }
4579
4580 sub qemu_block_set_io_throttle {
4581     my ($vmid, $deviceid,
4582         $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
4583         $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max,
4584         $bps_max_length, $bps_rd_max_length, $bps_wr_max_length,
4585         $iops_max_length, $iops_rd_max_length, $iops_wr_max_length) = @_;
4586
4587     return if !check_running($vmid) ;
4588
4589     mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
4590         bps => int($bps),
4591         bps_rd => int($bps_rd),
4592         bps_wr => int($bps_wr),
4593         iops => int($iops),
4594         iops_rd => int($iops_rd),
4595         iops_wr => int($iops_wr),
4596         bps_max => int($bps_max),
4597         bps_rd_max => int($bps_rd_max),
4598         bps_wr_max => int($bps_wr_max),
4599         iops_max => int($iops_max),
4600         iops_rd_max => int($iops_rd_max),
4601         iops_wr_max => int($iops_wr_max),
4602         bps_max_length => int($bps_max_length),
4603         bps_rd_max_length => int($bps_rd_max_length),
4604         bps_wr_max_length => int($bps_wr_max_length),
4605         iops_max_length => int($iops_max_length),
4606         iops_rd_max_length => int($iops_rd_max_length),
4607         iops_wr_max_length => int($iops_wr_max_length),
4608     );
4609
4610 }
4611
4612 # old code, only used to shutdown old VM after update
4613 sub __read_avail {
4614     my ($fh, $timeout) = @_;
4615
4616     my $sel = new IO::Select;
4617     $sel->add($fh);
4618
4619     my $res = '';
4620     my $buf;
4621
4622     my @ready;
4623     while (scalar (@ready = $sel->can_read($timeout))) {
4624         my $count;
4625         if ($count = $fh->sysread($buf, 8192)) {
4626             if ($buf =~ /^(.*)\(qemu\) $/s) {
4627                 $res .= $1;
4628                 last;
4629             } else {
4630                 $res .= $buf;
4631             }
4632         } else {
4633             if (!defined($count)) {
4634                 die "$!\n";
4635             }
4636             last;
4637         }
4638     }
4639
4640     die "monitor read timeout\n" if !scalar(@ready);
4641
4642     return $res;
4643 }
4644
4645 sub qemu_block_resize {
4646     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4647
4648     my $running = check_running($vmid);
4649
4650     $size = 0 if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4651
4652     return if !$running;
4653
4654     mon_cmd($vmid, "block_resize", device => $deviceid, size => int($size));
4655
4656 }
4657
4658 sub qemu_volume_snapshot {
4659     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4660
4661     my $running = check_running($vmid);
4662
4663     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4664         mon_cmd($vmid, 'blockdev-snapshot-internal-sync', device => $deviceid, name => $snap);
4665     } else {
4666         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4667     }
4668 }
4669
4670 sub qemu_volume_snapshot_delete {
4671     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4672
4673     my $running = check_running($vmid);
4674
4675     if($running) {
4676
4677         $running = undef;
4678         my $conf = PVE::QemuConfig->load_config($vmid);
4679         foreach_drive($conf, sub {
4680             my ($ds, $drive) = @_;
4681             $running = 1 if $drive->{file} eq $volid;
4682         });
4683     }
4684
4685     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4686         mon_cmd($vmid, 'blockdev-snapshot-delete-internal-sync', device => $deviceid, name => $snap);
4687     } else {
4688         PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4689     }
4690 }
4691
4692 sub set_migration_caps {
4693     my ($vmid) = @_;
4694
4695     my $cap_ref = [];
4696
4697     my $enabled_cap = {
4698         "auto-converge" => 1,
4699         "xbzrle" => 1,
4700         "x-rdma-pin-all" => 0,
4701         "zero-blocks" => 0,
4702         "compress" => 0
4703     };
4704
4705     my $supported_capabilities = mon_cmd($vmid, "query-migrate-capabilities");
4706
4707     for my $supported_capability (@$supported_capabilities) {
4708         push @$cap_ref, {
4709             capability => $supported_capability->{capability},
4710             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4711         };
4712     }
4713
4714     mon_cmd($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4715 }
4716
4717 my $fast_plug_option = {
4718     'lock' => 1,
4719     'name' => 1,
4720     'onboot' => 1,
4721     'shares' => 1,
4722     'startup' => 1,
4723     'description' => 1,
4724     'protection' => 1,
4725     'vmstatestorage' => 1,
4726     'hookscript' => 1,
4727     'tags' => 1,
4728 };
4729
4730 # hotplug changes in [PENDING]
4731 # $selection hash can be used to only apply specified options, for
4732 # example: { cores => 1 } (only apply changed 'cores')
4733 # $errors ref is used to return error messages
4734 sub vmconfig_hotplug_pending {
4735     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4736
4737     my $defaults = load_defaults();
4738     my ($arch, $machine_type) = get_basic_machine_info($conf, undef);
4739
4740     # commit values which do not have any impact on running VM first
4741     # Note: those option cannot raise errors, we we do not care about
4742     # $selection and always apply them.
4743
4744     my $add_error = sub {
4745         my ($opt, $msg) = @_;
4746         $errors->{$opt} = "hotplug problem - $msg";
4747     };
4748
4749     my $changes = 0;
4750     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4751         if ($fast_plug_option->{$opt}) {
4752             $conf->{$opt} = $conf->{pending}->{$opt};
4753             delete $conf->{pending}->{$opt};
4754             $changes = 1;
4755         }
4756     }
4757
4758     if ($changes) {
4759         PVE::QemuConfig->write_config($vmid, $conf);
4760         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4761     }
4762
4763     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4764
4765     my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4766     foreach my $opt (sort keys %$pending_delete_hash) {
4767         next if $selection && !$selection->{$opt};
4768         my $force = $pending_delete_hash->{$opt}->{force};
4769         eval {
4770             if ($opt eq 'hotplug') {
4771                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
4772             } elsif ($opt eq 'tablet') {
4773                 die "skip\n" if !$hotplug_features->{usb};
4774                 if ($defaults->{tablet}) {
4775                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4776                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4777                         if $arch eq 'aarch64';
4778                 } else {
4779                     vm_deviceunplug($vmid, $conf, 'tablet');
4780                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4781                 }
4782             } elsif ($opt =~ m/^usb\d+/) {
4783                 die "skip\n";
4784                 # since we cannot reliably hot unplug usb devices
4785                 # we are disabling it
4786                 die "skip\n" if !$hotplug_features->{usb} || $conf->{$opt} =~ m/spice/i;
4787                 vm_deviceunplug($vmid, $conf, $opt);
4788             } elsif ($opt eq 'vcpus') {
4789                 die "skip\n" if !$hotplug_features->{cpu};
4790                 qemu_cpu_hotplug($vmid, $conf, undef);
4791             } elsif ($opt eq 'balloon') {
4792                 # enable balloon device is not hotpluggable
4793                 die "skip\n" if defined($conf->{balloon}) && $conf->{balloon} == 0;
4794                 # here we reset the ballooning value to memory
4795                 my $balloon = $conf->{memory} || $defaults->{memory};
4796                 mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4797             } elsif ($fast_plug_option->{$opt}) {
4798                 # do nothing
4799             } elsif ($opt =~ m/^net(\d+)$/) {
4800                 die "skip\n" if !$hotplug_features->{network};
4801                 vm_deviceunplug($vmid, $conf, $opt);
4802             } elsif (is_valid_drivename($opt)) {
4803                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4804                 vm_deviceunplug($vmid, $conf, $opt);
4805                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4806             } elsif ($opt =~ m/^memory$/) {
4807                 die "skip\n" if !$hotplug_features->{memory};
4808                 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
4809             } elsif ($opt eq 'cpuunits') {
4810                 cgroups_write("cpu", $vmid, "cpu.shares", $defaults->{cpuunits});
4811             } elsif ($opt eq 'cpulimit') {
4812                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", -1);
4813             } else {
4814                 die "skip\n";
4815             }
4816         };
4817         if (my $err = $@) {
4818             &$add_error($opt, $err) if $err ne "skip\n";
4819         } else {
4820             # save new config if hotplug was successful
4821             delete $conf->{$opt};
4822             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4823             PVE::QemuConfig->write_config($vmid, $conf);
4824             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4825         }
4826     }
4827
4828     my ($apply_pending_cloudinit, $apply_pending_cloudinit_done);
4829     $apply_pending_cloudinit = sub {
4830         return if $apply_pending_cloudinit_done; # once is enough
4831         $apply_pending_cloudinit_done = 1; # once is enough
4832
4833         my ($key, $value) = @_;
4834
4835         my @cloudinit_opts = keys %$confdesc_cloudinit;
4836         foreach my $opt (keys %{$conf->{pending}}) {
4837             next if !grep { $_ eq $opt } @cloudinit_opts;
4838             $conf->{$opt} = delete $conf->{pending}->{$opt};
4839         }
4840
4841         my $new_conf = { %$conf };
4842         $new_conf->{$key} = $value;
4843         PVE::QemuServer::Cloudinit::generate_cloudinitconfig($new_conf, $vmid);
4844     };
4845
4846     foreach my $opt (keys %{$conf->{pending}}) {
4847         next if $selection && !$selection->{$opt};
4848         my $value = $conf->{pending}->{$opt};
4849         eval {
4850             if ($opt eq 'hotplug') {
4851                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
4852             } elsif ($opt eq 'tablet') {
4853                 die "skip\n" if !$hotplug_features->{usb};
4854                 if ($value == 1) {
4855                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4856                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4857                         if $arch eq 'aarch64';
4858                 } elsif ($value == 0) {
4859                     vm_deviceunplug($vmid, $conf, 'tablet');
4860                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4861                 }
4862             } elsif ($opt =~ m/^usb\d+$/) {
4863                 die "skip\n";
4864                 # since we cannot reliably hot unplug usb devices
4865                 # we are disabling it
4866                 die "skip\n" if !$hotplug_features->{usb} || $value =~ m/spice/i;
4867                 my $d = eval { PVE::JSONSchema::parse_property_string($usbdesc->{format}, $value) };
4868                 die "skip\n" if !$d;
4869                 qemu_usb_hotplug($storecfg, $conf, $vmid, $opt, $d, $arch, $machine_type);
4870             } elsif ($opt eq 'vcpus') {
4871                 die "skip\n" if !$hotplug_features->{cpu};
4872                 qemu_cpu_hotplug($vmid, $conf, $value);
4873             } elsif ($opt eq 'balloon') {
4874                 # enable/disable balloning device is not hotpluggable
4875                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
4876                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
4877                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
4878
4879                 # allow manual ballooning if shares is set to zero
4880                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
4881                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
4882                     mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4883                 }
4884             } elsif ($opt =~ m/^net(\d+)$/) {
4885                 # some changes can be done without hotplug
4886                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
4887                                     $vmid, $opt, $value, $arch, $machine_type);
4888             } elsif (is_valid_drivename($opt)) {
4889                 # some changes can be done without hotplug
4890                 my $drive = parse_drive($opt, $value);
4891                 if (drive_is_cloudinit($drive)) {
4892                     &$apply_pending_cloudinit($opt, $value);
4893                 }
4894                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
4895                                      $vmid, $opt, $value, 1, $arch, $machine_type);
4896             } elsif ($opt =~ m/^memory$/) { #dimms
4897                 die "skip\n" if !$hotplug_features->{memory};
4898                 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
4899             } elsif ($opt eq 'cpuunits') {
4900                 cgroups_write("cpu", $vmid, "cpu.shares", $conf->{pending}->{$opt});
4901             } elsif ($opt eq 'cpulimit') {
4902                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
4903                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", $cpulimit);
4904             } else {
4905                 die "skip\n";  # skip non-hot-pluggable options
4906             }
4907         };
4908         if (my $err = $@) {
4909             &$add_error($opt, $err) if $err ne "skip\n";
4910         } else {
4911             # save new config if hotplug was successful
4912             $conf->{$opt} = $value;
4913             delete $conf->{pending}->{$opt};
4914             PVE::QemuConfig->write_config($vmid, $conf);
4915             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4916         }
4917     }
4918 }
4919
4920 sub try_deallocate_drive {
4921     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
4922
4923     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
4924         my $volid = $drive->{file};
4925         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
4926             my $sid = PVE::Storage::parse_volume_id($volid);
4927             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
4928
4929             # check if the disk is really unused
4930             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
4931                 if is_volume_in_use($storecfg, $conf, $key, $volid);
4932             PVE::Storage::vdisk_free($storecfg, $volid);
4933             return 1;
4934         } else {
4935             # If vm is not owner of this disk remove from config
4936             return 1;
4937         }
4938     }
4939
4940     return undef;
4941 }
4942
4943 sub vmconfig_delete_or_detach_drive {
4944     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
4945
4946     my $drive = parse_drive($opt, $conf->{$opt});
4947
4948     my $rpcenv = PVE::RPCEnvironment::get();
4949     my $authuser = $rpcenv->get_user();
4950
4951     if ($force) {
4952         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
4953         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
4954     } else {
4955         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
4956     }
4957 }
4958
4959
4960
4961 sub vmconfig_apply_pending {
4962     my ($vmid, $conf, $storecfg) = @_;
4963
4964     # cold plug
4965
4966     my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4967     foreach my $opt (sort keys %$pending_delete_hash) {
4968         die "internal error" if $opt =~ m/^unused/;
4969         my $force = $pending_delete_hash->{$opt}->{force};
4970         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4971         if (!defined($conf->{$opt})) {
4972             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4973             PVE::QemuConfig->write_config($vmid, $conf);
4974         } elsif (is_valid_drivename($opt)) {
4975             vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4976             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4977             delete $conf->{$opt};
4978             PVE::QemuConfig->write_config($vmid, $conf);
4979         } else {
4980             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4981             delete $conf->{$opt};
4982             PVE::QemuConfig->write_config($vmid, $conf);
4983         }
4984     }
4985
4986     $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4987
4988     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4989         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4990
4991         if (defined($conf->{$opt}) && ($conf->{$opt} eq $conf->{pending}->{$opt})) {
4992             # skip if nothing changed
4993         } elsif (is_valid_drivename($opt)) {
4994             vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
4995                 if defined($conf->{$opt});
4996             $conf->{$opt} = $conf->{pending}->{$opt};
4997         } else {
4998             $conf->{$opt} = $conf->{pending}->{$opt};
4999         }
5000
5001         delete $conf->{pending}->{$opt};
5002         PVE::QemuConfig->write_config($vmid, $conf);
5003     }
5004 }
5005
5006 my $safe_num_ne = sub {
5007     my ($a, $b) = @_;
5008
5009     return 0 if !defined($a) && !defined($b);
5010     return 1 if !defined($a);
5011     return 1 if !defined($b);
5012
5013     return $a != $b;
5014 };
5015
5016 my $safe_string_ne = sub {
5017     my ($a, $b) = @_;
5018
5019     return 0 if !defined($a) && !defined($b);
5020     return 1 if !defined($a);
5021     return 1 if !defined($b);
5022
5023     return $a ne $b;
5024 };
5025
5026 sub vmconfig_update_net {
5027     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
5028
5029     my $newnet = parse_net($value);
5030
5031     if ($conf->{$opt}) {
5032         my $oldnet = parse_net($conf->{$opt});
5033
5034         if (&$safe_string_ne($oldnet->{model}, $newnet->{model}) ||
5035             &$safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
5036             &$safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
5037             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
5038
5039             # for non online change, we try to hot-unplug
5040             die "skip\n" if !$hotplug;
5041             vm_deviceunplug($vmid, $conf, $opt);
5042         } else {
5043
5044             die "internal error" if $opt !~ m/net(\d+)/;
5045             my $iface = "tap${vmid}i$1";
5046
5047             if (&$safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
5048                 &$safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
5049                 &$safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
5050                 &$safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
5051                 PVE::Network::tap_unplug($iface);
5052                 PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
5053             } elsif (&$safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
5054                 # Rate can be applied on its own but any change above needs to
5055                 # include the rate in tap_plug since OVS resets everything.
5056                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
5057             }
5058
5059             if (&$safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
5060                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
5061             }
5062
5063             return 1;
5064         }
5065     }
5066
5067     if ($hotplug) {
5068         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet, $arch, $machine_type);
5069     } else {
5070         die "skip\n";
5071     }
5072 }
5073
5074 sub vmconfig_update_disk {
5075     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $force, $arch, $machine_type) = @_;
5076
5077     # fixme: do we need force?
5078
5079     my $drive = parse_drive($opt, $value);
5080
5081     if ($conf->{$opt}) {
5082
5083         if (my $old_drive = parse_drive($opt, $conf->{$opt}))  {
5084
5085             my $media = $drive->{media} || 'disk';
5086             my $oldmedia = $old_drive->{media} || 'disk';
5087             die "unable to change media type\n" if $media ne $oldmedia;
5088
5089             if (!drive_is_cdrom($old_drive)) {
5090
5091                 if ($drive->{file} ne $old_drive->{file}) {
5092
5093                     die "skip\n" if !$hotplug;
5094
5095                     # unplug and register as unused
5096                     vm_deviceunplug($vmid, $conf, $opt);
5097                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
5098
5099                 } else {
5100                     # update existing disk
5101
5102                     # skip non hotpluggable value
5103                     if (&$safe_string_ne($drive->{discard}, $old_drive->{discard}) ||
5104                         &$safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
5105                         &$safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
5106                         &$safe_string_ne($drive->{cache}, $old_drive->{cache})) {
5107                         die "skip\n";
5108                     }
5109
5110                     # apply throttle
5111                     if (&$safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
5112                         &$safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
5113                         &$safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
5114                         &$safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
5115                         &$safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
5116                         &$safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
5117                         &$safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
5118                         &$safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
5119                         &$safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
5120                         &$safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
5121                         &$safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
5122                         &$safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max}) ||
5123                         &$safe_num_ne($drive->{bps_max_length}, $old_drive->{bps_max_length}) ||
5124                         &$safe_num_ne($drive->{bps_rd_max_length}, $old_drive->{bps_rd_max_length}) ||
5125                         &$safe_num_ne($drive->{bps_wr_max_length}, $old_drive->{bps_wr_max_length}) ||
5126                         &$safe_num_ne($drive->{iops_max_length}, $old_drive->{iops_max_length}) ||
5127                         &$safe_num_ne($drive->{iops_rd_max_length}, $old_drive->{iops_rd_max_length}) ||
5128                         &$safe_num_ne($drive->{iops_wr_max_length}, $old_drive->{iops_wr_max_length})) {
5129
5130                         qemu_block_set_io_throttle($vmid,"drive-$opt",
5131                                                    ($drive->{mbps} || 0)*1024*1024,
5132                                                    ($drive->{mbps_rd} || 0)*1024*1024,
5133                                                    ($drive->{mbps_wr} || 0)*1024*1024,
5134                                                    $drive->{iops} || 0,
5135                                                    $drive->{iops_rd} || 0,
5136                                                    $drive->{iops_wr} || 0,
5137                                                    ($drive->{mbps_max} || 0)*1024*1024,
5138                                                    ($drive->{mbps_rd_max} || 0)*1024*1024,
5139                                                    ($drive->{mbps_wr_max} || 0)*1024*1024,
5140                                                    $drive->{iops_max} || 0,
5141                                                    $drive->{iops_rd_max} || 0,
5142                                                    $drive->{iops_wr_max} || 0,
5143                                                    $drive->{bps_max_length} || 1,
5144                                                    $drive->{bps_rd_max_length} || 1,
5145                                                    $drive->{bps_wr_max_length} || 1,
5146                                                    $drive->{iops_max_length} || 1,
5147                                                    $drive->{iops_rd_max_length} || 1,
5148                                                    $drive->{iops_wr_max_length} || 1);
5149
5150                     }
5151
5152                     return 1;
5153                 }
5154
5155             } else { # cdrom
5156
5157                 if ($drive->{file} eq 'none') {
5158                     mon_cmd($vmid, "eject",force => JSON::true,device => "drive-$opt");
5159                     if (drive_is_cloudinit($old_drive)) {
5160                         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive);
5161                     }
5162                 } else {
5163                     my $path = get_iso_path($storecfg, $vmid, $drive->{file});
5164                     mon_cmd($vmid, "eject", force => JSON::true,device => "drive-$opt"); # force eject if locked
5165                     mon_cmd($vmid, "change", device => "drive-$opt",target => "$path") if $path;
5166                 }
5167
5168                 return 1;
5169             }
5170         }
5171     }
5172
5173     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
5174     # hotplug new disks
5175     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
5176     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive, $arch, $machine_type);
5177 }
5178
5179 sub vm_start {
5180     my ($storecfg, $vmid, $statefile, $skiplock, $migratedfrom, $paused,
5181         $forcemachine, $spice_ticket, $migration_network, $migration_type, $targetstorage) = @_;
5182
5183     PVE::QemuConfig->lock_config($vmid, sub {
5184         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
5185
5186         die "you can't start a vm if it's a template\n" if PVE::QemuConfig->is_template($conf);
5187
5188         my $is_suspended = PVE::QemuConfig->has_lock($conf, 'suspended');
5189
5190         PVE::QemuConfig->check_lock($conf)
5191             if !($skiplock || $is_suspended);
5192
5193         die "VM $vmid already running\n" if check_running($vmid, undef, $migratedfrom);
5194
5195         # clean up leftover reboot request files
5196         eval { clear_reboot_request($vmid); };
5197         warn $@ if $@;
5198
5199         if (!$statefile && scalar(keys %{$conf->{pending}})) {
5200             vmconfig_apply_pending($vmid, $conf, $storecfg);
5201             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5202         }
5203
5204         PVE::QemuServer::Cloudinit::generate_cloudinitconfig($conf, $vmid);
5205
5206         my $defaults = load_defaults();
5207
5208         # set environment variable useful inside network script
5209         $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
5210
5211         my $local_volumes = {};
5212
5213         if ($targetstorage) {
5214             foreach_drive($conf, sub {
5215                 my ($ds, $drive) = @_;
5216
5217                 return if drive_is_cdrom($drive);
5218
5219                 my $volid = $drive->{file};
5220
5221                 return if !$volid;
5222
5223                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
5224
5225                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5226                 return if $scfg->{shared};
5227                 $local_volumes->{$ds} = [$volid, $storeid, $volname];
5228             });
5229
5230             my $format = undef;
5231
5232             foreach my $opt (sort keys %$local_volumes) {
5233
5234                 my ($volid, $storeid, $volname) = @{$local_volumes->{$opt}};
5235                 my $drive = parse_drive($opt, $conf->{$opt});
5236
5237                 #if remote storage is specified, use default format
5238                 if ($targetstorage && $targetstorage ne "1") {
5239                     $storeid = $targetstorage;
5240                     my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
5241                     $format = $defFormat;
5242                 } else {
5243                     #else we use same format than original
5244                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5245                     $format = qemu_img_format($scfg, $volid);
5246                 }
5247
5248                 my $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $format, undef, ($drive->{size}/1024));
5249                 my $newdrive = $drive;
5250                 $newdrive->{format} = $format;
5251                 $newdrive->{file} = $newvolid;
5252                 my $drivestr = PVE::QemuServer::print_drive($vmid, $newdrive);
5253                 $local_volumes->{$opt} = $drivestr;
5254                 #pass drive to conf for command line
5255                 $conf->{$opt} = $drivestr;
5256             }
5257         }
5258
5259         PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-start', 1);
5260
5261         if ($is_suspended) {
5262             # enforce machine type on suspended vm to ensure HW compatibility
5263             $forcemachine = $conf->{runningmachine};
5264             print "Resuming suspended VM\n";
5265         }
5266
5267         my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine);
5268
5269         my $migration_ip;
5270         my $get_migration_ip = sub {
5271             my ($cidr, $nodename) = @_;
5272
5273             return $migration_ip if defined($migration_ip);
5274
5275             if (!defined($cidr)) {
5276                 my $dc_conf = PVE::Cluster::cfs_read_file('datacenter.cfg');
5277                 $cidr = $dc_conf->{migration}->{network};
5278             }
5279
5280             if (defined($cidr)) {
5281                 my $ips = PVE::Network::get_local_ip_from_cidr($cidr);
5282
5283                 die "could not get IP: no address configured on local " .
5284                     "node for network '$cidr'\n" if scalar(@$ips) == 0;
5285
5286                 die "could not get IP: multiple addresses configured on local " .
5287                     "node for network '$cidr'\n" if scalar(@$ips) > 1;
5288
5289                 $migration_ip = @$ips[0];
5290             }
5291
5292             $migration_ip = PVE::Cluster::remote_node_ip($nodename, 1)
5293                 if !defined($migration_ip);
5294
5295             return $migration_ip;
5296         };
5297
5298         my $migrate_uri;
5299         if ($statefile) {
5300             if ($statefile eq 'tcp') {
5301                 my $localip = "localhost";
5302                 my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
5303                 my $nodename = PVE::INotify::nodename();
5304
5305                 if (!defined($migration_type)) {
5306                     if (defined($datacenterconf->{migration}->{type})) {
5307                         $migration_type = $datacenterconf->{migration}->{type};
5308                     } else {
5309                         $migration_type = 'secure';
5310                     }
5311                 }
5312
5313                 if ($migration_type eq 'insecure') {
5314                     $localip = $get_migration_ip->($migration_network, $nodename);
5315                     $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5316                 }
5317
5318                 my $pfamily = PVE::Tools::get_host_address_family($nodename);
5319                 my $migrate_port = PVE::Tools::next_migrate_port($pfamily);
5320                 $migrate_uri = "tcp:${localip}:${migrate_port}";
5321                 push @$cmd, '-incoming', $migrate_uri;
5322                 push @$cmd, '-S';
5323
5324             } elsif ($statefile eq 'unix') {
5325                 # should be default for secure migrations as a ssh TCP forward
5326                 # tunnel is not deterministic reliable ready and fails regurarly
5327                 # to set up in time, so use UNIX socket forwards
5328                 my $socket_addr = "/run/qemu-server/$vmid.migrate";
5329                 unlink $socket_addr;
5330
5331                 $migrate_uri = "unix:$socket_addr";
5332
5333                 push @$cmd, '-incoming', $migrate_uri;
5334                 push @$cmd, '-S';
5335
5336             } elsif (-e $statefile) {
5337                 push @$cmd, '-loadstate', $statefile;
5338             } else {
5339                 my $statepath = PVE::Storage::path($storecfg, $statefile);
5340                 push @$vollist, $statefile;
5341                 push @$cmd, '-loadstate', $statepath;
5342             }
5343         } elsif ($paused) {
5344             push @$cmd, '-S';
5345         }
5346
5347         # host pci devices
5348         for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
5349           my $d = parse_hostpci($conf->{"hostpci$i"});
5350           next if !$d;
5351           my $pcidevices = $d->{pciid};
5352           foreach my $pcidevice (@$pcidevices) {
5353                 my $pciid = $pcidevice->{id};
5354
5355                 my $info = PVE::SysFSTools::pci_device_info("$pciid");
5356                 die "IOMMU not present\n" if !PVE::SysFSTools::check_iommu_support();
5357                 die "no pci device info for device '$pciid'\n" if !$info;
5358
5359                 if ($d->{mdev}) {
5360                     my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $i);
5361                     PVE::SysFSTools::pci_create_mdev_device($pciid, $uuid, $d->{mdev});
5362                 } else {
5363                     die "can't unbind/bind pci group to vfio '$pciid'\n"
5364                         if !PVE::SysFSTools::pci_dev_group_bind_to_vfio($pciid);
5365                     die "can't reset pci device '$pciid'\n"
5366                         if $info->{has_fl_reset} and !PVE::SysFSTools::pci_dev_reset($info);
5367                 }
5368           }
5369         }
5370
5371         PVE::Storage::activate_volumes($storecfg, $vollist);
5372
5373         eval {
5374             run_command(['/bin/systemctl', 'stop', "$vmid.scope"],
5375                 outfunc => sub {}, errfunc => sub {});
5376         };
5377         # Issues with the above 'stop' not being fully completed are extremely rare, a very low
5378         # timeout should be more than enough here...
5379         PVE::Systemd::wait_for_unit_removed("$vmid.scope", 5);
5380
5381         my $cpuunits = defined($conf->{cpuunits}) ? $conf->{cpuunits}
5382                                                   : $defaults->{cpuunits};
5383
5384         my $start_timeout = ($conf->{hugepages} || $is_suspended) ? 300 : 30;
5385         my %run_params = (timeout => $statefile ? undef : $start_timeout, umask => 0077);
5386
5387         my %properties = (
5388             Slice => 'qemu.slice',
5389             KillMode => 'none',
5390             CPUShares => $cpuunits
5391         );
5392
5393         if (my $cpulimit = $conf->{cpulimit}) {
5394             $properties{CPUQuota} = int($cpulimit * 100);
5395         }
5396         $properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
5397
5398         my $run_qemu = sub {
5399             PVE::Tools::run_fork sub {
5400                 PVE::Systemd::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
5401                 run_command($cmd, %run_params);
5402             };
5403         };
5404
5405         if ($conf->{hugepages}) {
5406
5407             my $code = sub {
5408                 my $hugepages_topology = PVE::QemuServer::Memory::hugepages_topology($conf);
5409                 my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
5410
5411                 PVE::QemuServer::Memory::hugepages_mount();
5412                 PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
5413
5414                 eval { $run_qemu->() };
5415                 if (my $err = $@) {
5416                     PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology);
5417                     die $err;
5418                 }
5419
5420                 PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology);
5421             };
5422             eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
5423
5424         } else {
5425             eval { $run_qemu->() };
5426         }
5427
5428         if (my $err = $@) {
5429             # deactivate volumes if start fails
5430             eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
5431             die "start failed: $err";
5432         }
5433
5434         print "migration listens on $migrate_uri\n" if $migrate_uri;
5435
5436         if ($statefile && $statefile ne 'tcp' && $statefile ne 'unix')  {
5437             eval { mon_cmd($vmid, "cont"); };
5438             warn $@ if $@;
5439         }
5440
5441         #start nbd server for storage migration
5442         if ($targetstorage) {
5443             my $nodename = PVE::INotify::nodename();
5444             my $localip = $get_migration_ip->($migration_network, $nodename);
5445             my $pfamily = PVE::Tools::get_host_address_family($nodename);
5446             my $storage_migrate_port = PVE::Tools::next_migrate_port($pfamily);
5447
5448             mon_cmd($vmid, "nbd-server-start", addr => { type => 'inet', data => { host => "${localip}", port => "${storage_migrate_port}" } } );
5449
5450             $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5451
5452             foreach my $opt (sort keys %$local_volumes) {
5453                 my $volid = $local_volumes->{$opt};
5454                 mon_cmd($vmid, "nbd-server-add", device => "drive-$opt", writable => JSON::true );
5455                 my $migrate_storage_uri = "nbd:${localip}:${storage_migrate_port}:exportname=drive-$opt";
5456                 print "storage migration listens on $migrate_storage_uri volume:$volid\n";
5457             }
5458         }
5459
5460         if ($migratedfrom) {
5461             eval {
5462                 set_migration_caps($vmid);
5463             };
5464             warn $@ if $@;
5465
5466             if ($spice_port) {
5467                 print "spice listens on port $spice_port\n";
5468                 if ($spice_ticket) {
5469                     mon_cmd($vmid, "set_password", protocol => 'spice', password => $spice_ticket);
5470                     mon_cmd($vmid, "expire_password", protocol => 'spice', time => "+30");
5471                 }
5472             }
5473
5474         } else {
5475             mon_cmd($vmid, "balloon", value => $conf->{balloon}*1024*1024)
5476                 if !$statefile && $conf->{balloon};
5477
5478             foreach my $opt (keys %$conf) {
5479                 next if $opt !~  m/^net\d+$/;
5480                 my $nicconf = parse_net($conf->{$opt});
5481                 qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
5482             }
5483         }
5484
5485         mon_cmd($vmid, 'qom-set',
5486                     path => "machine/peripheral/balloon0",
5487                     property => "guest-stats-polling-interval",
5488                     value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
5489
5490         if ($is_suspended && (my $vmstate = $conf->{vmstate})) {
5491             print "Resumed VM, removing state\n";
5492             delete $conf->@{qw(lock vmstate runningmachine)};
5493             PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
5494             PVE::Storage::vdisk_free($storecfg, $vmstate);
5495             PVE::QemuConfig->write_config($vmid, $conf);
5496         }
5497
5498         PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'post-start');
5499     });
5500 }
5501
5502 sub vm_commandline {
5503     my ($storecfg, $vmid, $snapname) = @_;
5504
5505     my $conf = PVE::QemuConfig->load_config($vmid);
5506     my $forcemachine;
5507
5508     if ($snapname) {
5509         my $snapshot = $conf->{snapshots}->{$snapname};
5510         die "snapshot '$snapname' does not exist\n" if !defined($snapshot);
5511
5512         # check for a 'runningmachine' in snapshot
5513         $forcemachine = $snapshot->{runningmachine} if $snapshot->{runningmachine};
5514
5515         $snapshot->{digest} = $conf->{digest}; # keep file digest for API
5516
5517         $conf = $snapshot;
5518     }
5519
5520     my $defaults = load_defaults();
5521
5522     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine);
5523
5524     return PVE::Tools::cmd2string($cmd);
5525 }
5526
5527 sub vm_reset {
5528     my ($vmid, $skiplock) = @_;
5529
5530     PVE::QemuConfig->lock_config($vmid, sub {
5531
5532         my $conf = PVE::QemuConfig->load_config($vmid);
5533
5534         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5535
5536         mon_cmd($vmid, "system_reset");
5537     });
5538 }
5539
5540 sub get_vm_volumes {
5541     my ($conf) = @_;
5542
5543     my $vollist = [];
5544     foreach_volid($conf, sub {
5545         my ($volid, $attr) = @_;
5546
5547         return if $volid =~ m|^/|;
5548
5549         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5550         return if !$sid;
5551
5552         push @$vollist, $volid;
5553     });
5554
5555     return $vollist;
5556 }
5557
5558 sub vm_stop_cleanup {
5559     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
5560
5561     eval {
5562
5563         if (!$keepActive) {
5564             my $vollist = get_vm_volumes($conf);
5565             PVE::Storage::deactivate_volumes($storecfg, $vollist);
5566         }
5567
5568         foreach my $ext (qw(mon qmp pid vnc qga)) {
5569             unlink "/var/run/qemu-server/${vmid}.$ext";
5570         }
5571
5572         if ($conf->{ivshmem}) {
5573             my $ivshmem = PVE::JSONSchema::parse_property_string($ivshmem_fmt, $conf->{ivshmem});
5574             # just delete it for now, VMs which have this already open do not
5575             # are affected, but new VMs will get a separated one. If this
5576             # becomes an issue we either add some sort of ref-counting or just
5577             # add a "don't delete on stop" flag to the ivshmem format.
5578             unlink '/dev/shm/pve-shm-' . ($ivshmem->{name} // $vmid);
5579         }
5580
5581         foreach my $key (keys %$conf) {
5582             next if $key !~ m/^hostpci(\d+)$/;
5583             my $hostpciindex = $1;
5584             my $d = parse_hostpci($conf->{$key});
5585             my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $hostpciindex);
5586
5587             foreach my $pci (@{$d->{pciid}}) {
5588                 my $pciid = $pci->{id};
5589                 PVE::SysFSTools::pci_cleanup_mdev_device($pciid, $uuid);
5590             }
5591         }
5592
5593         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
5594     };
5595     warn $@ if $@; # avoid errors - just warn
5596 }
5597
5598 # call only in locked context
5599 sub _do_vm_stop {
5600     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive) = @_;
5601
5602     my $pid = check_running($vmid, $nocheck);
5603     return if !$pid;
5604
5605     my $conf;
5606     if (!$nocheck) {
5607         $conf = PVE::QemuConfig->load_config($vmid);
5608         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5609         if (!defined($timeout) && $shutdown && $conf->{startup}) {
5610             my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
5611             $timeout = $opts->{down} if $opts->{down};
5612         }
5613         PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-stop');
5614     }
5615
5616     eval {
5617         if ($shutdown) {
5618             if (defined($conf) && parse_guest_agent($conf)->{enabled}) {
5619                 mon_cmd($vmid, "guest-shutdown", timeout => $timeout);
5620             } else {
5621                 mon_cmd($vmid, "system_powerdown");
5622             }
5623         } else {
5624             mon_cmd($vmid, "quit");
5625         }
5626     };
5627     my $err = $@;
5628
5629     if (!$err) {
5630         $timeout = 60 if !defined($timeout);
5631
5632         my $count = 0;
5633         while (($count < $timeout) && check_running($vmid, $nocheck)) {
5634             $count++;
5635             sleep 1;
5636         }
5637
5638         if ($count >= $timeout) {
5639             if ($force) {
5640                 warn "VM still running - terminating now with SIGTERM\n";
5641                 kill 15, $pid;
5642             } else {
5643                 die "VM quit/powerdown failed - got timeout\n";
5644             }
5645         } else {
5646             vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5647             return;
5648         }
5649     } else {
5650         if ($force) {
5651             warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
5652             kill 15, $pid;
5653         } else {
5654             die "VM quit/powerdown failed\n";
5655         }
5656     }
5657
5658     # wait again
5659     $timeout = 10;
5660
5661     my $count = 0;
5662     while (($count < $timeout) && check_running($vmid, $nocheck)) {
5663         $count++;
5664         sleep 1;
5665     }
5666
5667     if ($count >= $timeout) {
5668         warn "VM still running - terminating now with SIGKILL\n";
5669         kill 9, $pid;
5670         sleep 1;
5671     }
5672
5673     vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5674 }
5675
5676 # Note: use $nocheck to skip tests if VM configuration file exists.
5677 # We need that when migration VMs to other nodes (files already moved)
5678 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
5679 sub vm_stop {
5680     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
5681
5682     $force = 1 if !defined($force) && !$shutdown;
5683
5684     if ($migratedfrom){
5685         my $pid = check_running($vmid, $nocheck, $migratedfrom);
5686         kill 15, $pid if $pid;
5687         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
5688         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
5689         return;
5690     }
5691
5692     PVE::QemuConfig->lock_config($vmid, sub {
5693         _do_vm_stop($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive);
5694    });
5695 }
5696
5697 sub vm_reboot {
5698     my ($vmid, $timeout) = @_;
5699
5700     PVE::QemuConfig->lock_config($vmid, sub {
5701         eval {
5702
5703             # only reboot if running, as qmeventd starts it again on a stop event
5704             return if !check_running($vmid);
5705
5706             create_reboot_request($vmid);
5707
5708             my $storecfg = PVE::Storage::config();
5709             _do_vm_stop($storecfg, $vmid, undef, undef, $timeout, 1);
5710
5711         };
5712         if (my $err = $@) {
5713             # avoid that the next normal shutdown will be confused for a reboot
5714             clear_reboot_request($vmid);
5715             die $err;
5716         }
5717    });
5718 }
5719
5720 sub vm_suspend {
5721     my ($vmid, $skiplock, $includestate, $statestorage) = @_;
5722
5723     my $conf;
5724     my $path;
5725     my $storecfg;
5726     my $vmstate;
5727
5728     PVE::QemuConfig->lock_config($vmid, sub {
5729
5730         $conf = PVE::QemuConfig->load_config($vmid);
5731
5732         my $is_backing_up = PVE::QemuConfig->has_lock($conf, 'backup');
5733         PVE::QemuConfig->check_lock($conf)
5734             if !($skiplock || $is_backing_up);
5735
5736         die "cannot suspend to disk during backup\n"
5737             if $is_backing_up && $includestate;
5738
5739         if ($includestate) {
5740             $conf->{lock} = 'suspending';
5741             my $date = strftime("%Y-%m-%d", localtime(time()));
5742             $storecfg = PVE::Storage::config();
5743             $vmstate = PVE::QemuConfig->__snapshot_save_vmstate($vmid, $conf, "suspend-$date", $storecfg, $statestorage, 1);
5744             $path = PVE::Storage::path($storecfg, $vmstate);
5745             PVE::QemuConfig->write_config($vmid, $conf);
5746         } else {
5747             mon_cmd($vmid, "stop");
5748         }
5749     });
5750
5751     if ($includestate) {
5752         # save vm state
5753         PVE::Storage::activate_volumes($storecfg, [$vmstate]);
5754
5755         eval {
5756             mon_cmd($vmid, "savevm-start", statefile => $path);
5757             for(;;) {
5758                 my $state = mon_cmd($vmid, "query-savevm");
5759                 if (!$state->{status}) {
5760                     die "savevm not active\n";
5761                 } elsif ($state->{status} eq 'active') {
5762                     sleep(1);
5763                     next;
5764                 } elsif ($state->{status} eq 'completed') {
5765                     print "State saved, quitting\n";
5766                     last;
5767                 } elsif ($state->{status} eq 'failed' && $state->{error}) {
5768                     die "query-savevm failed with error '$state->{error}'\n"
5769                 } else {
5770                     die "query-savevm returned status '$state->{status}'\n";
5771                 }
5772             }
5773         };
5774         my $err = $@;
5775
5776         PVE::QemuConfig->lock_config($vmid, sub {
5777             $conf = PVE::QemuConfig->load_config($vmid);
5778             if ($err) {
5779                 # cleanup, but leave suspending lock, to indicate something went wrong
5780                 eval {
5781                     mon_cmd($vmid, "savevm-end");
5782                     PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
5783                     PVE::Storage::vdisk_free($storecfg, $vmstate);
5784                     delete $conf->@{qw(vmstate runningmachine)};
5785                     PVE::QemuConfig->write_config($vmid, $conf);
5786                 };
5787                 warn $@ if $@;
5788                 die $err;
5789             }
5790
5791             die "lock changed unexpectedly\n"
5792                 if !PVE::QemuConfig->has_lock($conf, 'suspending');
5793
5794             mon_cmd($vmid, "quit");
5795             $conf->{lock} = 'suspended';
5796             PVE::QemuConfig->write_config($vmid, $conf);
5797         });
5798     }
5799 }
5800
5801 sub vm_resume {
5802     my ($vmid, $skiplock, $nocheck) = @_;
5803
5804     PVE::QemuConfig->lock_config($vmid, sub {
5805         my $res = mon_cmd($vmid, 'query-status');
5806         my $resume_cmd = 'cont';
5807
5808         if ($res->{status} && $res->{status} eq 'suspended') {
5809             $resume_cmd = 'system_wakeup';
5810         }
5811
5812         if (!$nocheck) {
5813
5814             my $conf = PVE::QemuConfig->load_config($vmid);
5815
5816             PVE::QemuConfig->check_lock($conf)
5817                 if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
5818         }
5819
5820         mon_cmd($vmid, $resume_cmd);
5821     });
5822 }
5823
5824 sub vm_sendkey {
5825     my ($vmid, $skiplock, $key) = @_;
5826
5827     PVE::QemuConfig->lock_config($vmid, sub {
5828
5829         my $conf = PVE::QemuConfig->load_config($vmid);
5830
5831         # there is no qmp command, so we use the human monitor command
5832         my $res = PVE::QemuServer::Monitor::hmp_cmd($vmid, "sendkey $key");
5833         die $res if $res ne '';
5834     });
5835 }
5836
5837 # vzdump restore implementaion
5838
5839 sub tar_archive_read_firstfile {
5840     my $archive = shift;
5841
5842     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
5843
5844     # try to detect archive type first
5845     my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
5846         die "unable to open file '$archive'\n";
5847     my $firstfile = <$fh>;
5848     kill 15, $pid;
5849     close $fh;
5850
5851     die "ERROR: archive contaions no data\n" if !$firstfile;
5852     chomp $firstfile;
5853
5854     return $firstfile;
5855 }
5856
5857 sub tar_restore_cleanup {
5858     my ($storecfg, $statfile) = @_;
5859
5860     print STDERR "starting cleanup\n";
5861
5862     if (my $fd = IO::File->new($statfile, "r")) {
5863         while (defined(my $line = <$fd>)) {
5864             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5865                 my $volid = $2;
5866                 eval {
5867                     if ($volid =~ m|^/|) {
5868                         unlink $volid || die 'unlink failed\n';
5869                     } else {
5870                         PVE::Storage::vdisk_free($storecfg, $volid);
5871                     }
5872                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
5873                 };
5874                 print STDERR "unable to cleanup '$volid' - $@" if $@;
5875             } else {
5876                 print STDERR "unable to parse line in statfile - $line";
5877             }
5878         }
5879         $fd->close();
5880     }
5881 }
5882
5883 sub restore_archive {
5884     my ($archive, $vmid, $user, $opts) = @_;
5885
5886     my $format = $opts->{format};
5887     my $comp;
5888
5889     if ($archive =~ m/\.tgz$/ || $archive =~ m/\.tar\.gz$/) {
5890         $format = 'tar' if !$format;
5891         $comp = 'gzip';
5892     } elsif ($archive =~ m/\.tar$/) {
5893         $format = 'tar' if !$format;
5894     } elsif ($archive =~ m/.tar.lzo$/) {
5895         $format = 'tar' if !$format;
5896         $comp = 'lzop';
5897     } elsif ($archive =~ m/\.vma$/) {
5898         $format = 'vma' if !$format;
5899     } elsif ($archive =~ m/\.vma\.gz$/) {
5900         $format = 'vma' if !$format;
5901         $comp = 'gzip';
5902     } elsif ($archive =~ m/\.vma\.lzo$/) {
5903         $format = 'vma' if !$format;
5904         $comp = 'lzop';
5905     } else {
5906         $format = 'vma' if !$format; # default
5907     }
5908
5909     # try to detect archive format
5910     if ($format eq 'tar') {
5911         return restore_tar_archive($archive, $vmid, $user, $opts);
5912     } else {
5913         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
5914     }
5915 }
5916
5917 sub restore_update_config_line {
5918     my ($outfd, $cookie, $vmid, $map, $line, $unique) = @_;
5919
5920     return if $line =~ m/^\#qmdump\#/;
5921     return if $line =~ m/^\#vzdump\#/;
5922     return if $line =~ m/^lock:/;
5923     return if $line =~ m/^unused\d+:/;
5924     return if $line =~ m/^parent:/;
5925
5926     my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
5927     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
5928         # try to convert old 1.X settings
5929         my ($id, $ind, $ethcfg) = ($1, $2, $3);
5930         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
5931             my ($model, $macaddr) = split(/\=/, $devconfig);
5932             $macaddr = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if !$macaddr || $unique;
5933             my $net = {
5934                 model => $model,
5935                 bridge => "vmbr$ind",
5936                 macaddr => $macaddr,
5937             };
5938             my $netstr = print_net($net);
5939
5940             print $outfd "net$cookie->{netcount}: $netstr\n";
5941             $cookie->{netcount}++;
5942         }
5943     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
5944         my ($id, $netstr) = ($1, $2);
5945         my $net = parse_net($netstr);
5946         $net->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if $net->{macaddr};
5947         $netstr = print_net($net);
5948         print $outfd "$id: $netstr\n";
5949     } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk)\d+):\s*(\S+)\s*$/) {
5950         my $virtdev = $1;
5951         my $value = $3;
5952         my $di = parse_drive($virtdev, $value);
5953         if (defined($di->{backup}) && !$di->{backup}) {
5954             print $outfd "#$line";
5955         } elsif ($map->{$virtdev}) {
5956             delete $di->{format}; # format can change on restore
5957             $di->{file} = $map->{$virtdev};
5958             $value = print_drive($vmid, $di);
5959             print $outfd "$virtdev: $value\n";
5960         } else {
5961             print $outfd $line;
5962         }
5963     } elsif (($line =~ m/^vmgenid: (.*)/)) {
5964         my $vmgenid = $1;
5965         if ($vmgenid ne '0') {
5966             # always generate a new vmgenid if there was a valid one setup
5967             $vmgenid = generate_uuid();
5968         }
5969         print $outfd "vmgenid: $vmgenid\n";
5970     } elsif (($line =~ m/^(smbios1: )(.*)/) && $unique) {
5971         my ($uuid, $uuid_str);
5972         UUID::generate($uuid);
5973         UUID::unparse($uuid, $uuid_str);
5974         my $smbios1 = parse_smbios1($2);
5975         $smbios1->{uuid} = $uuid_str;
5976         print $outfd $1.print_smbios1($smbios1)."\n";
5977     } else {
5978         print $outfd $line;
5979     }
5980 }
5981
5982 sub scan_volids {
5983     my ($cfg, $vmid) = @_;
5984
5985     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid);
5986
5987     my $volid_hash = {};
5988     foreach my $storeid (keys %$info) {
5989         foreach my $item (@{$info->{$storeid}}) {
5990             next if !($item->{volid} && $item->{size});
5991             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
5992             $volid_hash->{$item->{volid}} = $item;
5993         }
5994     }
5995
5996     return $volid_hash;
5997 }
5998
5999 sub is_volume_in_use {
6000     my ($storecfg, $conf, $skip_drive, $volid) = @_;
6001
6002     my $path = PVE::Storage::path($storecfg, $volid);
6003
6004     my $scan_config = sub {
6005         my ($cref, $snapname) = @_;
6006
6007         foreach my $key (keys %$cref) {
6008             my $value = $cref->{$key};
6009             if (is_valid_drivename($key)) {
6010                 next if $skip_drive && $key eq $skip_drive;
6011                 my $drive = parse_drive($key, $value);
6012                 next if !$drive || !$drive->{file} || drive_is_cdrom($drive);
6013                 return 1 if $volid eq $drive->{file};
6014                 if ($drive->{file} =~ m!^/!) {
6015                     return 1 if $drive->{file} eq $path;
6016                 } else {
6017                     my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file}, 1);
6018                     next if !$storeid;
6019                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid, 1);
6020                     next if !$scfg;
6021                     return 1 if $path eq PVE::Storage::path($storecfg, $drive->{file}, $snapname);
6022                 }
6023             }
6024         }
6025
6026         return 0;
6027     };
6028
6029     return 1 if &$scan_config($conf);
6030
6031     undef $skip_drive;
6032
6033     foreach my $snapname (keys %{$conf->{snapshots}}) {
6034         return 1 if &$scan_config($conf->{snapshots}->{$snapname}, $snapname);
6035     }
6036
6037     return 0;
6038 }
6039
6040 sub update_disksize {
6041     my ($vmid, $conf, $volid_hash) = @_;
6042
6043     my $changes;
6044     my $prefix = "VM $vmid:";
6045
6046     # used and unused disks
6047     my $referenced = {};
6048
6049     # Note: it is allowed to define multiple storages with same path (alias), so
6050     # we need to check both 'volid' and real 'path' (two different volid can point
6051     # to the same path).
6052
6053     my $referencedpath = {};
6054
6055     # update size info
6056     foreach my $opt (keys %$conf) {
6057         if (is_valid_drivename($opt)) {
6058             my $drive = parse_drive($opt, $conf->{$opt});
6059             my $volid = $drive->{file};
6060             next if !$volid;
6061
6062             $referenced->{$volid} = 1;
6063             if ($volid_hash->{$volid} &&
6064                 (my $path = $volid_hash->{$volid}->{path})) {
6065                 $referencedpath->{$path} = 1;
6066             }
6067
6068             next if drive_is_cdrom($drive);
6069             next if !$volid_hash->{$volid};
6070
6071             $drive->{size} = $volid_hash->{$volid}->{size};
6072             my $new = print_drive($vmid, $drive);
6073             if ($new ne $conf->{$opt}) {
6074                 $changes = 1;
6075                 $conf->{$opt} = $new;
6076                 print "$prefix update disk '$opt' information.\n";
6077             }
6078         }
6079     }
6080
6081     # remove 'unusedX' entry if volume is used
6082     foreach my $opt (keys %$conf) {
6083         next if $opt !~ m/^unused\d+$/;
6084         my $volid = $conf->{$opt};
6085         my $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
6086         if ($referenced->{$volid} || ($path && $referencedpath->{$path})) {
6087             print "$prefix remove entry '$opt', its volume '$volid' is in use.\n";
6088             $changes = 1;
6089             delete $conf->{$opt};
6090         }
6091
6092         $referenced->{$volid} = 1;
6093         $referencedpath->{$path} = 1 if $path;
6094     }
6095
6096     foreach my $volid (sort keys %$volid_hash) {
6097         next if $volid =~ m/vm-$vmid-state-/;
6098         next if $referenced->{$volid};
6099         my $path = $volid_hash->{$volid}->{path};
6100         next if !$path; # just to be sure
6101         next if $referencedpath->{$path};
6102         $changes = 1;
6103         my $key = PVE::QemuConfig->add_unused_volume($conf, $volid);
6104         print "$prefix add unreferenced volume '$volid' as '$key' to config.\n";
6105         $referencedpath->{$path} = 1; # avoid to add more than once (aliases)
6106     }
6107
6108     return $changes;
6109 }
6110
6111 sub rescan {
6112     my ($vmid, $nolock, $dryrun) = @_;
6113
6114     my $cfg = PVE::Storage::config();
6115
6116     # FIXME: Remove once our RBD plugin can handle CT and VM on a single storage
6117     # see: https://pve.proxmox.com/pipermail/pve-devel/2018-July/032900.html
6118     foreach my $stor (keys %{$cfg->{ids}}) {
6119         delete($cfg->{ids}->{$stor}) if ! $cfg->{ids}->{$stor}->{content}->{images};
6120     }
6121
6122     print "rescan volumes...\n";
6123     my $volid_hash = scan_volids($cfg, $vmid);
6124
6125     my $updatefn =  sub {
6126         my ($vmid) = @_;
6127
6128         my $conf = PVE::QemuConfig->load_config($vmid);
6129
6130         PVE::QemuConfig->check_lock($conf);
6131
6132         my $vm_volids = {};
6133         foreach my $volid (keys %$volid_hash) {
6134             my $info = $volid_hash->{$volid};
6135             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
6136         }
6137
6138         my $changes = update_disksize($vmid, $conf, $vm_volids);
6139
6140         PVE::QemuConfig->write_config($vmid, $conf) if $changes && !$dryrun;
6141     };
6142
6143     if (defined($vmid)) {
6144         if ($nolock) {
6145             &$updatefn($vmid);
6146         } else {
6147             PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6148         }
6149     } else {
6150         my $vmlist = config_list();
6151         foreach my $vmid (keys %$vmlist) {
6152             if ($nolock) {
6153                 &$updatefn($vmid);
6154             } else {
6155                 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6156             }
6157         }
6158     }
6159 }
6160
6161 sub restore_vma_archive {
6162     my ($archive, $vmid, $user, $opts, $comp) = @_;
6163
6164     my $readfrom = $archive;
6165
6166     my $cfg = PVE::Storage::config();
6167     my $commands = [];
6168     my $bwlimit = $opts->{bwlimit};
6169
6170     my $dbg_cmdstring = '';
6171     my $add_pipe = sub {
6172         my ($cmd) = @_;
6173         push @$commands, $cmd;
6174         $dbg_cmdstring .= ' | ' if length($dbg_cmdstring);
6175         $dbg_cmdstring .= PVE::Tools::cmd2string($cmd);
6176         $readfrom = '-';
6177     };
6178
6179     my $input = undef;
6180     if ($archive eq '-') {
6181         $input = '<&STDIN';
6182     } else {
6183         # If we use a backup from a PVE defined storage we also consider that
6184         # storage's rate limit:
6185         my (undef, $volid) = PVE::Storage::path_to_volume_id($cfg, $archive);
6186         if (defined($volid)) {
6187             my ($sid, undef) = PVE::Storage::parse_volume_id($volid);
6188             my $readlimit = PVE::Storage::get_bandwidth_limit('restore', [$sid], $bwlimit);
6189             if ($readlimit) {
6190                 print STDERR "applying read rate limit: $readlimit\n";
6191                 my $cstream = ['cstream', '-t', $readlimit*1024, '--', $readfrom];
6192                 $add_pipe->($cstream);
6193             }
6194         }
6195     }
6196
6197     if ($comp) {
6198         my $cmd;
6199         if ($comp eq 'gzip') {
6200             $cmd = ['zcat', $readfrom];
6201         } elsif ($comp eq 'lzop') {
6202             $cmd = ['lzop', '-d', '-c', $readfrom];
6203         } else {
6204             die "unknown compression method '$comp'\n";
6205         }
6206         $add_pipe->($cmd);
6207     }
6208
6209     my $tmpdir = "/var/tmp/vzdumptmp$$";
6210     rmtree $tmpdir;
6211
6212     # disable interrupts (always do cleanups)
6213     local $SIG{INT} =
6214         local $SIG{TERM} =
6215         local $SIG{QUIT} =
6216         local $SIG{HUP} = sub { warn "got interrupt - ignored\n"; };
6217
6218     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
6219     POSIX::mkfifo($mapfifo, 0600);
6220     my $fifofh;
6221
6222     my $openfifo = sub {
6223         open($fifofh, '>', $mapfifo) || die $!;
6224     };
6225
6226     $add_pipe->(['vma', 'extract', '-v', '-r', $mapfifo, $readfrom, $tmpdir]);
6227
6228     my $oldtimeout;
6229     my $timeout = 5;
6230
6231     my $devinfo = {};
6232
6233     my $rpcenv = PVE::RPCEnvironment::get();
6234
6235     my $conffile = PVE::QemuConfig->config_file($vmid);
6236     my $tmpfn = "$conffile.$$.tmp";
6237
6238     # Note: $oldconf is undef if VM does not exists
6239     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
6240     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
6241
6242     my %storage_limits;
6243
6244     my $print_devmap = sub {
6245         my $virtdev_hash = {};
6246
6247         my $cfgfn = "$tmpdir/qemu-server.conf";
6248
6249         # we can read the config - that is already extracted
6250         my $fh = IO::File->new($cfgfn, "r") ||
6251             "unable to read qemu-server.conf - $!\n";
6252
6253         my $fwcfgfn = "$tmpdir/qemu-server.fw";
6254         if (-f $fwcfgfn) {
6255             my $pve_firewall_dir = '/etc/pve/firewall';
6256             mkdir $pve_firewall_dir; # make sure the dir exists
6257             PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
6258         }
6259
6260         while (defined(my $line = <$fh>)) {
6261             if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
6262                 my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
6263                 die "archive does not contain data for drive '$virtdev'\n"
6264                     if !$devinfo->{$devname};
6265                 if (defined($opts->{storage})) {
6266                     $storeid = $opts->{storage} || 'local';
6267                 } elsif (!$storeid) {
6268                     $storeid = 'local';
6269                 }
6270                 $format = 'raw' if !$format;
6271                 $devinfo->{$devname}->{devname} = $devname;
6272                 $devinfo->{$devname}->{virtdev} = $virtdev;
6273                 $devinfo->{$devname}->{format} = $format;
6274                 $devinfo->{$devname}->{storeid} = $storeid;
6275
6276                 # check permission on storage
6277                 my $pool = $opts->{pool}; # todo: do we need that?
6278                 if ($user ne 'root@pam') {
6279                     $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
6280                 }
6281
6282                 $storage_limits{$storeid} = $bwlimit;
6283
6284                 $virtdev_hash->{$virtdev} = $devinfo->{$devname};
6285             } elsif ($line =~ m/^((?:ide|sata|scsi)\d+):\s*(.*)\s*$/) {
6286                 my $virtdev = $1;
6287                 my $drive = parse_drive($virtdev, $2);
6288                 if (drive_is_cloudinit($drive)) {
6289                     my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6290                     my $scfg = PVE::Storage::storage_config($cfg, $storeid);
6291                     my $format = qemu_img_format($scfg, $volname); # has 'raw' fallback
6292
6293                     my $d = {
6294                         format => $format,
6295                         storeid => $opts->{storage} // $storeid,
6296                         size => PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE,
6297                         file => $drive->{file}, # to make drive_is_cloudinit check possible
6298                         name => "vm-$vmid-cloudinit",
6299                         is_cloudinit => 1,
6300                     };
6301                     $virtdev_hash->{$virtdev} = $d;
6302                 }
6303             }
6304         }
6305
6306         foreach my $key (keys %storage_limits) {
6307             my $limit = PVE::Storage::get_bandwidth_limit('restore', [$key], $bwlimit);
6308             next if !$limit;
6309             print STDERR "rate limit for storage $key: $limit KiB/s\n";
6310             $storage_limits{$key} = $limit * 1024;
6311         }
6312
6313         foreach my $devname (keys %$devinfo) {
6314             die "found no device mapping information for device '$devname'\n"
6315                 if !$devinfo->{$devname}->{virtdev};
6316         }
6317
6318         # create empty/temp config
6319         if ($oldconf) {
6320             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
6321             foreach_drive($oldconf, sub {
6322                 my ($ds, $drive) = @_;
6323
6324                 return if drive_is_cdrom($drive, 1);
6325
6326                 my $volid = $drive->{file};
6327                 return if !$volid || $volid =~ m|^/|;
6328
6329                 my ($path, $owner) = PVE::Storage::path($cfg, $volid);
6330                 return if !$path || !$owner || ($owner != $vmid);
6331
6332                 # Note: only delete disk we want to restore
6333                 # other volumes will become unused
6334                 if ($virtdev_hash->{$ds}) {
6335                     eval { PVE::Storage::vdisk_free($cfg, $volid); };
6336                     if (my $err = $@) {
6337                         warn $err;
6338                     }
6339                 }
6340             });
6341
6342             # delete vmstate files, after the restore we have no snapshots anymore
6343             foreach my $snapname (keys %{$oldconf->{snapshots}}) {
6344                 my $snap = $oldconf->{snapshots}->{$snapname};
6345                 if ($snap->{vmstate}) {
6346                     eval { PVE::Storage::vdisk_free($cfg, $snap->{vmstate}); };
6347                     if (my $err = $@) {
6348                         warn $err;
6349                     }
6350                 }
6351             }
6352         }
6353
6354         my $map = {};
6355         foreach my $virtdev (sort keys %$virtdev_hash) {
6356             my $d = $virtdev_hash->{$virtdev};
6357             my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
6358             my $storeid = $d->{storeid};
6359             my $scfg = PVE::Storage::storage_config($cfg, $storeid);
6360
6361             my $map_opts = '';
6362             if (my $limit = $storage_limits{$storeid}) {
6363                 $map_opts .= "throttling.bps=$limit:throttling.group=$storeid:";
6364             }
6365
6366             # test if requested format is supported
6367             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($cfg, $storeid);
6368             my $supported = grep { $_ eq $d->{format} } @$validFormats;
6369             $d->{format} = $defFormat if !$supported;
6370
6371             my $name;
6372             if ($d->{is_cloudinit}) {
6373                 $name = $d->{name};
6374                 $name .= ".$d->{format}" if $d->{format} ne 'raw';
6375             }
6376
6377             my $volid = PVE::Storage::vdisk_alloc($cfg, $storeid, $vmid, $d->{format}, $name, $alloc_size);
6378             print STDERR "new volume ID is '$volid'\n";
6379             $d->{volid} = $volid;
6380
6381             PVE::Storage::activate_volumes($cfg, [$volid]);
6382
6383             my $write_zeros = 1;
6384             if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
6385                 $write_zeros = 0;
6386             }
6387
6388             if (!$d->{is_cloudinit}) {
6389                 my $path = PVE::Storage::path($cfg, $volid);
6390
6391                 print $fifofh "${map_opts}format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
6392
6393                 print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
6394             }
6395             $map->{$virtdev} = $volid;
6396         }
6397
6398         $fh->seek(0, 0) || die "seek failed - $!\n";
6399
6400         my $outfd = new IO::File ($tmpfn, "w") ||
6401             die "unable to write config for VM $vmid\n";
6402
6403         my $cookie = { netcount => 0 };
6404         while (defined(my $line = <$fh>)) {
6405             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
6406         }
6407
6408         $fh->close();
6409         $outfd->close();
6410     };
6411
6412     eval {
6413         # enable interrupts
6414         local $SIG{INT} =
6415             local $SIG{TERM} =
6416             local $SIG{QUIT} =
6417             local $SIG{HUP} =
6418             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6419         local $SIG{ALRM} = sub { die "got timeout\n"; };
6420
6421         $oldtimeout = alarm($timeout);
6422
6423         my $parser = sub {
6424             my $line = shift;
6425
6426             print "$line\n";
6427
6428             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
6429                 my ($dev_id, $size, $devname) = ($1, $2, $3);
6430                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
6431             } elsif ($line =~ m/^CTIME: /) {
6432                 # we correctly received the vma config, so we can disable
6433                 # the timeout now for disk allocation (set to 10 minutes, so
6434                 # that we always timeout if something goes wrong)
6435                 alarm(600);
6436                 &$print_devmap();
6437                 print $fifofh "done\n";
6438                 my $tmp = $oldtimeout || 0;
6439                 $oldtimeout = undef;
6440                 alarm($tmp);
6441                 close($fifofh);
6442             }
6443         };
6444
6445         print "restore vma archive: $dbg_cmdstring\n";
6446         run_command($commands, input => $input, outfunc => $parser, afterfork => $openfifo);
6447     };
6448     my $err = $@;
6449
6450     alarm($oldtimeout) if $oldtimeout;
6451
6452     my $vollist = [];
6453     foreach my $devname (keys %$devinfo) {
6454         my $volid = $devinfo->{$devname}->{volid};
6455         push @$vollist, $volid if $volid;
6456     }
6457
6458     PVE::Storage::deactivate_volumes($cfg, $vollist);
6459
6460     unlink $mapfifo;
6461
6462     if ($err) {
6463         rmtree $tmpdir;
6464         unlink $tmpfn;
6465
6466         foreach my $devname (keys %$devinfo) {
6467             my $volid = $devinfo->{$devname}->{volid};
6468             next if !$volid;
6469             eval {
6470                 if ($volid =~ m|^/|) {
6471                     unlink $volid || die 'unlink failed\n';
6472                 } else {
6473                     PVE::Storage::vdisk_free($cfg, $volid);
6474                 }
6475                 print STDERR "temporary volume '$volid' sucessfuly removed\n";
6476             };
6477             print STDERR "unable to cleanup '$volid' - $@" if $@;
6478         }
6479         die $err;
6480     }
6481
6482     rmtree $tmpdir;
6483
6484     rename($tmpfn, $conffile) ||
6485         die "unable to commit configuration file '$conffile'\n";
6486
6487     PVE::Cluster::cfs_update(); # make sure we read new file
6488
6489     eval { rescan($vmid, 1); };
6490     warn $@ if $@;
6491 }
6492
6493 sub restore_tar_archive {
6494     my ($archive, $vmid, $user, $opts) = @_;
6495
6496     if ($archive ne '-') {
6497         my $firstfile = tar_archive_read_firstfile($archive);
6498         die "ERROR: file '$archive' dos not lock like a QemuServer vzdump backup\n"
6499             if $firstfile ne 'qemu-server.conf';
6500     }
6501
6502     my $storecfg = PVE::Storage::config();
6503
6504     # avoid zombie disks when restoring over an existing VM -> cleanup first
6505     # pass keep_empty_config=1 to keep the config (thus VMID) reserved for us
6506     # skiplock=1 because qmrestore has set the 'create' lock itself already
6507     my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
6508     destroy_vm($storecfg, $vmid, 1, { lock => 'restore' }) if -f $vmcfgfn;
6509
6510     my $tocmd = "/usr/lib/qemu-server/qmextract";
6511
6512     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
6513     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
6514     $tocmd .= ' --prealloc' if $opts->{prealloc};
6515     $tocmd .= ' --info' if $opts->{info};
6516
6517     # tar option "xf" does not autodetect compression when read from STDIN,
6518     # so we pipe to zcat
6519     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
6520         PVE::Tools::shellquote("--to-command=$tocmd");
6521
6522     my $tmpdir = "/var/tmp/vzdumptmp$$";
6523     mkpath $tmpdir;
6524
6525     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
6526     local $ENV{VZDUMP_VMID} = $vmid;
6527     local $ENV{VZDUMP_USER} = $user;
6528
6529     my $conffile = PVE::QemuConfig->config_file($vmid);
6530     my $tmpfn = "$conffile.$$.tmp";
6531
6532     # disable interrupts (always do cleanups)
6533     local $SIG{INT} =
6534         local $SIG{TERM} =
6535         local $SIG{QUIT} =
6536         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
6537
6538     eval {
6539         # enable interrupts
6540         local $SIG{INT} =
6541             local $SIG{TERM} =
6542             local $SIG{QUIT} =
6543             local $SIG{HUP} =
6544             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6545
6546         if ($archive eq '-') {
6547             print "extracting archive from STDIN\n";
6548             run_command($cmd, input => "<&STDIN");
6549         } else {
6550             print "extracting archive '$archive'\n";
6551             run_command($cmd);
6552         }
6553
6554         return if $opts->{info};
6555
6556         # read new mapping
6557         my $map = {};
6558         my $statfile = "$tmpdir/qmrestore.stat";
6559         if (my $fd = IO::File->new($statfile, "r")) {
6560             while (defined (my $line = <$fd>)) {
6561                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
6562                     $map->{$1} = $2 if $1;
6563                 } else {
6564                     print STDERR "unable to parse line in statfile - $line\n";
6565                 }
6566             }
6567             $fd->close();
6568         }
6569
6570         my $confsrc = "$tmpdir/qemu-server.conf";
6571
6572         my $srcfd = new IO::File($confsrc, "r") ||
6573             die "unable to open file '$confsrc'\n";
6574
6575         my $outfd = new IO::File ($tmpfn, "w") ||
6576             die "unable to write config for VM $vmid\n";
6577
6578         my $cookie = { netcount => 0 };
6579         while (defined (my $line = <$srcfd>)) {
6580             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
6581         }
6582
6583         $srcfd->close();
6584         $outfd->close();
6585     };
6586     if (my $err = $@) {
6587         unlink $tmpfn;
6588         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
6589         die $err;
6590     }
6591
6592     rmtree $tmpdir;
6593
6594     rename $tmpfn, $conffile ||
6595         die "unable to commit configuration file '$conffile'\n";
6596
6597     PVE::Cluster::cfs_update(); # make sure we read new file
6598
6599     eval { rescan($vmid, 1); };
6600     warn $@ if $@;
6601 };
6602
6603 sub foreach_storage_used_by_vm {
6604     my ($conf, $func) = @_;
6605
6606     my $sidhash = {};
6607
6608     foreach_drive($conf, sub {
6609         my ($ds, $drive) = @_;
6610         return if drive_is_cdrom($drive);
6611
6612         my $volid = $drive->{file};
6613
6614         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
6615         $sidhash->{$sid} = $sid if $sid;
6616     });
6617
6618     foreach my $sid (sort keys %$sidhash) {
6619         &$func($sid);
6620     }
6621 }
6622
6623 my $qemu_snap_storage = {
6624     rbd => 1,
6625 };
6626 sub do_snapshots_with_qemu {
6627     my ($storecfg, $volid) = @_;
6628
6629     my $storage_name = PVE::Storage::parse_volume_id($volid);
6630     my $scfg = $storecfg->{ids}->{$storage_name};
6631
6632     if ($qemu_snap_storage->{$scfg->{type}} && !$scfg->{krbd}){
6633         return 1;
6634     }
6635
6636     if ($volid =~ m/\.(qcow2|qed)$/){
6637         return 1;
6638     }
6639
6640     return undef;
6641 }
6642
6643 sub qga_check_running {
6644     my ($vmid, $nowarn) = @_;
6645
6646     eval { mon_cmd($vmid, "guest-ping", timeout => 3); };
6647     if ($@) {
6648         warn "Qemu Guest Agent is not running - $@" if !$nowarn;
6649         return 0;
6650     }
6651     return 1;
6652 }
6653
6654 sub template_create {
6655     my ($vmid, $conf, $disk) = @_;
6656
6657     my $storecfg = PVE::Storage::config();
6658
6659     foreach_drive($conf, sub {
6660         my ($ds, $drive) = @_;
6661
6662         return if drive_is_cdrom($drive);
6663         return if $disk && $ds ne $disk;
6664
6665         my $volid = $drive->{file};
6666         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
6667
6668         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
6669         $drive->{file} = $voliddst;
6670         $conf->{$ds} = print_drive($vmid, $drive);
6671         PVE::QemuConfig->write_config($vmid, $conf);
6672     });
6673 }
6674
6675 sub convert_iscsi_path {
6676     my ($path) = @_;
6677
6678     if ($path =~ m|^iscsi://([^/]+)/([^/]+)/(.+)$|) {
6679         my $portal = $1;
6680         my $target = $2;
6681         my $lun = $3;
6682
6683         my $initiator_name = get_initiator_name();
6684
6685         return "file.driver=iscsi,file.transport=tcp,file.initiator-name=$initiator_name,".
6686                "file.portal=$portal,file.target=$target,file.lun=$lun,driver=raw";
6687     }
6688
6689     die "cannot convert iscsi path '$path', unkown format\n";
6690 }
6691
6692 sub qemu_img_convert {
6693     my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized) = @_;
6694
6695     my $storecfg = PVE::Storage::config();
6696     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
6697     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
6698
6699     die "destination '$dst_volid' is not a valid volid form qemu-img convert\n" if !$dst_storeid;
6700
6701     my $cachemode;
6702     my $src_path;
6703     my $src_is_iscsi = 0;
6704     my $src_format = 'raw';
6705
6706     if ($src_storeid) {
6707         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
6708         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
6709         $src_format = qemu_img_format($src_scfg, $src_volname);
6710         $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
6711         $src_is_iscsi = ($src_path =~ m|^iscsi://|);
6712         $cachemode = 'none' if $src_scfg->{type} eq 'zfspool';
6713     } elsif (-f $src_volid) {
6714         $src_path = $src_volid;
6715         if ($src_path =~ m/\.($QEMU_FORMAT_RE)$/) {
6716             $src_format = $1;
6717         }
6718     }
6719
6720     die "source '$src_volid' is not a valid volid nor path for qemu-img convert\n" if !$src_path;
6721
6722     my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6723     my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
6724     my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6725     my $dst_is_iscsi = ($dst_path =~ m|^iscsi://|);
6726
6727     my $cmd = [];
6728     push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
6729     push @$cmd, '-l', "snapshot.name=$snapname" if($snapname && $src_format eq "qcow2");
6730     push @$cmd, '-t', 'none' if $dst_scfg->{type} eq 'zfspool';
6731     push @$cmd, '-T', $cachemode if defined($cachemode);
6732
6733     if ($src_is_iscsi) {
6734         push @$cmd, '--image-opts';
6735         $src_path = convert_iscsi_path($src_path);
6736     } else {
6737         push @$cmd, '-f', $src_format;
6738     }
6739
6740     if ($dst_is_iscsi) {
6741         push @$cmd, '--target-image-opts';
6742         $dst_path = convert_iscsi_path($dst_path);
6743     } else {
6744         push @$cmd, '-O', $dst_format;
6745     }
6746
6747     push @$cmd, $src_path;
6748
6749     if (!$dst_is_iscsi && $is_zero_initialized) {
6750         push @$cmd, "zeroinit:$dst_path";
6751     } else {
6752         push @$cmd, $dst_path;
6753     }
6754
6755     my $parser = sub {
6756         my $line = shift;
6757         if($line =~ m/\((\S+)\/100\%\)/){
6758             my $percent = $1;
6759             my $transferred = int($size * $percent / 100);
6760             my $remaining = $size - $transferred;
6761
6762             print "transferred: $transferred bytes remaining: $remaining bytes total: $size bytes progression: $percent %\n";
6763         }
6764
6765     };
6766
6767     eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
6768     my $err = $@;
6769     die "copy failed: $err" if $err;
6770 }
6771
6772 sub qemu_img_format {
6773     my ($scfg, $volname) = @_;
6774
6775     if ($scfg->{path} && $volname =~ m/\.($QEMU_FORMAT_RE)$/) {
6776         return $1;
6777     } else {
6778         return "raw";
6779     }
6780 }
6781
6782 sub qemu_drive_mirror {
6783     my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $skipcomplete, $qga, $bwlimit) = @_;
6784
6785     $jobs = {} if !$jobs;
6786
6787     my $qemu_target;
6788     my $format;
6789     $jobs->{"drive-$drive"} = {};
6790
6791     if ($dst_volid =~ /^nbd:/) {
6792         $qemu_target = $dst_volid;
6793         $format = "nbd";
6794     } else {
6795         my $storecfg = PVE::Storage::config();
6796         my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
6797
6798         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6799
6800         $format = qemu_img_format($dst_scfg, $dst_volname);
6801
6802         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6803
6804         $qemu_target = $is_zero_initialized ? "zeroinit:$dst_path" : $dst_path;
6805     }
6806
6807     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $qemu_target };
6808     $opts->{format} = $format if $format;
6809
6810     if (defined($bwlimit)) {
6811         $opts->{speed} = $bwlimit * 1024;
6812         print "drive mirror is starting for drive-$drive with bandwidth limit: ${bwlimit} KB/s\n";
6813     } else {
6814         print "drive mirror is starting for drive-$drive\n";
6815     }
6816
6817     # if a job already runs for this device we get an error, catch it for cleanup
6818     eval { mon_cmd($vmid, "drive-mirror", %$opts); };
6819     if (my $err = $@) {
6820         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6821         warn "$@\n" if $@;
6822         die "mirroring error: $err\n";
6823     }
6824
6825     qemu_drive_mirror_monitor ($vmid, $vmiddst, $jobs, $skipcomplete, $qga);
6826 }
6827
6828 sub qemu_drive_mirror_monitor {
6829     my ($vmid, $vmiddst, $jobs, $skipcomplete, $qga) = @_;
6830
6831     eval {
6832         my $err_complete = 0;
6833
6834         while (1) {
6835             die "storage migration timed out\n" if $err_complete > 300;
6836
6837             my $stats = mon_cmd($vmid, "query-block-jobs");
6838
6839             my $running_mirror_jobs = {};
6840             foreach my $stat (@$stats) {
6841                 next if $stat->{type} ne 'mirror';
6842                 $running_mirror_jobs->{$stat->{device}} = $stat;
6843             }
6844
6845             my $readycounter = 0;
6846
6847             foreach my $job (keys %$jobs) {
6848
6849                 if(defined($jobs->{$job}->{complete}) && !defined($running_mirror_jobs->{$job})) {
6850                     print "$job : finished\n";
6851                     delete $jobs->{$job};
6852                     next;
6853                 }
6854
6855                 die "$job: mirroring has been cancelled\n" if !defined($running_mirror_jobs->{$job});
6856
6857                 my $busy = $running_mirror_jobs->{$job}->{busy};
6858                 my $ready = $running_mirror_jobs->{$job}->{ready};
6859                 if (my $total = $running_mirror_jobs->{$job}->{len}) {
6860                     my $transferred = $running_mirror_jobs->{$job}->{offset} || 0;
6861                     my $remaining = $total - $transferred;
6862                     my $percent = sprintf "%.2f", ($transferred * 100 / $total);
6863
6864                     print "$job: transferred: $transferred bytes remaining: $remaining bytes total: $total bytes progression: $percent % busy: $busy ready: $ready \n";
6865                 }
6866
6867                 $readycounter++ if $running_mirror_jobs->{$job}->{ready};
6868             }
6869
6870             last if scalar(keys %$jobs) == 0;
6871
6872             if ($readycounter == scalar(keys %$jobs)) {
6873                 print "all mirroring jobs are ready \n";
6874                 last if $skipcomplete; #do the complete later
6875
6876                 if ($vmiddst && $vmiddst != $vmid) {
6877                     my $agent_running = $qga && qga_check_running($vmid);
6878                     if ($agent_running) {
6879                         print "freeze filesystem\n";
6880                         eval { mon_cmd($vmid, "guest-fsfreeze-freeze"); };
6881                     } else {
6882                         print "suspend vm\n";
6883                         eval { PVE::QemuServer::vm_suspend($vmid, 1); };
6884                     }
6885
6886                     # if we clone a disk for a new target vm, we don't switch the disk
6887                     PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs);
6888
6889                     if ($agent_running) {
6890                         print "unfreeze filesystem\n";
6891                         eval { mon_cmd($vmid, "guest-fsfreeze-thaw"); };
6892                     } else {
6893                         print "resume vm\n";
6894                         eval {  PVE::QemuServer::vm_resume($vmid, 1, 1); };
6895                     }
6896
6897                     last;
6898                 } else {
6899
6900                     foreach my $job (keys %$jobs) {
6901                         # try to switch the disk if source and destination are on the same guest
6902                         print "$job: Completing block job...\n";
6903
6904                         eval { mon_cmd($vmid, "block-job-complete", device => $job) };
6905                         if ($@ =~ m/cannot be completed/) {
6906                             print "$job: Block job cannot be completed, try again.\n";
6907                             $err_complete++;
6908                         }else {
6909                             print "$job: Completed successfully.\n";
6910                             $jobs->{$job}->{complete} = 1;
6911                         }
6912                     }
6913                 }
6914             }
6915             sleep 1;
6916         }
6917     };
6918     my $err = $@;
6919
6920     if ($err) {
6921         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6922         die "mirroring error: $err";
6923     }
6924
6925 }
6926
6927 sub qemu_blockjobs_cancel {
6928     my ($vmid, $jobs) = @_;
6929
6930     foreach my $job (keys %$jobs) {
6931         print "$job: Cancelling block job\n";
6932         eval { mon_cmd($vmid, "block-job-cancel", device => $job); };
6933         $jobs->{$job}->{cancel} = 1;
6934     }
6935
6936     while (1) {
6937         my $stats = mon_cmd($vmid, "query-block-jobs");
6938
6939         my $running_jobs = {};
6940         foreach my $stat (@$stats) {
6941             $running_jobs->{$stat->{device}} = $stat;
6942         }
6943
6944         foreach my $job (keys %$jobs) {
6945
6946             if (defined($jobs->{$job}->{cancel}) && !defined($running_jobs->{$job})) {
6947                 print "$job: Done.\n";
6948                 delete $jobs->{$job};
6949             }
6950         }
6951
6952         last if scalar(keys %$jobs) == 0;
6953
6954         sleep 1;
6955     }
6956 }
6957
6958 sub clone_disk {
6959     my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
6960         $newvmid, $storage, $format, $full, $newvollist, $jobs, $skipcomplete, $qga, $bwlimit) = @_;
6961
6962     my $newvolid;
6963
6964     if (!$full) {
6965         print "create linked clone of drive $drivename ($drive->{file})\n";
6966         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
6967         push @$newvollist, $newvolid;
6968     } else {
6969
6970         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6971         $storeid = $storage if $storage;
6972
6973         my $dst_format = resolve_dst_disk_format($storecfg, $storeid, $volname, $format);
6974         my ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 3);
6975
6976         print "create full clone of drive $drivename ($drive->{file})\n";
6977         my $name = undef;
6978         if (drive_is_cloudinit($drive)) {
6979             $name = "vm-$newvmid-cloudinit";
6980             $name .= ".$dst_format" if $dst_format ne 'raw';
6981             $snapname = undef;
6982             $size = PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE;
6983         }
6984         $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $newvmid, $dst_format, $name, ($size/1024));
6985         push @$newvollist, $newvolid;
6986
6987         PVE::Storage::activate_volumes($storecfg, [$newvolid]);
6988
6989         if (drive_is_cloudinit($drive)) {
6990             goto no_data_clone;
6991         }
6992
6993         my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
6994         if (!$running || $snapname) {
6995             # TODO: handle bwlimits
6996             qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit);
6997         } else {
6998
6999             my $kvmver = get_running_qemu_version ($vmid);
7000             if (!min_version($kvmver, 2, 7)) {
7001                 die "drive-mirror with iothread requires qemu version 2.7 or higher\n"
7002                     if $drive->{iothread};
7003             }
7004
7005             qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid, $sparseinit, $jobs, $skipcomplete, $qga, $bwlimit);
7006         }
7007     }
7008
7009 no_data_clone:
7010     my ($size) = PVE::Storage::volume_size_info($storecfg, $newvolid, 3);
7011
7012     my $disk = $drive;
7013     $disk->{format} = undef;
7014     $disk->{file} = $newvolid;
7015     $disk->{size} = $size;
7016
7017     return $disk;
7018 }
7019
7020 sub get_running_qemu_version {
7021     my ($vmid) = @_;
7022     my $res = mon_cmd($vmid, "query-version");
7023     return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
7024 }
7025
7026 sub qemu_use_old_bios_files {
7027     my ($machine_type) = @_;
7028
7029     return if !$machine_type;
7030
7031     my $use_old_bios_files = undef;
7032
7033     if ($machine_type =~ m/^(\S+)\.pxe$/) {
7034         $machine_type = $1;
7035         $use_old_bios_files = 1;
7036     } else {
7037         my $version = PVE::QemuServer::Machine::extract_version($machine_type) // kvm_user_version();
7038         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
7039         # load new efi bios files on migration. So this hack is required to allow
7040         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
7041         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
7042         $use_old_bios_files = !min_version($version, 2, 4);
7043     }
7044
7045     return ($use_old_bios_files, $machine_type);
7046 }
7047
7048 sub create_efidisk($$$$$) {
7049     my ($storecfg, $storeid, $vmid, $fmt, $arch) = @_;
7050
7051     my (undef, $ovmf_vars) = get_ovmf_files($arch);
7052     die "EFI vars default image not found\n" if ! -f $ovmf_vars;
7053
7054     my $vars_size_b = -s $ovmf_vars;
7055     my $vars_size = PVE::Tools::convert_size($vars_size_b, 'b' => 'kb');
7056     my $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $fmt, undef, $vars_size);
7057     PVE::Storage::activate_volumes($storecfg, [$volid]);
7058
7059     qemu_img_convert($ovmf_vars, $volid, $vars_size_b, undef, 0);
7060
7061     return ($volid, $vars_size);
7062 }
7063
7064 sub vm_iothreads_list {
7065     my ($vmid) = @_;
7066
7067     my $res = mon_cmd($vmid, 'query-iothreads');
7068
7069     my $iothreads = {};
7070     foreach my $iothread (@$res) {
7071         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
7072     }
7073
7074     return $iothreads;
7075 }
7076
7077 sub scsihw_infos {
7078     my ($conf, $drive) = @_;
7079
7080     my $maxdev = 0;
7081
7082     if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)) {
7083         $maxdev = 7;
7084     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
7085         $maxdev = 1;
7086     } else {
7087         $maxdev = 256;
7088     }
7089
7090     my $controller = int($drive->{index} / $maxdev);
7091     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single') ? "virtioscsi" : "scsihw";
7092
7093     return ($maxdev, $controller, $controller_prefix);
7094 }
7095
7096 sub add_hyperv_enlightenments {
7097     my ($cpuFlags, $winversion, $machine_version, $bios, $gpu_passthrough, $hv_vendor_id) = @_;
7098
7099     return if $winversion < 6;
7100     return if $bios && $bios eq 'ovmf' && $winversion < 8;
7101
7102     if ($gpu_passthrough || defined($hv_vendor_id)) {
7103         $hv_vendor_id //= 'proxmox';
7104         push @$cpuFlags , "hv_vendor_id=$hv_vendor_id";
7105     }
7106
7107     if (min_version($machine_version, 2, 3)) {
7108         push @$cpuFlags , 'hv_spinlocks=0x1fff';
7109         push @$cpuFlags , 'hv_vapic';
7110         push @$cpuFlags , 'hv_time';
7111     } else {
7112         push @$cpuFlags , 'hv_spinlocks=0xffff';
7113     }
7114
7115     if (min_version($machine_version, 2, 6)) {
7116         push @$cpuFlags , 'hv_reset';
7117         push @$cpuFlags , 'hv_vpindex';
7118         push @$cpuFlags , 'hv_runtime';
7119     }
7120
7121     if ($winversion >= 7) {
7122         push @$cpuFlags , 'hv_relaxed';
7123
7124         if (min_version($machine_version, 2, 12)) {
7125             push @$cpuFlags , 'hv_synic';
7126             push @$cpuFlags , 'hv_stimer';
7127         }
7128
7129         if (min_version($machine_version, 3, 1)) {
7130             push @$cpuFlags , 'hv_ipi';
7131         }
7132     }
7133 }
7134
7135 sub windows_version {
7136     my ($ostype) = @_;
7137
7138     return 0 if !$ostype;
7139
7140     my $winversion = 0;
7141
7142     if($ostype eq 'wxp' || $ostype eq 'w2k3' || $ostype eq 'w2k') {
7143         $winversion = 5;
7144     } elsif($ostype eq 'w2k8' || $ostype eq 'wvista') {
7145         $winversion = 6;
7146     } elsif ($ostype =~ m/^win(\d+)$/) {
7147         $winversion = $1;
7148     }
7149
7150     return $winversion;
7151 }
7152
7153 sub resolve_dst_disk_format {
7154         my ($storecfg, $storeid, $src_volname, $format) = @_;
7155         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
7156
7157         if (!$format) {
7158             # if no target format is specified, use the source disk format as hint
7159             if ($src_volname) {
7160                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
7161                 $format = qemu_img_format($scfg, $src_volname);
7162             } else {
7163                 return $defFormat;
7164             }
7165         }
7166
7167         # test if requested format is supported - else use default
7168         my $supported = grep { $_ eq $format } @$validFormats;
7169         $format = $defFormat if !$supported;
7170         return $format;
7171 }
7172
7173 sub resolve_first_disk {
7174     my $conf = shift;
7175     my @disks = PVE::QemuServer::valid_drive_names();
7176     my $firstdisk;
7177     foreach my $ds (reverse @disks) {
7178         next if !$conf->{$ds};
7179         my $disk = PVE::QemuServer::parse_drive($ds, $conf->{$ds});
7180         next if PVE::QemuServer::drive_is_cdrom($disk);
7181         $firstdisk = $ds;
7182     }
7183     return $firstdisk;
7184 }
7185
7186 sub generate_uuid {
7187     my ($uuid, $uuid_str);
7188     UUID::generate($uuid);
7189     UUID::unparse($uuid, $uuid_str);
7190     return $uuid_str;
7191 }
7192
7193 sub generate_smbios1_uuid {
7194     return "uuid=".generate_uuid();
7195 }
7196
7197 sub nbd_stop {
7198     my ($vmid) = @_;
7199
7200     mon_cmd($vmid, 'nbd-server-stop');
7201 }
7202
7203 sub create_reboot_request {
7204     my ($vmid) = @_;
7205     open(my $fh, '>', "/run/qemu-server/$vmid.reboot")
7206         or die "failed to create reboot trigger file: $!\n";
7207     close($fh);
7208 }
7209
7210 sub clear_reboot_request {
7211     my ($vmid) = @_;
7212     my $path = "/run/qemu-server/$vmid.reboot";
7213     my $res = 0;
7214
7215     $res = unlink($path);
7216     die "could not remove reboot request for $vmid: $!"
7217         if !$res && $! != POSIX::ENOENT;
7218
7219     return $res;
7220 }
7221
7222 # bash completion helper
7223
7224 sub complete_backup_archives {
7225     my ($cmdname, $pname, $cvalue) = @_;
7226
7227     my $cfg = PVE::Storage::config();
7228
7229     my $storeid;
7230
7231     if ($cvalue =~ m/^([^:]+):/) {
7232         $storeid = $1;
7233     }
7234
7235     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
7236
7237     my $res = [];
7238     foreach my $id (keys %$data) {
7239         foreach my $item (@{$data->{$id}}) {
7240             next if $item->{format} !~ m/^vma\.(gz|lzo)$/;
7241             push @$res, $item->{volid} if defined($item->{volid});
7242         }
7243     }
7244
7245     return $res;
7246 }
7247
7248 my $complete_vmid_full = sub {
7249     my ($running) = @_;
7250
7251     my $idlist = vmstatus();
7252
7253     my $res = [];
7254
7255     foreach my $id (keys %$idlist) {
7256         my $d = $idlist->{$id};
7257         if (defined($running)) {
7258             next if $d->{template};
7259             next if $running && $d->{status} ne 'running';
7260             next if !$running && $d->{status} eq 'running';
7261         }
7262         push @$res, $id;
7263
7264     }
7265     return $res;
7266 };
7267
7268 sub complete_vmid {
7269     return &$complete_vmid_full();
7270 }
7271
7272 sub complete_vmid_stopped {
7273     return &$complete_vmid_full(0);
7274 }
7275
7276 sub complete_vmid_running {
7277     return &$complete_vmid_full(1);
7278 }
7279
7280 sub complete_storage {
7281
7282     my $cfg = PVE::Storage::config();
7283     my $ids = $cfg->{ids};
7284
7285     my $res = [];
7286     foreach my $sid (keys %$ids) {
7287         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
7288         next if !$ids->{$sid}->{content}->{images};
7289         push @$res, $sid;
7290     }
7291
7292     return $res;
7293 }
7294
7295 sub complete_migration_storage {
7296     my ($cmd, $param, $current_value, $all_args) = @_;
7297
7298     my $targetnode = @$all_args[1];
7299
7300     my $cfg = PVE::Storage::config();
7301     my $ids = $cfg->{ids};
7302
7303     my $res = [];
7304     foreach my $sid (keys %$ids) {
7305         next if !PVE::Storage::storage_check_enabled($cfg, $sid, $targetnode, 1);
7306         next if !$ids->{$sid}->{content}->{images};
7307         push @$res, $sid;
7308     }
7309
7310     return $res;
7311 }
7312
7313 1;