PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5
   6 use Cwd 'abs_path';
   7 use Digest::SHA;
   8 use Fcntl ':flock';
   9 use Fcntl;
  10 use File::Basename;
  11 use File::Copy qw(copy);
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use IO::Dir;
  16 use IO::File;
  17 use IO::Handle;
  18 use IO::Select;
  19 use IO::Socket::UNIX;
  20 use IPC::Open3;
  21 use JSON;
  22 use MIME::Base64;
  23 use POSIX;
  24 use Storable qw(dclone);
  25 use Time::HiRes qw(gettimeofday);
  26 use URI::Escape;
  27 use UUID;
  28
  29 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file cfs_lock_file);
  30 use PVE::DataCenterConfig;
  31 use PVE::Exception qw(raise raise_param_exc);
  32 use PVE::GuestHelpers;
  33 use PVE::INotify;
  34 use PVE::JSONSchema qw(get_standard_option);
  35 use PVE::ProcFSTools;
  36 use PVE::RPCEnvironment;
  37 use PVE::Storage;
  38 use PVE::SysFSTools;
  39 use PVE::Systemd;
  40 use PVE::Tools qw(run_command lock_file lock_file_full file_read_firstline dir_glob_foreach get_host_arch $IPV6RE);
  41
  42 use PVE::QMPClient;
  43 use PVE::QemuConfig;
  44 use PVE::QemuServer::Helpers qw(min_version);
  45 use PVE::QemuServer::Cloudinit;
  46 use PVE::QemuServer::Machine;
  47 use PVE::QemuServer::Memory;
  48 use PVE::QemuServer::Monitor qw(mon_cmd);
  49 use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr print_pcie_root_port);
  50 use PVE::QemuServer::USB qw(parse_usb_device);
  51
  52 my $EDK2_FW_BASE = '/usr/share/pve-edk2-firmware/';
  53 my $OVMF = {
  54     x86_64 => [
  55         "$EDK2_FW_BASE/OVMF_CODE.fd",
  56         "$EDK2_FW_BASE/OVMF_VARS.fd"
  57     ],
  58     aarch64 => [
  59         "$EDK2_FW_BASE/AAVMF_CODE.fd",
  60         "$EDK2_FW_BASE/AAVMF_VARS.fd"
  61     ],
  62 };
  63
  64 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  65
  66 my $QEMU_FORMAT_RE = qr/raw|cow|qcow|qcow2|qed|vmdk|cloop/;
  67
  68 # Note about locking: we use flock on the config file protect
  69 # against concurent actions.
  70 # Aditionaly, we have a 'lock' setting in the config file. This
  71 # can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
  72 # allowed when such lock is set. But you can ignore this kind of
  73 # lock with the --skiplock flag.
  74
  75 cfs_register_file('/qemu-server/',
  76                   \&parse_vm_config,
  77                   \&write_vm_config);
  78
  79 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
  80     description => "Some command save/restore state from this location.",
  81     type => 'string',
  82     maxLength => 128,
  83     optional => 1,
  84 });
  85
  86 PVE::JSONSchema::register_standard_option('pve-qm-image-format', {
  87     type => 'string',
  88     enum => [qw(raw cow qcow qed qcow2 vmdk cloop)],
  89     description => "The drive's backing file's data format.",
  90     optional => 1,
  91 });
  92
  93 PVE::JSONSchema::register_standard_option('pve-qemu-machine', {
  94         description => "Specifies the Qemu machine type.",
  95         type => 'string',
  96         pattern => '(pc|pc(-i440fx)?-\d+(\.\d+)+(\.pxe)?|q35|pc-q35-\d+(\.\d+)+(\.pxe)?|virt(?:-\d+(\.\d+)+)?)',
  97         maxLength => 40,
  98         optional => 1,
  99 });
 100
 101 #no warnings 'redefine';
 102
 103 sub cgroups_write {
 104    my ($controller, $vmid, $option, $value) = @_;
 105
 106    my $path = "/sys/fs/cgroup/$controller/qemu.slice/$vmid.scope/$option";
 107    PVE::ProcFSTools::write_proc_entry($path, $value);
 108
 109 }
 110
 111 my $nodename = PVE::INotify::nodename();
 112
 113 my $cpu_vendor_list = {
 114     # Intel CPUs
 115     486 => 'GenuineIntel',
 116     pentium => 'GenuineIntel',
 117     pentium2  => 'GenuineIntel',
 118     pentium3  => 'GenuineIntel',
 119     coreduo => 'GenuineIntel',
 120     core2duo => 'GenuineIntel',
 121     Conroe  => 'GenuineIntel',
 122     Penryn  => 'GenuineIntel',
 123     Nehalem  => 'GenuineIntel',
 124     'Nehalem-IBRS'  => 'GenuineIntel',
 125     Westmere => 'GenuineIntel',
 126     'Westmere-IBRS' => 'GenuineIntel',
 127     SandyBridge => 'GenuineIntel',
 128     'SandyBridge-IBRS' => 'GenuineIntel',
 129     IvyBridge => 'GenuineIntel',
 130     'IvyBridge-IBRS' => 'GenuineIntel',
 131     Haswell => 'GenuineIntel',
 132     'Haswell-IBRS' => 'GenuineIntel',
 133     'Haswell-noTSX' => 'GenuineIntel',
 134     'Haswell-noTSX-IBRS' => 'GenuineIntel',
 135     Broadwell => 'GenuineIntel',
 136     'Broadwell-IBRS' => 'GenuineIntel',
 137     'Broadwell-noTSX' => 'GenuineIntel',
 138     'Broadwell-noTSX-IBRS' => 'GenuineIntel',
 139     'Skylake-Client' => 'GenuineIntel',
 140     'Skylake-Client-IBRS' => 'GenuineIntel',
 141     'Skylake-Server' => 'GenuineIntel',
 142     'Skylake-Server-IBRS' => 'GenuineIntel',
 143     'Cascadelake-Server' => 'GenuineIntel',
 144     KnightsMill => 'GenuineIntel',
 145
 146
 147     # AMD CPUs
 148     athlon => 'AuthenticAMD',
 149     phenom  => 'AuthenticAMD',
 150     Opteron_G1  => 'AuthenticAMD',
 151     Opteron_G2  => 'AuthenticAMD',
 152     Opteron_G3  => 'AuthenticAMD',
 153     Opteron_G4  => 'AuthenticAMD',
 154     Opteron_G5  => 'AuthenticAMD',
 155     EPYC => 'AuthenticAMD',
 156     'EPYC-IBPB' => 'AuthenticAMD',
 157
 158     # generic types, use vendor from host node
 159     host => 'default',
 160     kvm32 => 'default',
 161     kvm64 => 'default',
 162     qemu32 => 'default',
 163     qemu64 => 'default',
 164     max => 'default',
 165 };
 166
 167 my @supported_cpu_flags = (
 168     'pcid',
 169     'spec-ctrl',
 170     'ibpb',
 171     'ssbd',
 172     'virt-ssbd',
 173     'amd-ssbd',
 174     'amd-no-ssb',
 175     'pdpe1gb',
 176     'md-clear',
 177     'hv-tlbflush',
 178     'hv-evmcs',
 179     'aes'
 180 );
 181 my $cpu_flag = qr/[+-](@{[join('|', @supported_cpu_flags)]})/;
 182
 183 my $cpu_fmt = {
 184     cputype => {
 185         description => "Emulated CPU type.",
 186         type => 'string',
 187         enum => [ sort { "\L$a" cmp "\L$b" } keys %$cpu_vendor_list ],
 188         default => 'kvm64',
 189         default_key => 1,
 190     },
 191     hidden => {
 192         description => "Do not identify as a KVM virtual machine.",
 193         type => 'boolean',
 194         optional => 1,
 195         default => 0
 196     },
 197     'hv-vendor-id' => {
 198         type => 'string',
 199         pattern => qr/[a-zA-Z0-9]{1,12}/,
 200         format_description => 'vendor-id',
 201         description => 'The Hyper-V vendor ID. Some drivers or programs inside Windows guests need a specific ID.',
 202         optional => 1,
 203     },
 204     flags => {
 205         description => "List of additional CPU flags separated by ';'."
 206                      . " Use '+FLAG' to enable, '-FLAG' to disable a flag."
 207                      . " Currently supported flags: @{[join(', ', @supported_cpu_flags)]}.",
 208         format_description => '+FLAG[;-FLAG...]',
 209         type => 'string',
 210         pattern => qr/$cpu_flag(;$cpu_flag)*/,
 211         optional => 1,
 212     },
 213 };
 214
 215 my $watchdog_fmt = {
 216     model => {
 217         default_key => 1,
 218         type => 'string',
 219         enum => [qw(i6300esb ib700)],
 220         description => "Watchdog type to emulate.",
 221         default => 'i6300esb',
 222         optional => 1,
 223     },
 224     action => {
 225         type => 'string',
 226         enum => [qw(reset shutdown poweroff pause debug none)],
 227         description => "The action to perform if after activation the guest fails to poll the watchdog in time.",
 228         optional => 1,
 229     },
 230 };
 231 PVE::JSONSchema::register_format('pve-qm-watchdog', $watchdog_fmt);
 232
 233 my $agent_fmt = {
 234     enabled => {
 235         description => "Enable/disable Qemu GuestAgent.",
 236         type => 'boolean',
 237         default => 0,
 238         default_key => 1,
 239     },
 240     fstrim_cloned_disks => {
 241         description => "Run fstrim after cloning/moving a disk.",
 242         type => 'boolean',
 243         optional => 1,
 244         default => 0
 245     },
 246     type => {
 247         description => "Select the agent type",
 248         type => 'string',
 249         default => 'virtio',
 250         optional => 1,
 251         enum => [qw(virtio isa)],
 252     },
 253 };
 254
 255 my $vga_fmt = {
 256     type => {
 257         description => "Select the VGA type.",
 258         type => 'string',
 259         default => 'std',
 260         optional => 1,
 261         default_key => 1,
 262         enum => [qw(cirrus qxl qxl2 qxl3 qxl4 none serial0 serial1 serial2 serial3 std virtio vmware)],
 263     },
 264     memory => {
 265         description => "Sets the VGA memory (in MiB). Has no effect with serial display.",
 266         type => 'integer',
 267         optional => 1,
 268         minimum => 4,
 269         maximum => 512,
 270     },
 271 };
 272
 273 my $ivshmem_fmt = {
 274     size => {
 275         type => 'integer',
 276         minimum => 1,
 277         description => "The size of the file in MB.",
 278     },
 279     name => {
 280         type => 'string',
 281         pattern => '[a-zA-Z0-9\-]+',
 282         optional => 1,
 283         format_description => 'string',
 284         description => "The name of the file. Will be prefixed with 'pve-shm-'. Default is the VMID. Will be deleted when the VM is stopped.",
 285     },
 286 };
 287
 288 my $audio_fmt = {
 289     device => {
 290         type => 'string',
 291         enum => [qw(ich9-intel-hda intel-hda AC97)],
 292         description =>  "Configure an audio device."
 293     },
 294     driver =>  {
 295         type => 'string',
 296         enum => ['spice'],
 297         default => 'spice',
 298         optional => 1,
 299         description => "Driver backend for the audio device."
 300     },
 301 };
 302
 303 my $spice_enhancements_fmt = {
 304     foldersharing => {
 305         type => 'boolean',
 306         optional => 1,
 307         default => '0',
 308         description =>  "Enable folder sharing via SPICE. Needs Spice-WebDAV daemon installed in the VM."
 309     },
 310     videostreaming =>  {
 311         type => 'string',
 312         enum => ['off', 'all', 'filter'],
 313         default => 'off',
 314         optional => 1,
 315         description => "Enable video streaming. Uses compression for detected video streams."
 316     },
 317 };
 318
 319 my $confdesc = {
 320     onboot => {
 321         optional => 1,
 322         type => 'boolean',
 323         description => "Specifies whether a VM will be started during system bootup.",
 324         default => 0,
 325     },
 326     autostart => {
 327         optional => 1,
 328         type => 'boolean',
 329         description => "Automatic restart after crash (currently ignored).",
 330         default => 0,
 331     },
 332     hotplug => {
 333         optional => 1,
 334         type => 'string', format => 'pve-hotplug-features',
 335         description => "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
 336         default => 'network,disk,usb',
 337     },
 338     reboot => {
 339         optional => 1,
 340         type => 'boolean',
 341         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 342         default => 1,
 343     },
 344     lock => {
 345         optional => 1,
 346         type => 'string',
 347         description => "Lock/unlock the VM.",
 348         enum => [qw(backup clone create migrate rollback snapshot snapshot-delete suspending suspended)],
 349     },
 350     cpulimit => {
 351         optional => 1,
 352         type => 'number',
 353         description => "Limit of CPU usage.",
 354         verbose_description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
 355         minimum => 0,
 356         maximum => 128,
 357         default => 0,
 358     },
 359     cpuunits => {
 360         optional => 1,
 361         type => 'integer',
 362         description => "CPU weight for a VM.",
 363         verbose_description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.",
 364         minimum => 2,
 365         maximum => 262144,
 366         default => 1024,
 367     },
 368     memory => {
 369         optional => 1,
 370         type => 'integer',
 371         description => "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
 372         minimum => 16,
 373         default => 512,
 374     },
 375     balloon => {
 376         optional => 1,
 377         type => 'integer',
 378         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 379         minimum => 0,
 380     },
 381     shares => {
 382         optional => 1,
 383         type => 'integer',
 384         description => "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning. Auto-ballooning is done by pvestatd.",
 385         minimum => 0,
 386         maximum => 50000,
 387         default => 1000,
 388     },
 389     keyboard => {
 390         optional => 1,
 391         type => 'string',
 392         description => "Keybord layout for vnc server. Default is read from the '/etc/pve/datacenter.cfg' configuration file.".
 393                        "It should not be necessary to set it.",
 394         enum => PVE::Tools::kvmkeymaplist(),
 395         default => undef,
 396     },
 397     name => {
 398         optional => 1,
 399         type => 'string', format => 'dns-name',
 400         description => "Set a name for the VM. Only used on the configuration web interface.",
 401     },
 402     scsihw => {
 403         optional => 1,
 404         type => 'string',
 405         description => "SCSI controller model",
 406         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 407         default => 'lsi',
 408     },
 409     description => {
 410         optional => 1,
 411         type => 'string',
 412         description => "Description for the VM. Only used on the configuration web interface. This is saved as comment inside the configuration file.",
 413     },
 414     ostype => {
 415         optional => 1,
 416         type => 'string',
 417         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 l24 l26 solaris)],
 418         description => "Specify guest operating system.",
 419         verbose_description => <<EODESC,
 420 Specify guest operating system. This is used to enable special
 421 optimization/features for specific operating systems:
 422
 423 [horizontal]
 424 other;; unspecified OS
 425 wxp;; Microsoft Windows XP
 426 w2k;; Microsoft Windows 2000
 427 w2k3;; Microsoft Windows 2003
 428 w2k8;; Microsoft Windows 2008
 429 wvista;; Microsoft Windows Vista
 430 win7;; Microsoft Windows 7
 431 win8;; Microsoft Windows 8/2012/2012r2
 432 win10;; Microsoft Windows 10/2016
 433 l24;; Linux 2.4 Kernel
 434 l26;; Linux 2.6 - 5.X Kernel
 435 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
 436 EODESC
 437     },
 438     boot => {
 439         optional => 1,
 440         type => 'string',
 441         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n).",
 442         pattern => '[acdn]{1,4}',
 443         default => 'cdn',
 444     },
 445     bootdisk => {
 446         optional => 1,
 447         type => 'string', format => 'pve-qm-bootdisk',
 448         description => "Enable booting from specified disk.",
 449         pattern => '(ide|sata|scsi|virtio)\d+',
 450     },
 451     smp => {
 452         optional => 1,
 453         type => 'integer',
 454         description => "The number of CPUs. Please use option -sockets instead.",
 455         minimum => 1,
 456         default => 1,
 457     },
 458     sockets => {
 459         optional => 1,
 460         type => 'integer',
 461         description => "The number of CPU sockets.",
 462         minimum => 1,
 463         default => 1,
 464     },
 465     cores => {
 466         optional => 1,
 467         type => 'integer',
 468         description => "The number of cores per socket.",
 469         minimum => 1,
 470         default => 1,
 471     },
 472     numa => {
 473         optional => 1,
 474         type => 'boolean',
 475         description => "Enable/disable NUMA.",
 476         default => 0,
 477     },
 478     hugepages => {
 479         optional => 1,
 480         type => 'string',
 481         description => "Enable/disable hugepages memory.",
 482         enum => [qw(any 2 1024)],
 483     },
 484     vcpus => {
 485         optional => 1,
 486         type => 'integer',
 487         description => "Number of hotplugged vcpus.",
 488         minimum => 1,
 489         default => 0,
 490     },
 491     acpi => {
 492         optional => 1,
 493         type => 'boolean',
 494         description => "Enable/disable ACPI.",
 495         default => 1,
 496     },
 497     agent => {
 498         optional => 1,
 499         description => "Enable/disable Qemu GuestAgent and its properties.",
 500         type => 'string',
 501         format => $agent_fmt,
 502     },
 503     kvm => {
 504         optional => 1,
 505         type => 'boolean',
 506         description => "Enable/disable KVM hardware virtualization.",
 507         default => 1,
 508     },
 509     tdf => {
 510         optional => 1,
 511         type => 'boolean',
 512         description => "Enable/disable time drift fix.",
 513         default => 0,
 514     },
 515     localtime => {
 516         optional => 1,
 517         type => 'boolean',
 518         description => "Set the real time clock to local time. This is enabled by default if ostype indicates a Microsoft OS.",
 519     },
 520     freeze => {
 521         optional => 1,
 522         type => 'boolean',
 523         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 524     },
 525     vga => {
 526         optional => 1,
 527         type => 'string', format => $vga_fmt,
 528         description => "Configure the VGA hardware.",
 529         verbose_description => "Configure the VGA Hardware. If you want to use ".
 530             "high resolution modes (>= 1280x1024x16) you may need to increase " .
 531             "the vga memory option. Since QEMU 2.9 the default VGA display type " .
 532             "is 'std' for all OS types besides some Windows versions (XP and " .
 533             "older) which use 'cirrus'. The 'qxl' option enables the SPICE " .
 534             "display server. For win* OS you can select how many independent " .
 535             "displays you want, Linux guests can add displays them self.\n".
 536             "You can also run without any graphic card, using a serial device as terminal.",
 537     },
 538     watchdog => {
 539         optional => 1,
 540         type => 'string', format => 'pve-qm-watchdog',
 541         description => "Create a virtual hardware watchdog device.",
 542         verbose_description => "Create a virtual hardware watchdog device. Once enabled" .
 543             " (by a guest action), the watchdog must be periodically polled " .
 544             "by an agent inside the guest or else the watchdog will reset " .
 545             "the guest (or execute the respective action specified)",
 546     },
 547     startdate => {
 548         optional => 1,
 549         type => 'string',
 550         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 551         description => "Set the initial date of the real time clock. Valid format for date are: 'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 552         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 553         default => 'now',
 554     },
 555     startup =>  get_standard_option('pve-startup-order'),
 556     template => {
 557         optional => 1,
 558         type => 'boolean',
 559         description => "Enable/disable Template.",
 560         default => 0,
 561     },
 562     args => {
 563         optional => 1,
 564         type => 'string',
 565         description => "Arbitrary arguments passed to kvm.",
 566         verbose_description => <<EODESCR,
 567 Arbitrary arguments passed to kvm, for example:
 568
 569 args: -no-reboot -no-hpet
 570
 571 NOTE: this option is for experts only.
 572 EODESCR
 573     },
 574     tablet => {
 575         optional => 1,
 576         type => 'boolean',
 577         default => 1,
 578         description => "Enable/disable the USB tablet device.",
 579         verbose_description => "Enable/disable the USB tablet device. This device is " .
 580             "usually needed to allow absolute mouse positioning with VNC. " .
 581             "Else the mouse runs out of sync with normal VNC clients. " .
 582             "If you're running lots of console-only guests on one host, " .
 583             "you may consider disabling this to save some context switches. " .
 584             "This is turned off by default if you use spice (-vga=qxl).",
 585     },
 586     migrate_speed => {
 587         optional => 1,
 588         type => 'integer',
 589         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 590         minimum => 0,
 591         default => 0,
 592     },
 593     migrate_downtime => {
 594         optional => 1,
 595         type => 'number',
 596         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 597         minimum => 0,
 598         default => 0.1,
 599     },
 600     cdrom => {
 601         optional => 1,
 602         type => 'string', format => 'pve-qm-ide',
 603         typetext => '<volume>',
 604         description => "This is an alias for option -ide2",
 605     },
 606     cpu => {
 607         optional => 1,
 608         description => "Emulated CPU type.",
 609         type => 'string',
 610         format => $cpu_fmt,
 611     },
 612     parent => get_standard_option('pve-snapshot-name', {
 613         optional => 1,
 614         description => "Parent snapshot name. This is used internally, and should not be modified.",
 615     }),
 616     snaptime => {
 617         optional => 1,
 618         description => "Timestamp for snapshots.",
 619         type => 'integer',
 620         minimum => 0,
 621     },
 622     vmstate => {
 623         optional => 1,
 624         type => 'string', format => 'pve-volume-id',
 625         description => "Reference to a volume which stores the VM state. This is used internally for snapshots.",
 626     },
 627     vmstatestorage => get_standard_option('pve-storage-id', {
 628         description => "Default storage for VM state volumes/files.",
 629         optional => 1,
 630     }),
 631     runningmachine => get_standard_option('pve-qemu-machine', {
 632         description => "Specifies the Qemu machine type of the running vm. This is used internally for snapshots.",
 633     }),
 634     machine => get_standard_option('pve-qemu-machine'),
 635     arch => {
 636         description => "Virtual processor architecture. Defaults to the host.",
 637         optional => 1,
 638         type => 'string',
 639         enum => [qw(x86_64 aarch64)],
 640     },
 641     smbios1 => {
 642         description => "Specify SMBIOS type 1 fields.",
 643         type => 'string', format => 'pve-qm-smbios1',
 644         maxLength => 512,
 645         optional => 1,
 646     },
 647     protection => {
 648         optional => 1,
 649         type => 'boolean',
 650         description => "Sets the protection flag of the VM. This will disable the remove VM and remove disk operations.",
 651         default => 0,
 652     },
 653     bios => {
 654         optional => 1,
 655         type => 'string',
 656         enum => [ qw(seabios ovmf) ],
 657         description => "Select BIOS implementation.",
 658         default => 'seabios',
 659     },
 660     vmgenid => {
 661         type => 'string',
 662         pattern => '(?:[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}|[01])',
 663         format_description => 'UUID',
 664         description => "Set VM Generation ID. Use '1' to autogenerate on create or update, pass '0' to disable explicitly.",
 665         verbose_description => "The VM generation ID (vmgenid) device exposes a".
 666             " 128-bit integer value identifier to the guest OS. This allows to".
 667             " notify the guest operating system when the virtual machine is".
 668             " executed with a different configuration (e.g. snapshot execution".
 669             " or creation from a template). The guest operating system notices".
 670             " the change, and is then able to react as appropriate by marking".
 671             " its copies of distributed databases as dirty, re-initializing its".
 672             " random number generator, etc.\n".
 673             "Note that auto-creation only works when done throug API/CLI create".
 674             " or update methods, but not when manually editing the config file.",
 675         default => "1 (autogenerated)",
 676         optional => 1,
 677     },
 678     hookscript => {
 679         type => 'string',
 680         format => 'pve-volume-id',
 681         optional => 1,
 682         description => "Script that will be executed during various steps in the vms lifetime.",
 683     },
 684     ivshmem => {
 685         type => 'string',
 686         format => $ivshmem_fmt,
 687         description => "Inter-VM shared memory. Useful for direct communication between VMs, or to the host.",
 688         optional => 1,
 689     },
 690     audio0 => {
 691         type => 'string',
 692         format => $audio_fmt,
 693         description => "Configure a audio device, useful in combination with QXL/Spice.",
 694         optional => 1
 695     },
 696     spice_enhancements => {
 697         type => 'string',
 698         format => $spice_enhancements_fmt,
 699         description => "Configure additional enhancements for SPICE.",
 700         optional => 1
 701     },
 702     tags => {
 703         type => 'string', format => 'pve-tag-list',
 704         description => 'Tags of the VM. This is only meta information.',
 705         optional => 1,
 706     },
 707 };
 708
 709 my $cicustom_fmt = {
 710     meta => {
 711         type => 'string',
 712         optional => 1,
 713         description => 'Specify a custom file containing all meta data passed to the VM via cloud-init. This is provider specific meaning configdrive2 and nocloud differ.',
 714         format => 'pve-volume-id',
 715         format_description => 'volume',
 716     },
 717     network => {
 718         type => 'string',
 719         optional => 1,
 720         description => 'Specify a custom file containing all network data passed to the VM via cloud-init.',
 721         format => 'pve-volume-id',
 722         format_description => 'volume',
 723     },
 724     user => {
 725         type => 'string',
 726         optional => 1,
 727         description => 'Specify a custom file containing all user data passed to the VM via cloud-init.',
 728         format => 'pve-volume-id',
 729         format_description => 'volume',
 730     },
 731 };
 732 PVE::JSONSchema::register_format('pve-qm-cicustom', $cicustom_fmt);
 733
 734 my $confdesc_cloudinit = {
 735     citype => {
 736         optional => 1,
 737         type => 'string',
 738         description => 'Specifies the cloud-init configuration format. The default depends on the configured operating system type (`ostype`. We use the `nocloud` format for Linux, and `configdrive2` for windows.',
 739         enum => ['configdrive2', 'nocloud'],
 740     },
 741     ciuser => {
 742         optional => 1,
 743         type => 'string',
 744         description => "cloud-init: User name to change ssh keys and password for instead of the image's configured default user.",
 745     },
 746     cipassword => {
 747         optional => 1,
 748         type => 'string',
 749         description => 'cloud-init: Password to assign the user. Using this is generally not recommended. Use ssh keys instead. Also note that older cloud-init versions do not support hashed passwords.',
 750     },
 751     cicustom => {
 752         optional => 1,
 753         type => 'string',
 754         description => 'cloud-init: Specify custom files to replace the automatically generated ones at start.',
 755         format => 'pve-qm-cicustom',
 756     },
 757     searchdomain => {
 758         optional => 1,
 759         type => 'string',
 760         description => "cloud-init: Sets DNS search domains for a container. Create will automatically use the setting from the host if neither searchdomain nor nameserver are set.",
 761     },
 762     nameserver => {
 763         optional => 1,
 764         type => 'string', format => 'address-list',
 765         description => "cloud-init: Sets DNS server IP address for a container. Create will automatically use the setting from the host if neither searchdomain nor nameserver are set.",
 766     },
 767     sshkeys => {
 768         optional => 1,
 769         type => 'string',
 770         format => 'urlencoded',
 771         description => "cloud-init: Setup public SSH keys (one key per line, OpenSSH format).",
 772     },
 773 };
 774
 775 # what about other qemu settings ?
 776 #cpu => 'string',
 777 #machine => 'string',
 778 #fda => 'file',
 779 #fdb => 'file',
 780 #mtdblock => 'file',
 781 #sd => 'file',
 782 #pflash => 'file',
 783 #snapshot => 'bool',
 784 #bootp => 'file',
 785 ##tftp => 'dir',
 786 ##smb => 'dir',
 787 #kernel => 'file',
 788 #append => 'string',
 789 #initrd => 'file',
 790 ##soundhw => 'string',
 791
 792 while (my ($k, $v) = each %$confdesc) {
 793     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 794 }
 795
 796 my $MAX_IDE_DISKS = 4;
 797 my $MAX_SCSI_DISKS = 14;
 798 my $MAX_VIRTIO_DISKS = 16;
 799 my $MAX_SATA_DISKS = 6;
 800 my $MAX_USB_DEVICES = 5;
 801 my $MAX_NETS = 32;
 802 my $MAX_UNUSED_DISKS = 256;
 803 my $MAX_HOSTPCI_DEVICES = 16;
 804 my $MAX_SERIAL_PORTS = 4;
 805 my $MAX_PARALLEL_PORTS = 3;
 806 my $MAX_NUMA = 8;
 807
 808 my $numa_fmt = {
 809     cpus => {
 810         type => "string",
 811         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 812         description => "CPUs accessing this NUMA node.",
 813         format_description => "id[-id];...",
 814     },
 815     memory => {
 816         type => "number",
 817         description => "Amount of memory this NUMA node provides.",
 818         optional => 1,
 819     },
 820     hostnodes => {
 821         type => "string",
 822         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 823         description => "Host NUMA nodes to use.",
 824         format_description => "id[-id];...",
 825         optional => 1,
 826     },
 827     policy => {
 828         type => 'string',
 829         enum => [qw(preferred bind interleave)],
 830         description => "NUMA allocation policy.",
 831         optional => 1,
 832     },
 833 };
 834 PVE::JSONSchema::register_format('pve-qm-numanode', $numa_fmt);
 835 my $numadesc = {
 836     optional => 1,
 837     type => 'string', format => $numa_fmt,
 838     description => "NUMA topology.",
 839 };
 840 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 841
 842 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 843     $confdesc->{"numa$i"} = $numadesc;
 844 }
 845
 846 my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000',  'pcnet',  'virtio',
 847                       'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3',
 848                       'e1000-82540em', 'e1000-82544gc', 'e1000-82545em'];
 849 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 850
 851 my $net_fmt_bridge_descr = <<__EOD__;
 852 Bridge to attach the network device to. The Proxmox VE standard bridge
 853 is called 'vmbr0'.
 854
 855 If you do not specify a bridge, we create a kvm user (NATed) network
 856 device, which provides DHCP and DNS services. The following addresses
 857 are used:
 858
 859  10.0.2.2   Gateway
 860  10.0.2.3   DNS Server
 861  10.0.2.4   SMB Server
 862
 863 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 864 __EOD__
 865
 866 my $net_fmt = {
 867     macaddr  => get_standard_option('mac-addr', {
 868         description => "MAC address. That address must be unique withing your network. This is automatically generated if not specified.",
 869     }),
 870     model => {
 871         type => 'string',
 872         description => "Network Card Model. The 'virtio' model provides the best performance with very low CPU overhead. If your guest does not support this driver, it is usually best to use 'e1000'.",
 873         enum => $nic_model_list,
 874         default_key => 1,
 875     },
 876     (map { $_ => { keyAlias => 'model', alias => 'macaddr' }} @$nic_model_list),
 877     bridge => {
 878         type => 'string',
 879         description => $net_fmt_bridge_descr,
 880         format_description => 'bridge',
 881         optional => 1,
 882     },
 883     queues => {
 884         type => 'integer',
 885         minimum => 0, maximum => 16,
 886         description => 'Number of packet queues to be used on the device.',
 887         optional => 1,
 888     },
 889     rate => {
 890         type => 'number',
 891         minimum => 0,
 892         description => "Rate limit in mbps (megabytes per second) as floating point number.",
 893         optional => 1,
 894     },
 895     tag => {
 896         type => 'integer',
 897         minimum => 1, maximum => 4094,
 898         description => 'VLAN tag to apply to packets on this interface.',
 899         optional => 1,
 900     },
 901     trunks => {
 902         type => 'string',
 903         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 904         description => 'VLAN trunks to pass through this interface.',
 905         format_description => 'vlanid[;vlanid...]',
 906         optional => 1,
 907     },
 908     firewall => {
 909         type => 'boolean',
 910         description => 'Whether this interface should be protected by the firewall.',
 911         optional => 1,
 912     },
 913     link_down => {
 914         type => 'boolean',
 915         description => 'Whether this interface should be disconnected (like pulling the plug).',
 916         optional => 1,
 917     },
 918 };
 919
 920 my $netdesc = {
 921     optional => 1,
 922     type => 'string', format => $net_fmt,
 923     description => "Specify network devices.",
 924 };
 925
 926 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 927
 928 my $ipconfig_fmt = {
 929     ip => {
 930         type => 'string',
 931         format => 'pve-ipv4-config',
 932         format_description => 'IPv4Format/CIDR',
 933         description => 'IPv4 address in CIDR format.',
 934         optional => 1,
 935         default => 'dhcp',
 936     },
 937     gw => {
 938         type => 'string',
 939         format => 'ipv4',
 940         format_description => 'GatewayIPv4',
 941         description => 'Default gateway for IPv4 traffic.',
 942         optional => 1,
 943         requires => 'ip',
 944     },
 945     ip6 => {
 946         type => 'string',
 947         format => 'pve-ipv6-config',
 948         format_description => 'IPv6Format/CIDR',
 949         description => 'IPv6 address in CIDR format.',
 950         optional => 1,
 951         default => 'dhcp',
 952     },
 953     gw6 => {
 954         type => 'string',
 955         format => 'ipv6',
 956         format_description => 'GatewayIPv6',
 957         description => 'Default gateway for IPv6 traffic.',
 958         optional => 1,
 959         requires => 'ip6',
 960     },
 961 };
 962 PVE::JSONSchema::register_format('pve-qm-ipconfig', $ipconfig_fmt);
 963 my $ipconfigdesc = {
 964     optional => 1,
 965     type => 'string', format => 'pve-qm-ipconfig',
 966     description => <<'EODESCR',
 967 cloud-init: Specify IP addresses and gateways for the corresponding interface.
 968
 969 IP addresses use CIDR notation, gateways are optional but need an IP of the same type specified.
 970
 971 The special string 'dhcp' can be used for IP addresses to use DHCP, in which case no explicit gateway should be provided.
 972 For IPv6 the special string 'auto' can be used to use stateless autoconfiguration.
 973
 974 If cloud-init is enabled and neither an IPv4 nor an IPv6 address is specified, it defaults to using dhcp on IPv4.
 975 EODESCR
 976 };
 977 PVE::JSONSchema::register_standard_option("pve-qm-ipconfig", $netdesc);
 978
 979 for (my $i = 0; $i < $MAX_NETS; $i++)  {
 980     $confdesc->{"net$i"} = $netdesc;
 981     $confdesc_cloudinit->{"ipconfig$i"} = $ipconfigdesc;
 982 }
 983
 984 foreach my $key (keys %$confdesc_cloudinit) {
 985     $confdesc->{$key} = $confdesc_cloudinit->{$key};
 986 }
 987
 988 PVE::JSONSchema::register_format('pve-volume-id-or-qm-path', \&verify_volume_id_or_qm_path);
 989 sub verify_volume_id_or_qm_path {
 990     my ($volid, $noerr) = @_;
 991
 992     if ($volid eq 'none' || $volid eq 'cdrom' || $volid =~ m|^/|) {
 993         return $volid;
 994     }
 995
 996     # if its neither 'none' nor 'cdrom' nor a path, check if its a volume-id
 997     $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') };
 998     if ($@) {
 999         return undef if $noerr;
1000         die $@;
1001     }
1002     return $volid;
1003 }
1004
1005 my $drivename_hash;
1006
1007 my %drivedesc_base = (
1008     volume => { alias => 'file' },
1009     file => {
1010         type => 'string',
1011         format => 'pve-volume-id-or-qm-path',
1012         default_key => 1,
1013         format_description => 'volume',
1014         description => "The drive's backing volume.",
1015     },
1016     media => {
1017         type => 'string',
1018         enum => [qw(cdrom disk)],
1019         description => "The drive's media type.",
1020         default => 'disk',
1021         optional => 1
1022     },
1023     cyls => {
1024         type => 'integer',
1025         description => "Force the drive's physical geometry to have a specific cylinder count.",
1026         optional => 1
1027     },
1028     heads => {
1029         type => 'integer',
1030         description => "Force the drive's physical geometry to have a specific head count.",
1031         optional => 1
1032     },
1033     secs => {
1034         type => 'integer',
1035         description => "Force the drive's physical geometry to have a specific sector count.",
1036         optional => 1
1037     },
1038     trans => {
1039         type => 'string',
1040         enum => [qw(none lba auto)],
1041         description => "Force disk geometry bios translation mode.",
1042         optional => 1,
1043     },
1044     snapshot => {
1045         type => 'boolean',
1046         description => "Controls qemu's snapshot mode feature."
1047             . " If activated, changes made to the disk are temporary and will"
1048             . " be discarded when the VM is shutdown.",
1049         optional => 1,
1050     },
1051     cache => {
1052         type => 'string',
1053         enum => [qw(none writethrough writeback unsafe directsync)],
1054         description => "The drive's cache mode",
1055         optional => 1,
1056     },
1057     format => get_standard_option('pve-qm-image-format'),
1058     size => {
1059         type => 'string',
1060         format => 'disk-size',
1061         format_description => 'DiskSize',
1062         description => "Disk size. This is purely informational and has no effect.",
1063         optional => 1,
1064     },
1065     backup => {
1066         type => 'boolean',
1067         description => "Whether the drive should be included when making backups.",
1068         optional => 1,
1069     },
1070     replicate => {
1071         type => 'boolean',
1072         description => 'Whether the drive should considered for replication jobs.',
1073         optional => 1,
1074         default => 1,
1075     },
1076     rerror => {
1077         type => 'string',
1078         enum => [qw(ignore report stop)],
1079         description => 'Read error action.',
1080         optional => 1,
1081     },
1082     werror => {
1083         type => 'string',
1084         enum => [qw(enospc ignore report stop)],
1085         description => 'Write error action.',
1086         optional => 1,
1087     },
1088     aio => {
1089         type => 'string',
1090         enum => [qw(native threads)],
1091         description => 'AIO type to use.',
1092         optional => 1,
1093     },
1094     discard => {
1095         type => 'string',
1096         enum => [qw(ignore on)],
1097         description => 'Controls whether to pass discard/trim requests to the underlying storage.',
1098         optional => 1,
1099     },
1100     detect_zeroes => {
1101         type => 'boolean',
1102         description => 'Controls whether to detect and try to optimize writes of zeroes.',
1103         optional => 1,
1104     },
1105     serial => {
1106         type => 'string',
1107         format => 'urlencoded',
1108         format_description => 'serial',
1109         maxLength => 20*3, # *3 since it's %xx url enoded
1110         description => "The drive's reported serial number, url-encoded, up to 20 bytes long.",
1111         optional => 1,
1112     },
1113     shared => {
1114         type => 'boolean',
1115         description => 'Mark this locally-managed volume as available on all nodes',
1116         verbose_description => "Mark this locally-managed volume as available on all nodes.\n\nWARNING: This option does not share the volume automatically, it assumes it is shared already!",
1117         optional => 1,
1118         default => 0,
1119     }
1120 );
1121
1122 my %iothread_fmt = ( iothread => {
1123         type => 'boolean',
1124         description => "Whether to use iothreads for this drive",
1125         optional => 1,
1126 });
1127
1128 my %model_fmt = (
1129     model => {
1130         type => 'string',
1131         format => 'urlencoded',
1132         format_description => 'model',
1133         maxLength => 40*3, # *3 since it's %xx url enoded
1134         description => "The drive's reported model name, url-encoded, up to 40 bytes long.",
1135         optional => 1,
1136     },
1137 );
1138
1139 my %queues_fmt = (
1140     queues => {
1141         type => 'integer',
1142         description => "Number of queues.",
1143         minimum => 2,
1144         optional => 1
1145     }
1146 );
1147
1148 my %scsiblock_fmt = (
1149     scsiblock => {
1150         type => 'boolean',
1151         description => "whether to use scsi-block for full passthrough of host block device\n\nWARNING: can lead to I/O errors in combination with low memory or high memory fragmentation on host",
1152         optional => 1,
1153         default => 0,
1154     },
1155 );
1156
1157 my %ssd_fmt = (
1158     ssd => {
1159         type => 'boolean',
1160         description => "Whether to expose this drive as an SSD, rather than a rotational hard disk.",
1161         optional => 1,
1162     },
1163 );
1164
1165 my %wwn_fmt = (
1166     wwn => {
1167         type => 'string',
1168         pattern => qr/^(0x)[0-9a-fA-F]{16}/,
1169         format_description => 'wwn',
1170         description => "The drive's worldwide name, encoded as 16 bytes hex string, prefixed by '0x'.",
1171         optional => 1,
1172     },
1173 );
1174
1175 my $add_throttle_desc = sub {
1176     my ($key, $type, $what, $unit, $longunit, $minimum) = @_;
1177     my $d = {
1178         type => $type,
1179         format_description => $unit,
1180         description => "Maximum $what in $longunit.",
1181         optional => 1,
1182     };
1183     $d->{minimum} = $minimum if defined($minimum);
1184     $drivedesc_base{$key} = $d;
1185 };
1186 # throughput: (leaky bucket)
1187 $add_throttle_desc->('bps',     'integer', 'r/w speed',   'bps',  'bytes per second');
1188 $add_throttle_desc->('bps_rd',  'integer', 'read speed',  'bps',  'bytes per second');
1189 $add_throttle_desc->('bps_wr',  'integer', 'write speed', 'bps',  'bytes per second');
1190 $add_throttle_desc->('mbps',    'number',  'r/w speed',   'mbps', 'megabytes per second');
1191 $add_throttle_desc->('mbps_rd', 'number',  'read speed',  'mbps', 'megabytes per second');
1192 $add_throttle_desc->('mbps_wr', 'number',  'write speed', 'mbps', 'megabytes per second');
1193 $add_throttle_desc->('iops',    'integer', 'r/w I/O',     'iops', 'operations per second');
1194 $add_throttle_desc->('iops_rd', 'integer', 'read I/O',    'iops', 'operations per second');
1195 $add_throttle_desc->('iops_wr', 'integer', 'write I/O',   'iops', 'operations per second');
1196
1197 # pools: (pool of IO before throttling starts taking effect)
1198 $add_throttle_desc->('mbps_max',    'number',  'unthrottled r/w pool',       'mbps', 'megabytes per second');
1199 $add_throttle_desc->('mbps_rd_max', 'number',  'unthrottled read pool',      'mbps', 'megabytes per second');
1200 $add_throttle_desc->('mbps_wr_max', 'number',  'unthrottled write pool',     'mbps', 'megabytes per second');
1201 $add_throttle_desc->('iops_max',    'integer', 'unthrottled r/w I/O pool',   'iops', 'operations per second');
1202 $add_throttle_desc->('iops_rd_max', 'integer', 'unthrottled read I/O pool',  'iops', 'operations per second');
1203 $add_throttle_desc->('iops_wr_max', 'integer', 'unthrottled write I/O pool', 'iops', 'operations per second');
1204
1205 # burst lengths
1206 $add_throttle_desc->('bps_max_length',     'integer', 'length of I/O bursts',       'seconds', 'seconds', 1);
1207 $add_throttle_desc->('bps_rd_max_length',  'integer', 'length of read I/O bursts',  'seconds', 'seconds', 1);
1208 $add_throttle_desc->('bps_wr_max_length',  'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
1209 $add_throttle_desc->('iops_max_length',    'integer', 'length of I/O bursts',       'seconds', 'seconds', 1);
1210 $add_throttle_desc->('iops_rd_max_length', 'integer', 'length of read I/O bursts',  'seconds', 'seconds', 1);
1211 $add_throttle_desc->('iops_wr_max_length', 'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
1212
1213 # legacy support
1214 $drivedesc_base{'bps_rd_length'} = { alias => 'bps_rd_max_length' };
1215 $drivedesc_base{'bps_wr_length'} = { alias => 'bps_wr_max_length' };
1216 $drivedesc_base{'iops_rd_length'} = { alias => 'iops_rd_max_length' };
1217 $drivedesc_base{'iops_wr_length'} = { alias => 'iops_wr_max_length' };
1218
1219 my $ide_fmt = {
1220     %drivedesc_base,
1221     %model_fmt,
1222     %ssd_fmt,
1223     %wwn_fmt,
1224 };
1225 PVE::JSONSchema::register_format("pve-qm-ide", $ide_fmt);
1226
1227 my $idedesc = {
1228     optional => 1,
1229     type => 'string', format => $ide_fmt,
1230     description => "Use volume as IDE hard disk or CD-ROM (n is 0 to " .($MAX_IDE_DISKS -1) . ").",
1231 };
1232 PVE::JSONSchema::register_standard_option("pve-qm-ide", $idedesc);
1233
1234 my $scsi_fmt = {
1235     %drivedesc_base,
1236     %iothread_fmt,
1237     %queues_fmt,
1238     %scsiblock_fmt,
1239     %ssd_fmt,
1240     %wwn_fmt,
1241 };
1242 my $scsidesc = {
1243     optional => 1,
1244     type => 'string', format => $scsi_fmt,
1245     description => "Use volume as SCSI hard disk or CD-ROM (n is 0 to " . ($MAX_SCSI_DISKS - 1) . ").",
1246 };
1247 PVE::JSONSchema::register_standard_option("pve-qm-scsi", $scsidesc);
1248
1249 my $sata_fmt = {
1250     %drivedesc_base,
1251     %ssd_fmt,
1252     %wwn_fmt,
1253 };
1254 my $satadesc = {
1255     optional => 1,
1256     type => 'string', format => $sata_fmt,
1257     description => "Use volume as SATA hard disk or CD-ROM (n is 0 to " . ($MAX_SATA_DISKS - 1). ").",
1258 };
1259 PVE::JSONSchema::register_standard_option("pve-qm-sata", $satadesc);
1260
1261 my $virtio_fmt = {
1262     %drivedesc_base,
1263     %iothread_fmt,
1264 };
1265 my $virtiodesc = {
1266     optional => 1,
1267     type => 'string', format => $virtio_fmt,
1268     description => "Use volume as VIRTIO hard disk (n is 0 to " . ($MAX_VIRTIO_DISKS - 1) . ").",
1269 };
1270 PVE::JSONSchema::register_standard_option("pve-qm-virtio", $virtiodesc);
1271
1272 my $alldrive_fmt = {
1273     %drivedesc_base,
1274     %iothread_fmt,
1275     %model_fmt,
1276     %queues_fmt,
1277     %scsiblock_fmt,
1278     %ssd_fmt,
1279     %wwn_fmt,
1280 };
1281
1282 my $efidisk_fmt = {
1283     volume => { alias => 'file' },
1284     file => {
1285         type => 'string',
1286         format => 'pve-volume-id-or-qm-path',
1287         default_key => 1,
1288         format_description => 'volume',
1289         description => "The drive's backing volume.",
1290     },
1291     format => get_standard_option('pve-qm-image-format'),
1292     size => {
1293         type => 'string',
1294         format => 'disk-size',
1295         format_description => 'DiskSize',
1296         description => "Disk size. This is purely informational and has no effect.",
1297         optional => 1,
1298     },
1299 };
1300
1301 my $efidisk_desc = {
1302     optional => 1,
1303     type => 'string', format => $efidisk_fmt,
1304     description => "Configure a Disk for storing EFI vars",
1305 };
1306
1307 PVE::JSONSchema::register_standard_option("pve-qm-efidisk", $efidisk_desc);
1308
1309 my $usb_fmt = {
1310     host => {
1311         default_key => 1,
1312         type => 'string', format => 'pve-qm-usb-device',
1313         format_description => 'HOSTUSBDEVICE|spice',
1314         description => <<EODESCR,
1315 The Host USB device or port or the value 'spice'. HOSTUSBDEVICE syntax is:
1316
1317  'bus-port(.port)*' (decimal numbers) or
1318  'vendor_id:product_id' (hexadeciaml numbers) or
1319  'spice'
1320
1321 You can use the 'lsusb -t' command to list existing usb devices.
1322
1323 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1324
1325 The value 'spice' can be used to add a usb redirection devices for spice.
1326 EODESCR
1327     },
1328     usb3 => {
1329         optional => 1,
1330         type => 'boolean',
1331         description => "Specifies whether if given host option is a USB3 device or port.",
1332         default => 0,
1333     },
1334 };
1335
1336 my $usbdesc = {
1337     optional => 1,
1338     type => 'string', format => $usb_fmt,
1339     description => "Configure an USB device (n is 0 to 4).",
1340 };
1341 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
1342
1343 my $PCIRE = qr/([a-f0-9]{4}:)?[a-f0-9]{2}:[a-f0-9]{2}(?:\.[a-f0-9])?/;
1344 my $hostpci_fmt = {
1345     host => {
1346         default_key => 1,
1347         type => 'string',
1348         pattern => qr/$PCIRE(;$PCIRE)*/,
1349         format_description => 'HOSTPCIID[;HOSTPCIID2...]',
1350         description => <<EODESCR,
1351 Host PCI device pass through. The PCI ID of a host's PCI device or a list
1352 of PCI virtual functions of the host. HOSTPCIID syntax is:
1353
1354 'bus:dev.func' (hexadecimal numbers)
1355
1356 You can us the 'lspci' command to list existing PCI devices.
1357 EODESCR
1358     },
1359     rombar => {
1360         type => 'boolean',
1361         description =>  "Specify whether or not the device's ROM will be visible in the guest's memory map.",
1362         optional => 1,
1363         default => 1,
1364     },
1365     romfile => {
1366         type => 'string',
1367         pattern => '[^,;]+',
1368         format_description => 'string',
1369         description => "Custom pci device rom filename (must be located in /usr/share/kvm/).",
1370         optional => 1,
1371     },
1372     pcie => {
1373         type => 'boolean',
1374         description =>  "Choose the PCI-express bus (needs the 'q35' machine model).",
1375         optional => 1,
1376         default => 0,
1377     },
1378     'x-vga' => {
1379         type => 'boolean',
1380         description =>  "Enable vfio-vga device support.",
1381         optional => 1,
1382         default => 0,
1383     },
1384     'mdev' => {
1385         type => 'string',
1386         format_description => 'string',
1387         pattern => '[^/\.:]+',
1388         optional => 1,
1389         description => <<EODESCR
1390 The type of mediated device to use.
1391 An instance of this type will be created on startup of the VM and
1392 will be cleaned up when the VM stops.
1393 EODESCR
1394     }
1395 };
1396 PVE::JSONSchema::register_format('pve-qm-hostpci', $hostpci_fmt);
1397
1398 my $hostpcidesc = {
1399         optional => 1,
1400         type => 'string', format => 'pve-qm-hostpci',
1401         description => "Map host PCI devices into guest.",
1402         verbose_description =>  <<EODESCR,
1403 Map host PCI devices into guest.
1404
1405 NOTE: This option allows direct access to host hardware. So it is no longer
1406 possible to migrate such machines - use with special care.
1407
1408 CAUTION: Experimental! User reported problems with this option.
1409 EODESCR
1410 };
1411 PVE::JSONSchema::register_standard_option("pve-qm-hostpci", $hostpcidesc);
1412
1413 my $serialdesc = {
1414         optional => 1,
1415         type => 'string',
1416         pattern => '(/dev/.+|socket)',
1417         description =>  "Create a serial device inside the VM (n is 0 to 3)",
1418         verbose_description =>  <<EODESCR,
1419 Create a serial device inside the VM (n is 0 to 3), and pass through a
1420 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1421 host side (use 'qm terminal' to open a terminal connection).
1422
1423 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines - use with special care.
1424
1425 CAUTION: Experimental! User reported problems with this option.
1426 EODESCR
1427 };
1428
1429 my $paralleldesc= {
1430         optional => 1,
1431         type => 'string',
1432         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
1433         description =>  "Map host parallel devices (n is 0 to 2).",
1434         verbose_description =>  <<EODESCR,
1435 Map host parallel devices (n is 0 to 2).
1436
1437 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1438
1439 CAUTION: Experimental! User reported problems with this option.
1440 EODESCR
1441 };
1442
1443 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
1444     $confdesc->{"parallel$i"} = $paralleldesc;
1445 }
1446
1447 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
1448     $confdesc->{"serial$i"} = $serialdesc;
1449 }
1450
1451 for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
1452     $confdesc->{"hostpci$i"} = $hostpcidesc;
1453 }
1454
1455 for (my $i = 0; $i < $MAX_IDE_DISKS; $i++)  {
1456     $drivename_hash->{"ide$i"} = 1;
1457     $confdesc->{"ide$i"} = $idedesc;
1458 }
1459
1460 for (my $i = 0; $i < $MAX_SATA_DISKS; $i++)  {
1461     $drivename_hash->{"sata$i"} = 1;
1462     $confdesc->{"sata$i"} = $satadesc;
1463 }
1464
1465 for (my $i = 0; $i < $MAX_SCSI_DISKS; $i++)  {
1466     $drivename_hash->{"scsi$i"} = 1;
1467     $confdesc->{"scsi$i"} = $scsidesc ;
1468 }
1469
1470 for (my $i = 0; $i < $MAX_VIRTIO_DISKS; $i++)  {
1471     $drivename_hash->{"virtio$i"} = 1;
1472     $confdesc->{"virtio$i"} = $virtiodesc;
1473 }
1474
1475 $drivename_hash->{efidisk0} = 1;
1476 $confdesc->{efidisk0} = $efidisk_desc;
1477
1478 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
1479     $confdesc->{"usb$i"} = $usbdesc;
1480 }
1481
1482 my $unuseddesc = {
1483     optional => 1,
1484     type => 'string', format => 'pve-volume-id',
1485     description => "Reference to unused volumes. This is used internally, and should not be modified manually.",
1486 };
1487
1488 for (my $i = 0; $i < $MAX_UNUSED_DISKS; $i++)  {
1489     $confdesc->{"unused$i"} = $unuseddesc;
1490 }
1491
1492 my $kvm_api_version = 0;
1493
1494 sub kvm_version {
1495     return $kvm_api_version if $kvm_api_version;
1496
1497     open my $fh, '<', '/dev/kvm'
1498         or return undef;
1499
1500     # 0xae00 => KVM_GET_API_VERSION
1501     $kvm_api_version = ioctl($fh, 0xae00, 0);
1502
1503     return $kvm_api_version;
1504 }
1505
1506 my $kvm_user_version = {};
1507 my $kvm_mtime = {};
1508
1509 sub kvm_user_version {
1510     my ($binary) = @_;
1511
1512     $binary //= get_command_for_arch(get_host_arch()); # get the native arch by default
1513     my $st = stat($binary);
1514
1515     my $cachedmtime = $kvm_mtime->{$binary} // -1;
1516     return $kvm_user_version->{$binary} if $kvm_user_version->{$binary} &&
1517         $cachedmtime == $st->mtime;
1518
1519     $kvm_user_version->{$binary} = 'unknown';
1520     $kvm_mtime->{$binary} = $st->mtime;
1521
1522     my $code = sub {
1523         my $line = shift;
1524         if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1525             $kvm_user_version->{$binary} = $2;
1526         }
1527     };
1528
1529     eval { run_command([$binary, '--version'], outfunc => $code); };
1530     warn $@ if $@;
1531
1532     return $kvm_user_version->{$binary};
1533
1534 }
1535
1536 sub kernel_has_vhost_net {
1537     return -c '/dev/vhost-net';
1538 }
1539
1540 sub valid_drive_names {
1541     # order is important - used to autoselect boot disk
1542     return ((map { "ide$_" } (0 .. ($MAX_IDE_DISKS - 1))),
1543             (map { "scsi$_" } (0 .. ($MAX_SCSI_DISKS - 1))),
1544             (map { "virtio$_" } (0 .. ($MAX_VIRTIO_DISKS - 1))),
1545             (map { "sata$_" } (0 .. ($MAX_SATA_DISKS - 1))),
1546             'efidisk0');
1547 }
1548
1549 sub is_valid_drivename {
1550     my $dev = shift;
1551
1552     return defined($drivename_hash->{$dev});
1553 }
1554
1555 sub option_exists {
1556     my $key = shift;
1557     return defined($confdesc->{$key});
1558 }
1559
1560 my $cdrom_path;
1561 sub get_cdrom_path {
1562
1563     return  $cdrom_path if $cdrom_path;
1564
1565     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
1566     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
1567     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
1568 }
1569
1570 sub get_iso_path {
1571     my ($storecfg, $vmid, $cdrom) = @_;
1572
1573     if ($cdrom eq 'cdrom') {
1574         return get_cdrom_path();
1575     } elsif ($cdrom eq 'none') {
1576         return '';
1577     } elsif ($cdrom =~ m|^/|) {
1578         return $cdrom;
1579     } else {
1580         return PVE::Storage::path($storecfg, $cdrom);
1581     }
1582 }
1583
1584 # try to convert old style file names to volume IDs
1585 sub filename_to_volume_id {
1586     my ($vmid, $file, $media) = @_;
1587
1588      if (!($file eq 'none' || $file eq 'cdrom' ||
1589           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1590
1591         return undef if $file =~ m|/|;
1592
1593         if ($media && $media eq 'cdrom') {
1594             $file = "local:iso/$file";
1595         } else {
1596             $file = "local:$vmid/$file";
1597         }
1598     }
1599
1600     return $file;
1601 }
1602
1603 sub verify_media_type {
1604     my ($opt, $vtype, $media) = @_;
1605
1606     return if !$media;
1607
1608     my $etype;
1609     if ($media eq 'disk') {
1610         $etype = 'images';
1611     } elsif ($media eq 'cdrom') {
1612         $etype = 'iso';
1613     } else {
1614         die "internal error";
1615     }
1616
1617     return if ($vtype eq $etype);
1618
1619     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1620 }
1621
1622 sub cleanup_drive_path {
1623     my ($opt, $storecfg, $drive) = @_;
1624
1625     # try to convert filesystem paths to volume IDs
1626
1627     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1628         ($drive->{file} !~ m|^/dev/.+|) &&
1629         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1630         ($drive->{file} !~ m/^\d+$/)) {
1631         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1632         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"}) if !$vtype;
1633         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1634         verify_media_type($opt, $vtype, $drive->{media});
1635         $drive->{file} = $volid;
1636     }
1637
1638     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1639 }
1640
1641 sub parse_hotplug_features {
1642     my ($data) = @_;
1643
1644     my $res = {};
1645
1646     return $res if $data eq '0';
1647
1648     $data = $confdesc->{hotplug}->{default} if $data eq '1';
1649
1650     foreach my $feature (PVE::Tools::split_list($data)) {
1651         if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
1652             $res->{$1} = 1;
1653         } else {
1654             die "invalid hotplug feature '$feature'\n";
1655         }
1656     }
1657     return $res;
1658 }
1659
1660 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1661 sub pve_verify_hotplug_features {
1662     my ($value, $noerr) = @_;
1663
1664     return $value if parse_hotplug_features($value);
1665
1666     return undef if $noerr;
1667
1668     die "unable to parse hotplug option\n";
1669 }
1670
1671 # ideX = [volume=]volume-id[,media=d][,cyls=c,heads=h,secs=s[,trans=t]]
1672 #        [,snapshot=on|off][,cache=on|off][,format=f][,backup=yes|no]
1673 #        [,rerror=ignore|report|stop][,werror=enospc|ignore|report|stop]
1674 #        [,aio=native|threads][,discard=ignore|on][,detect_zeroes=on|off]
1675 #        [,iothread=on][,serial=serial][,model=model]
1676
1677 sub parse_drive {
1678     my ($key, $data) = @_;
1679
1680     my ($interface, $index);
1681
1682     if ($key =~ m/^([^\d]+)(\d+)$/) {
1683         $interface = $1;
1684         $index = $2;
1685     } else {
1686         return undef;
1687     }
1688
1689     my $desc = $key =~ /^unused\d+$/ ? $alldrive_fmt
1690                                      : $confdesc->{$key}->{format};
1691     if (!$desc) {
1692         warn "invalid drive key: $key\n";
1693         return undef;
1694     }
1695     my $res = eval { PVE::JSONSchema::parse_property_string($desc, $data) };
1696     return undef if !$res;
1697     $res->{interface} = $interface;
1698     $res->{index} = $index;
1699
1700     my $error = 0;
1701     foreach my $opt (qw(bps bps_rd bps_wr)) {
1702         if (my $bps = defined(delete $res->{$opt})) {
1703             if (defined($res->{"m$opt"})) {
1704                 warn "both $opt and m$opt specified\n";
1705                 ++$error;
1706                 next;
1707             }
1708             $res->{"m$opt"} = sprintf("%.3f", $bps / (1024*1024.0));
1709         }
1710     }
1711
1712     # can't use the schema's 'requires' because of the mbps* => bps* "transforming aliases"
1713     for my $requirement (
1714         [mbps_max => 'mbps'],
1715         [mbps_rd_max => 'mbps_rd'],
1716         [mbps_wr_max => 'mbps_wr'],
1717         [miops_max => 'miops'],
1718         [miops_rd_max => 'miops_rd'],
1719         [miops_wr_max => 'miops_wr'],
1720         [bps_max_length => 'mbps_max'],
1721         [bps_rd_max_length => 'mbps_rd_max'],
1722         [bps_wr_max_length => 'mbps_wr_max'],
1723         [iops_max_length => 'iops_max'],
1724         [iops_rd_max_length => 'iops_rd_max'],
1725         [iops_wr_max_length => 'iops_wr_max']) {
1726         my ($option, $requires) = @$requirement;
1727         if ($res->{$option} && !$res->{$requires}) {
1728             warn "$option requires $requires\n";
1729             ++$error;
1730         }
1731     }
1732
1733     return undef if $error;
1734
1735     return undef if $res->{mbps_rd} && $res->{mbps};
1736     return undef if $res->{mbps_wr} && $res->{mbps};
1737     return undef if $res->{iops_rd} && $res->{iops};
1738     return undef if $res->{iops_wr} && $res->{iops};
1739
1740     if ($res->{media} && ($res->{media} eq 'cdrom')) {
1741         return undef if $res->{snapshot} || $res->{trans} || $res->{format};
1742         return undef if $res->{heads} || $res->{secs} || $res->{cyls};
1743         return undef if $res->{interface} eq 'virtio';
1744     }
1745
1746     if (my $size = $res->{size}) {
1747         return undef if !defined($res->{size} = PVE::JSONSchema::parse_size($size));
1748     }
1749
1750     return $res;
1751 }
1752
1753 sub print_drive {
1754     my ($vmid, $drive) = @_;
1755     my $data = { %$drive };
1756     delete $data->{$_} for qw(index interface);
1757     return PVE::JSONSchema::print_property_string($data, $alldrive_fmt);
1758 }
1759
1760 sub scsi_inquiry {
1761     my($fh, $noerr) = @_;
1762
1763     my $SG_IO = 0x2285;
1764     my $SG_GET_VERSION_NUM = 0x2282;
1765
1766     my $versionbuf = "\x00" x 8;
1767     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1768     if (!$ret) {
1769         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1770         return undef;
1771     }
1772     my $version = unpack("I", $versionbuf);
1773     if ($version < 30000) {
1774         die "scsi generic interface too old\n"  if !$noerr;
1775         return undef;
1776     }
1777
1778     my $buf = "\x00" x 36;
1779     my $sensebuf = "\x00" x 8;
1780     my $cmd = pack("C x3 C x1", 0x12, 36);
1781
1782     # see /usr/include/scsi/sg.h
1783     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1784
1785     my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1786                       length($sensebuf), 0, length($buf), $buf,
1787                       $cmd, $sensebuf, 6000);
1788
1789     $ret = ioctl($fh, $SG_IO, $packet);
1790     if (!$ret) {
1791         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1792         return undef;
1793     }
1794
1795     my @res = unpack($sg_io_hdr_t, $packet);
1796     if ($res[17] || $res[18]) {
1797         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1798         return undef;
1799     }
1800
1801     my $res = {};
1802     (my $byte0, my $byte1, $res->{vendor},
1803      $res->{product}, $res->{revision}) = unpack("C C x6 A8 A16 A4", $buf);
1804
1805     $res->{removable} = $byte1 & 128 ? 1 : 0;
1806     $res->{type} = $byte0 & 31;
1807
1808     return $res;
1809 }
1810
1811 sub path_is_scsi {
1812     my ($path) = @_;
1813
1814     my $fh = IO::File->new("+<$path") || return undef;
1815     my $res = scsi_inquiry($fh, 1);
1816     close($fh);
1817
1818     return $res;
1819 }
1820
1821 sub print_tabletdevice_full {
1822     my ($conf, $arch) = @_;
1823
1824     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
1825
1826     # we use uhci for old VMs because tablet driver was buggy in older qemu
1827     my $usbbus;
1828     if (PVE::QemuServer::Machine::machine_type_is_q35($conf) || $arch eq 'aarch64') {
1829         $usbbus = 'ehci';
1830     } else {
1831         $usbbus = 'uhci';
1832     }
1833
1834     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1835 }
1836
1837 sub print_keyboarddevice_full {
1838     my ($conf, $arch, $machine) = @_;
1839
1840     return undef if $arch ne 'aarch64';
1841
1842     return "usb-kbd,id=keyboard,bus=ehci.0,port=2";
1843 }
1844
1845 sub print_drivedevice_full {
1846     my ($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type) = @_;
1847
1848     my $device = '';
1849     my $maxdev = 0;
1850
1851     if ($drive->{interface} eq 'virtio') {
1852         my $pciaddr = print_pci_addr("$drive->{interface}$drive->{index}", $bridges, $arch, $machine_type);
1853         $device = "virtio-blk-pci,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}$pciaddr";
1854         $device .= ",iothread=iothread-$drive->{interface}$drive->{index}" if $drive->{iothread};
1855     } elsif ($drive->{interface} eq 'scsi') {
1856
1857         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1858         my $unit = $drive->{index} % $maxdev;
1859         my $devicetype = 'hd';
1860         my $path = '';
1861         if (drive_is_cdrom($drive)) {
1862             $devicetype = 'cd';
1863         } else {
1864             if ($drive->{file} =~ m|^/|) {
1865                 $path = $drive->{file};
1866                 if (my $info = path_is_scsi($path)) {
1867                     if ($info->{type} == 0 && $drive->{scsiblock}) {
1868                         $devicetype = 'block';
1869                     } elsif ($info->{type} == 1) { # tape
1870                         $devicetype = 'generic';
1871                     }
1872                 }
1873             } else {
1874                  $path = PVE::Storage::path($storecfg, $drive->{file});
1875             }
1876
1877             # for compatibility only, we prefer scsi-hd (#2408, #2355, #2380)
1878             my $version = PVE::QemuServer::Machine::extract_version($machine_type) // kvm_user_version();
1879             if ($path =~ m/^iscsi\:\/\// &&
1880                !min_version($version, 4, 1)) {
1881                 $devicetype = 'generic';
1882             }
1883         }
1884
1885         if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)){
1886             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1887         } else {
1888             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0,lun=$drive->{index},drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1889         }
1890
1891         if ($drive->{ssd} && ($devicetype eq 'block' || $devicetype eq 'hd')) {
1892             $device .= ",rotation_rate=1";
1893         }
1894         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1895
1896     } elsif ($drive->{interface} eq 'ide' || $drive->{interface} eq 'sata') {
1897         my $maxdev = ($drive->{interface} eq 'sata') ? $MAX_SATA_DISKS : 2;
1898         my $controller = int($drive->{index} / $maxdev);
1899         my $unit = $drive->{index} % $maxdev;
1900         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1901
1902         $device = "ide-$devicetype";
1903         if ($drive->{interface} eq 'ide') {
1904             $device .= ",bus=ide.$controller,unit=$unit";
1905         } else {
1906             $device .= ",bus=ahci$controller.$unit";
1907         }
1908         $device .= ",drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1909
1910         if ($devicetype eq 'hd') {
1911             if (my $model = $drive->{model}) {
1912                 $model = URI::Escape::uri_unescape($model);
1913                 $device .= ",model=$model";
1914             }
1915             if ($drive->{ssd}) {
1916                 $device .= ",rotation_rate=1";
1917             }
1918         }
1919         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1920     } elsif ($drive->{interface} eq 'usb') {
1921         die "implement me";
1922         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1923     } else {
1924         die "unsupported interface type";
1925     }
1926
1927     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1928
1929     if (my $serial = $drive->{serial}) {
1930         $serial = URI::Escape::uri_unescape($serial);
1931         $device .= ",serial=$serial";
1932     }
1933
1934
1935     return $device;
1936 }
1937
1938 sub get_initiator_name {
1939     my $initiator;
1940
1941     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return undef;
1942     while (defined(my $line = <$fh>)) {
1943         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1944         $initiator = $1;
1945         last;
1946     }
1947     $fh->close();
1948
1949     return $initiator;
1950 }
1951
1952 sub print_drive_full {
1953     my ($storecfg, $vmid, $drive) = @_;
1954
1955     my $path;
1956     my $volid = $drive->{file};
1957     my $format;
1958
1959     if (drive_is_cdrom($drive)) {
1960         $path = get_iso_path($storecfg, $vmid, $volid);
1961     } else {
1962         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1963         if ($storeid) {
1964             $path = PVE::Storage::path($storecfg, $volid);
1965             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
1966             $format = qemu_img_format($scfg, $volname);
1967         } else {
1968             $path = $volid;
1969             $format = "raw";
1970         }
1971    }
1972
1973     my $opts = '';
1974     my @qemu_drive_options = qw(heads secs cyls trans media format cache rerror werror aio discard);
1975     foreach my $o (@qemu_drive_options) {
1976         $opts .= ",$o=$drive->{$o}" if defined($drive->{$o});
1977     }
1978
1979     # snapshot only accepts on|off
1980     if (defined($drive->{snapshot})) {
1981         my $v = $drive->{snapshot} ? 'on' : 'off';
1982         $opts .= ",snapshot=$v";
1983     }
1984
1985     foreach my $type (['', '-total'], [_rd => '-read'], [_wr => '-write']) {
1986         my ($dir, $qmpname) = @$type;
1987         if (my $v = $drive->{"mbps$dir"}) {
1988             $opts .= ",throttling.bps$qmpname=".int($v*1024*1024);
1989         }
1990         if (my $v = $drive->{"mbps${dir}_max"}) {
1991             $opts .= ",throttling.bps$qmpname-max=".int($v*1024*1024);
1992         }
1993         if (my $v = $drive->{"bps${dir}_max_length"}) {
1994             $opts .= ",throttling.bps$qmpname-max-length=$v";
1995         }
1996         if (my $v = $drive->{"iops${dir}"}) {
1997             $opts .= ",throttling.iops$qmpname=$v";
1998         }
1999         if (my $v = $drive->{"iops${dir}_max"}) {
2000             $opts .= ",throttling.iops$qmpname-max=$v";
2001         }
2002         if (my $v = $drive->{"iops${dir}_max_length"}) {
2003             $opts .= ",throttling.iops$qmpname-max-length=$v";
2004         }
2005     }
2006
2007     $opts .= ",format=$format" if $format && !$drive->{format};
2008
2009     my $cache_direct = 0;
2010
2011     if (my $cache = $drive->{cache}) {
2012         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
2013     } elsif (!drive_is_cdrom($drive)) {
2014         $opts .= ",cache=none";
2015         $cache_direct = 1;
2016     }
2017
2018     # aio native works only with O_DIRECT
2019     if (!$drive->{aio}) {
2020         if($cache_direct) {
2021             $opts .= ",aio=native";
2022         } else {
2023             $opts .= ",aio=threads";
2024         }
2025     }
2026
2027     if (!drive_is_cdrom($drive)) {
2028         my $detectzeroes;
2029         if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
2030             $detectzeroes = 'off';
2031         } elsif ($drive->{discard}) {
2032             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
2033         } else {
2034             # This used to be our default with discard not being specified:
2035             $detectzeroes = 'on';
2036         }
2037         $opts .= ",detect-zeroes=$detectzeroes" if $detectzeroes;
2038     }
2039
2040     my $pathinfo = $path ? "file=$path," : '';
2041
2042     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
2043 }
2044
2045 sub print_netdevice_full {
2046     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files, $arch, $machine_type) = @_;
2047
2048     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
2049
2050     my $device = $net->{model};
2051     if ($net->{model} eq 'virtio') {
2052          $device = 'virtio-net-pci';
2053      };
2054
2055     my $pciaddr = print_pci_addr("$netid", $bridges, $arch, $machine_type);
2056     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
2057     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
2058         #Consider we have N queues, the number of vectors needed is 2*N + 2 (plus one config interrupt and control vq)
2059         my $vectors = $net->{queues} * 2 + 2;
2060         $tmpstr .= ",vectors=$vectors,mq=on";
2061     }
2062     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
2063
2064     if ($use_old_bios_files) {
2065         my $romfile;
2066         if ($device eq 'virtio-net-pci') {
2067             $romfile = 'pxe-virtio.rom';
2068         } elsif ($device eq 'e1000') {
2069             $romfile = 'pxe-e1000.rom';
2070         } elsif ($device eq 'ne2k') {
2071             $romfile = 'pxe-ne2k_pci.rom';
2072         } elsif ($device eq 'pcnet') {
2073             $romfile = 'pxe-pcnet.rom';
2074         } elsif ($device eq 'rtl8139') {
2075             $romfile = 'pxe-rtl8139.rom';
2076         }
2077         $tmpstr .= ",romfile=$romfile" if $romfile;
2078     }
2079
2080     return $tmpstr;
2081 }
2082
2083 sub print_netdev_full {
2084     my ($vmid, $conf, $arch, $net, $netid, $hotplug) = @_;
2085
2086     my $i = '';
2087     if ($netid =~ m/^net(\d+)$/) {
2088         $i = int($1);
2089     }
2090
2091     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
2092
2093     my $ifname = "tap${vmid}i$i";
2094
2095     # kvm uses TUNSETIFF ioctl, and that limits ifname length
2096     die "interface name '$ifname' is too long (max 15 character)\n"
2097         if length($ifname) >= 16;
2098
2099     my $vhostparam = '';
2100     if (is_native($arch)) {
2101         $vhostparam = ',vhost=on' if kernel_has_vhost_net() && $net->{model} eq 'virtio';
2102     }
2103
2104     my $vmname = $conf->{name} || "vm$vmid";
2105
2106     my $netdev = "";
2107     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
2108
2109     if ($net->{bridge}) {
2110         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script,downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
2111     } else {
2112         $netdev = "type=user,id=$netid,hostname=$vmname";
2113     }
2114
2115     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
2116
2117     return $netdev;
2118 }
2119
2120
2121 sub print_cpu_device {
2122     my ($conf, $id) = @_;
2123
2124     my $kvm = $conf->{kvm} // 1;
2125     my $cpu = $kvm ? "kvm64" : "qemu64";
2126     if (my $cputype = $conf->{cpu}) {
2127         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
2128             or die "Cannot parse cpu description: $cputype\n";
2129         $cpu = $cpuconf->{cputype};
2130     }
2131
2132     my $cores = $conf->{cores} || 1;
2133
2134     my $current_core = ($id - 1) % $cores;
2135     my $current_socket = int(($id - 1 - $current_core)/$cores);
2136
2137     return "$cpu-x86_64-cpu,id=cpu$id,socket-id=$current_socket,core-id=$current_core,thread-id=0";
2138 }
2139
2140 my $vga_map = {
2141     'cirrus' => 'cirrus-vga',
2142     'std' => 'VGA',
2143     'vmware' => 'vmware-svga',
2144     'virtio' => 'virtio-vga',
2145 };
2146
2147 sub print_vga_device {
2148     my ($conf, $vga, $arch, $machine_version, $machine, $id, $qxlnum, $bridges) = @_;
2149
2150     my $type = $vga_map->{$vga->{type}};
2151     if ($arch eq 'aarch64' && defined($type) && $type eq 'virtio-vga') {
2152         $type = 'virtio-gpu';
2153     }
2154     my $vgamem_mb = $vga->{memory};
2155
2156     my $max_outputs = '';
2157     if ($qxlnum) {
2158         $type = $id ? 'qxl' : 'qxl-vga';
2159
2160         if (!$conf->{ostype} || $conf->{ostype} =~ m/^(?:l\d\d)|(?:other)$/) {
2161             # set max outputs so linux can have up to 4 qxl displays with one device
2162             if (min_version($machine_version, 4, 1)) {
2163                 $max_outputs = ",max_outputs=4";
2164             }
2165         }
2166     }
2167
2168     die "no devicetype for $vga->{type}\n" if !$type;
2169
2170     my $memory = "";
2171     if ($vgamem_mb) {
2172         if ($vga->{type} eq 'virtio') {
2173             my $bytes = PVE::Tools::convert_size($vgamem_mb, "mb" => "b");
2174             $memory = ",max_hostmem=$bytes";
2175         } elsif ($qxlnum) {
2176             # from https://www.spice-space.org/multiple-monitors.html
2177             $memory = ",vgamem_mb=$vga->{memory}";
2178             my $ram = $vgamem_mb * 4;
2179             my $vram = $vgamem_mb * 2;
2180             $memory .= ",ram_size_mb=$ram,vram_size_mb=$vram";
2181         } else {
2182             $memory = ",vgamem_mb=$vga->{memory}";
2183         }
2184     } elsif ($qxlnum && $id) {
2185         $memory = ",ram_size=67108864,vram_size=33554432";
2186     }
2187
2188     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
2189     my $vgaid = "vga" . ($id // '');
2190     my $pciaddr;
2191
2192     if ($q35 && $vgaid eq 'vga') {
2193         # the first display uses pcie.0 bus on q35 machines
2194         $pciaddr = print_pcie_addr($vgaid, $bridges, $arch, $machine);
2195     } else {
2196         $pciaddr = print_pci_addr($vgaid, $bridges, $arch, $machine);
2197     }
2198
2199     return "$type,id=${vgaid}${memory}${max_outputs}${pciaddr}";
2200 }
2201
2202 sub drive_is_cloudinit {
2203     my ($drive) = @_;
2204     return $drive->{file} =~ m@[:/]vm-\d+-cloudinit(?:\.$QEMU_FORMAT_RE)?$@;
2205 }
2206
2207 sub drive_is_cdrom {
2208     my ($drive, $exclude_cloudinit) = @_;
2209
2210     return 0 if $exclude_cloudinit && drive_is_cloudinit($drive);
2211
2212     return $drive && $drive->{media} && ($drive->{media} eq 'cdrom');
2213
2214 }
2215
2216 sub parse_number_sets {
2217     my ($set) = @_;
2218     my $res = [];
2219     foreach my $part (split(/;/, $set)) {
2220         if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
2221             die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
2222             push @$res, [ $1, $2 ];
2223         } else {
2224             die "invalid range: $part\n";
2225         }
2226     }
2227     return $res;
2228 }
2229
2230 sub parse_numa {
2231     my ($data) = @_;
2232
2233     my $res = PVE::JSONSchema::parse_property_string($numa_fmt, $data);
2234     $res->{cpus} = parse_number_sets($res->{cpus}) if defined($res->{cpus});
2235     $res->{hostnodes} = parse_number_sets($res->{hostnodes}) if defined($res->{hostnodes});
2236     return $res;
2237 }
2238
2239 sub parse_hostpci {
2240     my ($value) = @_;
2241
2242     return undef if !$value;
2243
2244     my $res = PVE::JSONSchema::parse_property_string($hostpci_fmt, $value);
2245
2246     my @idlist = split(/;/, $res->{host});
2247     delete $res->{host};
2248     foreach my $id (@idlist) {
2249         if ($id =~ m/\./) { # full id 00:00.1
2250             push @{$res->{pciid}}, {
2251                 id => $id,
2252             };
2253         } else { # partial id 00:00
2254             $res->{pciid} = PVE::SysFSTools::lspci($id);
2255         }
2256     }
2257     return $res;
2258 }
2259
2260 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
2261 sub parse_net {
2262     my ($data) = @_;
2263
2264     my $res = eval { PVE::JSONSchema::parse_property_string($net_fmt, $data) };
2265     if ($@) {
2266         warn $@;
2267         return undef;
2268     }
2269     if (!defined($res->{macaddr})) {
2270         my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
2271         $res->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix});
2272     }
2273     return $res;
2274 }
2275
2276 # ipconfigX ip=cidr,gw=ip,ip6=cidr,gw6=ip
2277 sub parse_ipconfig {
2278     my ($data) = @_;
2279
2280     my $res = eval { PVE::JSONSchema::parse_property_string($ipconfig_fmt, $data) };
2281     if ($@) {
2282         warn $@;
2283         return undef;
2284     }
2285
2286     if ($res->{gw} && !$res->{ip}) {
2287         warn 'gateway specified without specifying an IP address';
2288         return undef;
2289     }
2290     if ($res->{gw6} && !$res->{ip6}) {
2291         warn 'IPv6 gateway specified without specifying an IPv6 address';
2292         return undef;
2293     }
2294     if ($res->{gw} && $res->{ip} eq 'dhcp') {
2295         warn 'gateway specified together with DHCP';
2296         return undef;
2297     }
2298     if ($res->{gw6} && $res->{ip6} !~ /^$IPV6RE/) {
2299         # gw6 + auto/dhcp
2300         warn "IPv6 gateway specified together with $res->{ip6} address";
2301         return undef;
2302     }
2303
2304     if (!$res->{ip} && !$res->{ip6}) {
2305         return { ip => 'dhcp', ip6 => 'dhcp' };
2306     }
2307
2308     return $res;
2309 }
2310
2311 sub print_net {
2312     my $net = shift;
2313
2314     return PVE::JSONSchema::print_property_string($net, $net_fmt);
2315 }
2316
2317 sub add_random_macs {
2318     my ($settings) = @_;
2319
2320     foreach my $opt (keys %$settings) {
2321         next if $opt !~ m/^net(\d+)$/;
2322         my $net = parse_net($settings->{$opt});
2323         next if !$net;
2324         $settings->{$opt} = print_net($net);
2325     }
2326 }
2327
2328 sub vm_is_volid_owner {
2329     my ($storecfg, $vmid, $volid) = @_;
2330
2331     if ($volid !~  m|^/|) {
2332         my ($path, $owner);
2333         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
2334         if ($owner && ($owner == $vmid)) {
2335             return 1;
2336         }
2337     }
2338
2339     return undef;
2340 }
2341
2342 sub vmconfig_register_unused_drive {
2343     my ($storecfg, $vmid, $conf, $drive) = @_;
2344
2345     if (drive_is_cloudinit($drive)) {
2346         eval { PVE::Storage::vdisk_free($storecfg, $drive->{file}) };
2347         warn $@ if $@;
2348     } elsif (!drive_is_cdrom($drive)) {
2349         my $volid = $drive->{file};
2350         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
2351             PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
2352         }
2353     }
2354 }
2355
2356 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str][,base64=bool]
2357 my $smbios1_fmt = {
2358     uuid => {
2359         type => 'string',
2360         pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
2361         format_description => 'UUID',
2362         description => "Set SMBIOS1 UUID.",
2363         optional => 1,
2364     },
2365     version => {
2366         type => 'string',
2367         pattern => '[A-Za-z0-9+\/]+={0,2}',
2368         format_description => 'Base64 encoded string',
2369         description => "Set SMBIOS1 version.",
2370         optional => 1,
2371     },
2372     serial => {
2373         type => 'string',
2374         pattern => '[A-Za-z0-9+\/]+={0,2}',
2375         format_description => 'Base64 encoded string',
2376         description => "Set SMBIOS1 serial number.",
2377         optional => 1,
2378     },
2379     manufacturer => {
2380         type => 'string',
2381         pattern => '[A-Za-z0-9+\/]+={0,2}',
2382         format_description => 'Base64 encoded string',
2383         description => "Set SMBIOS1 manufacturer.",
2384         optional => 1,
2385     },
2386     product => {
2387         type => 'string',
2388         pattern => '[A-Za-z0-9+\/]+={0,2}',
2389         format_description => 'Base64 encoded string',
2390         description => "Set SMBIOS1 product ID.",
2391         optional => 1,
2392     },
2393     sku => {
2394         type => 'string',
2395         pattern => '[A-Za-z0-9+\/]+={0,2}',
2396         format_description => 'Base64 encoded string',
2397         description => "Set SMBIOS1 SKU string.",
2398         optional => 1,
2399     },
2400     family => {
2401         type => 'string',
2402         pattern => '[A-Za-z0-9+\/]+={0,2}',
2403         format_description => 'Base64 encoded string',
2404         description => "Set SMBIOS1 family string.",
2405         optional => 1,
2406     },
2407     base64 => {
2408         type => 'boolean',
2409         description => 'Flag to indicate that the SMBIOS values are base64 encoded',
2410         optional => 1,
2411     },
2412 };
2413
2414 sub parse_smbios1 {
2415     my ($data) = @_;
2416
2417     my $res = eval { PVE::JSONSchema::parse_property_string($smbios1_fmt, $data) };
2418     warn $@ if $@;
2419     return $res;
2420 }
2421
2422 sub print_smbios1 {
2423     my ($smbios1) = @_;
2424     return PVE::JSONSchema::print_property_string($smbios1, $smbios1_fmt);
2425 }
2426
2427 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_fmt);
2428
2429 PVE::JSONSchema::register_format('pve-qm-bootdisk', \&verify_bootdisk);
2430 sub verify_bootdisk {
2431     my ($value, $noerr) = @_;
2432
2433     return $value if is_valid_drivename($value);
2434
2435     return undef if $noerr;
2436
2437     die "invalid boot disk '$value'\n";
2438 }
2439
2440 sub parse_watchdog {
2441     my ($value) = @_;
2442
2443     return undef if !$value;
2444
2445     my $res = eval { PVE::JSONSchema::parse_property_string($watchdog_fmt, $value) };
2446     warn $@ if $@;
2447     return $res;
2448 }
2449
2450 sub parse_guest_agent {
2451     my ($value) = @_;
2452
2453     return {} if !defined($value->{agent});
2454
2455     my $res = eval { PVE::JSONSchema::parse_property_string($agent_fmt, $value->{agent}) };
2456     warn $@ if $@;
2457
2458     # if the agent is disabled ignore the other potentially set properties
2459     return {} if !$res->{enabled};
2460     return $res;
2461 }
2462
2463 sub parse_vga {
2464     my ($value) = @_;
2465
2466     return {} if !$value;
2467     my $res = eval { PVE::JSONSchema::parse_property_string($vga_fmt, $value) };
2468     warn $@ if $@;
2469     return $res;
2470 }
2471
2472 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
2473 sub verify_usb_device {
2474     my ($value, $noerr) = @_;
2475
2476     return $value if parse_usb_device($value);
2477
2478     return undef if $noerr;
2479
2480     die "unable to parse usb device\n";
2481 }
2482
2483 # add JSON properties for create and set function
2484 sub json_config_properties {
2485     my $prop = shift;
2486
2487     foreach my $opt (keys %$confdesc) {
2488         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate' || $opt eq 'runningmachine';
2489         $prop->{$opt} = $confdesc->{$opt};
2490     }
2491
2492     return $prop;
2493 }
2494
2495 # return copy of $confdesc_cloudinit to generate documentation
2496 sub cloudinit_config_properties {
2497
2498     return dclone($confdesc_cloudinit);
2499 }
2500
2501 sub check_type {
2502     my ($key, $value) = @_;
2503
2504     die "unknown setting '$key'\n" if !$confdesc->{$key};
2505
2506     my $type = $confdesc->{$key}->{type};
2507
2508     if (!defined($value)) {
2509         die "got undefined value\n";
2510     }
2511
2512     if ($value =~ m/[\n\r]/) {
2513         die "property contains a line feed\n";
2514     }
2515
2516     if ($type eq 'boolean') {
2517         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
2518         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
2519         die "type check ('boolean') failed - got '$value'\n";
2520     } elsif ($type eq 'integer') {
2521         return int($1) if $value =~ m/^(\d+)$/;
2522         die "type check ('integer') failed - got '$value'\n";
2523     } elsif ($type eq 'number') {
2524         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
2525         die "type check ('number') failed - got '$value'\n";
2526     } elsif ($type eq 'string') {
2527         if (my $fmt = $confdesc->{$key}->{format}) {
2528             PVE::JSONSchema::check_format($fmt, $value);
2529             return $value;
2530         }
2531         $value =~ s/^\"(.*)\"$/$1/;
2532         return $value;
2533     } else {
2534         die "internal error"
2535     }
2536 }
2537
2538 sub destroy_vm {
2539     my ($storecfg, $vmid, $skiplock, $replacement_conf) = @_;
2540
2541     my $conf = PVE::QemuConfig->load_config($vmid);
2542
2543     PVE::QemuConfig->check_lock($conf) if !$skiplock;
2544
2545     if ($conf->{template}) {
2546         # check if any base image is still used by a linked clone
2547         foreach_drive($conf, sub {
2548                 my ($ds, $drive) = @_;
2549                 return if drive_is_cdrom($drive);
2550
2551                 my $volid = $drive->{file};
2552                 return if !$volid || $volid =~ m|^/|;
2553
2554                 die "base volume '$volid' is still in use by linked cloned\n"
2555                     if PVE::Storage::volume_is_base_and_used($storecfg, $volid);
2556
2557         });
2558     }
2559
2560     # only remove disks owned by this VM
2561     foreach_drive($conf, sub {
2562         my ($ds, $drive) = @_;
2563         return if drive_is_cdrom($drive, 1);
2564
2565         my $volid = $drive->{file};
2566         return if !$volid || $volid =~ m|^/|;
2567
2568         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
2569         return if !$path || !$owner || ($owner != $vmid);
2570
2571         eval { PVE::Storage::vdisk_free($storecfg, $volid) };
2572         warn "Could not remove disk '$volid', check manually: $@" if $@;
2573     });
2574
2575     # also remove unused disk
2576     my $vmdisks = PVE::Storage::vdisk_list($storecfg, undef, $vmid);
2577     PVE::Storage::foreach_volid($vmdisks, sub {
2578         my ($volid, $sid, $volname, $d) = @_;
2579         eval { PVE::Storage::vdisk_free($storecfg, $volid) };
2580         warn $@ if $@;
2581     });
2582
2583     if (defined $replacement_conf) {
2584         PVE::QemuConfig->write_config($vmid, $replacement_conf);
2585     } else {
2586         PVE::QemuConfig->destroy_config($vmid);
2587     }
2588 }
2589
2590 sub parse_vm_config {
2591     my ($filename, $raw) = @_;
2592
2593     return undef if !defined($raw);
2594
2595     my $res = {
2596         digest => Digest::SHA::sha1_hex($raw),
2597         snapshots => {},
2598         pending => {},
2599     };
2600
2601     $filename =~ m|/qemu-server/(\d+)\.conf$|
2602         || die "got strange filename '$filename'";
2603
2604     my $vmid = $1;
2605
2606     my $conf = $res;
2607     my $descr;
2608     my $section = '';
2609
2610     my @lines = split(/\n/, $raw);
2611     foreach my $line (@lines) {
2612         next if $line =~ m/^\s*$/;
2613
2614         if ($line =~ m/^\[PENDING\]\s*$/i) {
2615             $section = 'pending';
2616             if (defined($descr)) {
2617                 $descr =~ s/\s+$//;
2618                 $conf->{description} = $descr;
2619             }
2620             $descr = undef;
2621             $conf = $res->{$section} = {};
2622             next;
2623
2624         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2625             $section = $1;
2626             if (defined($descr)) {
2627                 $descr =~ s/\s+$//;
2628                 $conf->{description} = $descr;
2629             }
2630             $descr = undef;
2631             $conf = $res->{snapshots}->{$section} = {};
2632             next;
2633         }
2634
2635         if ($line =~ m/^\#(.*)\s*$/) {
2636             $descr = '' if !defined($descr);
2637             $descr .= PVE::Tools::decode_text($1) . "\n";
2638             next;
2639         }
2640
2641         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2642             $descr = '' if !defined($descr);
2643             $descr .= PVE::Tools::decode_text($2);
2644         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2645             $conf->{snapstate} = $1;
2646         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2647             my $key = $1;
2648             my $value = $2;
2649             $conf->{$key} = $value;
2650         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2651             my $value = $1;
2652             if ($section eq 'pending') {
2653                 $conf->{delete} = $value; # we parse this later
2654             } else {
2655                 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
2656             }
2657         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(.+?)\s*$/) {
2658             my $key = $1;
2659             my $value = $2;
2660             eval { $value = check_type($key, $value); };
2661             if ($@) {
2662                 warn "vm $vmid - unable to parse value of '$key' - $@";
2663             } else {
2664                 $key = 'ide2' if $key eq 'cdrom';
2665                 my $fmt = $confdesc->{$key}->{format};
2666                 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2667                     my $v = parse_drive($key, $value);
2668                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2669                         $v->{file} = $volid;
2670                         $value = print_drive($vmid, $v);
2671                     } else {
2672                         warn "vm $vmid - unable to parse value of '$key'\n";
2673                         next;
2674                     }
2675                 }
2676
2677                 $conf->{$key} = $value;
2678             }
2679         }
2680     }
2681
2682     if (defined($descr)) {
2683         $descr =~ s/\s+$//;
2684         $conf->{description} = $descr;
2685     }
2686     delete $res->{snapstate}; # just to be sure
2687
2688     return $res;
2689 }
2690
2691 sub write_vm_config {
2692     my ($filename, $conf) = @_;
2693
2694     delete $conf->{snapstate}; # just to be sure
2695
2696     if ($conf->{cdrom}) {
2697         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2698         $conf->{ide2} = $conf->{cdrom};
2699         delete $conf->{cdrom};
2700     }
2701
2702     # we do not use 'smp' any longer
2703     if ($conf->{sockets}) {
2704         delete $conf->{smp};
2705     } elsif ($conf->{smp}) {
2706         $conf->{sockets} = $conf->{smp};
2707         delete $conf->{cores};
2708         delete $conf->{smp};
2709     }
2710
2711     my $used_volids = {};
2712
2713     my $cleanup_config = sub {
2714         my ($cref, $pending, $snapname) = @_;
2715
2716         foreach my $key (keys %$cref) {
2717             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2718                 $key eq 'snapstate' || $key eq 'pending';
2719             my $value = $cref->{$key};
2720             if ($key eq 'delete') {
2721                 die "propertry 'delete' is only allowed in [PENDING]\n"
2722                     if !$pending;
2723                 # fixme: check syntax?
2724                 next;
2725             }
2726             eval { $value = check_type($key, $value); };
2727             die "unable to parse value of '$key' - $@" if $@;
2728
2729             $cref->{$key} = $value;
2730
2731             if (!$snapname && is_valid_drivename($key)) {
2732                 my $drive = parse_drive($key, $value);
2733                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2734             }
2735         }
2736     };
2737
2738     &$cleanup_config($conf);
2739
2740     &$cleanup_config($conf->{pending}, 1);
2741
2742     foreach my $snapname (keys %{$conf->{snapshots}}) {
2743         # TODO: we can allow snapshots with name 'pending' after PVE 7.0
2744         # since pending section is namespaced with 'pve:'
2745         # but for now, we should forbid it to avoid confusion in parser
2746         die "internal error: snapshot name '$snapname' is forbidden" if lc($snapname) eq 'pending';
2747         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2748     }
2749
2750     # remove 'unusedX' settings if we re-add a volume
2751     foreach my $key (keys %$conf) {
2752         my $value = $conf->{$key};
2753         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2754             delete $conf->{$key};
2755         }
2756     }
2757
2758     my $generate_raw_config = sub {
2759         my ($conf, $pending) = @_;
2760
2761         my $raw = '';
2762
2763         # add description as comment to top of file
2764         if (defined(my $descr = $conf->{description})) {
2765             if ($descr) {
2766                 foreach my $cl (split(/\n/, $descr)) {
2767                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2768                 }
2769             } else {
2770                 $raw .= "#\n" if $pending;
2771             }
2772         }
2773
2774         foreach my $key (sort keys %$conf) {
2775             next if $key eq 'digest' || $key eq 'description' || $key eq 'pending' || $key eq 'snapshots';
2776             $raw .= "$key: $conf->{$key}\n";
2777         }
2778         return $raw;
2779     };
2780
2781     my $raw = &$generate_raw_config($conf);
2782
2783     if (scalar(keys %{$conf->{pending}})){
2784         $raw .= "\n[PENDING]\n";
2785         $raw .= &$generate_raw_config($conf->{pending}, 1);
2786     }
2787
2788     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2789         $raw .= "\n[$snapname]\n";
2790         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2791     }
2792
2793     return $raw;
2794 }
2795
2796 sub load_defaults {
2797
2798     my $res = {};
2799
2800     # we use static defaults from our JSON schema configuration
2801     foreach my $key (keys %$confdesc) {
2802         if (defined(my $default = $confdesc->{$key}->{default})) {
2803             $res->{$key} = $default;
2804         }
2805     }
2806
2807     return $res;
2808 }
2809
2810 sub config_list {
2811     my $vmlist = PVE::Cluster::get_vmlist();
2812     my $res = {};
2813     return $res if !$vmlist || !$vmlist->{ids};
2814     my $ids = $vmlist->{ids};
2815
2816     foreach my $vmid (keys %$ids) {
2817         my $d = $ids->{$vmid};
2818         next if !$d->{node} || $d->{node} ne $nodename;
2819         next if !$d->{type} || $d->{type} ne 'qemu';
2820         $res->{$vmid}->{exists} = 1;
2821     }
2822     return $res;
2823 }
2824
2825 # test if VM uses local resources (to prevent migration)
2826 sub check_local_resources {
2827     my ($conf, $noerr) = @_;
2828
2829     my @loc_res = ();
2830
2831     push @loc_res, "hostusb" if $conf->{hostusb}; # old syntax
2832     push @loc_res, "hostpci" if $conf->{hostpci}; # old syntax
2833
2834     push @loc_res, "ivshmem" if $conf->{ivshmem};
2835
2836     foreach my $k (keys %$conf) {
2837         next if $k =~ m/^usb/ && ($conf->{$k} =~ m/^spice(?![^,])/);
2838         # sockets are safe: they will recreated be on the target side post-migrate
2839         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2840         push @loc_res, $k if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2841     }
2842
2843     die "VM uses local resources\n" if scalar @loc_res && !$noerr;
2844
2845     return \@loc_res;
2846 }
2847
2848 # check if used storages are available on all nodes (use by migrate)
2849 sub check_storage_availability {
2850     my ($storecfg, $conf, $node) = @_;
2851
2852     foreach_drive($conf, sub {
2853         my ($ds, $drive) = @_;
2854
2855         my $volid = $drive->{file};
2856         return if !$volid;
2857
2858         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2859         return if !$sid;
2860
2861         # check if storage is available on both nodes
2862         my $scfg = PVE::Storage::storage_check_node($storecfg, $sid);
2863         PVE::Storage::storage_check_node($storecfg, $sid, $node);
2864    });
2865 }
2866
2867 # list nodes where all VM images are available (used by has_feature API)
2868 sub shared_nodes {
2869     my ($conf, $storecfg) = @_;
2870
2871     my $nodelist = PVE::Cluster::get_nodelist();
2872     my $nodehash = { map { $_ => 1 } @$nodelist };
2873     my $nodename = PVE::INotify::nodename();
2874
2875     foreach_drive($conf, sub {
2876         my ($ds, $drive) = @_;
2877
2878         my $volid = $drive->{file};
2879         return if !$volid;
2880
2881         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2882         if ($storeid) {
2883             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2884             if ($scfg->{disable}) {
2885                 $nodehash = {};
2886             } elsif (my $avail = $scfg->{nodes}) {
2887                 foreach my $node (keys %$nodehash) {
2888                     delete $nodehash->{$node} if !$avail->{$node};
2889                 }
2890             } elsif (!$scfg->{shared}) {
2891                 foreach my $node (keys %$nodehash) {
2892                     delete $nodehash->{$node} if $node ne $nodename
2893                 }
2894             }
2895         }
2896     });
2897
2898     return $nodehash
2899 }
2900
2901 sub check_local_storage_availability {
2902     my ($conf, $storecfg) = @_;
2903
2904     my $nodelist = PVE::Cluster::get_nodelist();
2905     my $nodehash = { map { $_ => {} } @$nodelist };
2906
2907     foreach_drive($conf, sub {
2908         my ($ds, $drive) = @_;
2909
2910         my $volid = $drive->{file};
2911         return if !$volid;
2912
2913         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2914         if ($storeid) {
2915             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2916
2917             if ($scfg->{disable}) {
2918                 foreach my $node (keys %$nodehash) {
2919                     $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2920                 }
2921             } elsif (my $avail = $scfg->{nodes}) {
2922                 foreach my $node (keys %$nodehash) {
2923                     if (!$avail->{$node}) {
2924                         $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2925                     }
2926                 }
2927             }
2928         }
2929     });
2930
2931     foreach my $node (values %$nodehash) {
2932         if (my $unavail = $node->{unavailable_storages}) {
2933             $node->{unavailable_storages} = [ sort keys %$unavail ];
2934         }
2935     }
2936
2937     return $nodehash
2938 }
2939
2940 # Compat only, use assert_config_exists_on_node and vm_running_locally where possible
2941 sub check_running {
2942     my ($vmid, $nocheck, $node) = @_;
2943
2944     PVE::QemuConfig::assert_config_exists_on_node($vmid, $node) if !$nocheck;
2945     return PVE::QemuServer::Helpers::vm_running_locally($vmid);
2946 }
2947
2948 sub vzlist {
2949
2950     my $vzlist = config_list();
2951
2952     my $fd = IO::Dir->new($PVE::QemuServer::Helpers::var_run_tmpdir) || return $vzlist;
2953
2954     while (defined(my $de = $fd->read)) {
2955         next if $de !~ m/^(\d+)\.pid$/;
2956         my $vmid = $1;
2957         next if !defined($vzlist->{$vmid});
2958         if (my $pid = check_running($vmid)) {
2959             $vzlist->{$vmid}->{pid} = $pid;
2960         }
2961     }
2962
2963     return $vzlist;
2964 }
2965
2966 sub disksize {
2967     my ($storecfg, $conf) = @_;
2968
2969     my $bootdisk = $conf->{bootdisk};
2970     return undef if !$bootdisk;
2971     return undef if !is_valid_drivename($bootdisk);
2972
2973     return undef if !$conf->{$bootdisk};
2974
2975     my $drive = parse_drive($bootdisk, $conf->{$bootdisk});
2976     return undef if !defined($drive);
2977
2978     return undef if drive_is_cdrom($drive);
2979
2980     my $volid = $drive->{file};
2981     return undef if !$volid;
2982
2983     return $drive->{size};
2984 }
2985
2986 our $vmstatus_return_properties = {
2987     vmid => get_standard_option('pve-vmid'),
2988     status => {
2989         description => "Qemu process status.",
2990         type => 'string',
2991         enum => ['stopped', 'running'],
2992     },
2993     maxmem => {
2994         description => "Maximum memory in bytes.",
2995         type => 'integer',
2996         optional => 1,
2997         renderer => 'bytes',
2998     },
2999     maxdisk => {
3000         description => "Root disk size in bytes.",
3001         type => 'integer',
3002         optional => 1,
3003         renderer => 'bytes',
3004     },
3005     name => {
3006         description => "VM name.",
3007         type => 'string',
3008         optional => 1,
3009     },
3010     qmpstatus => {
3011         description => "Qemu QMP agent status.",
3012         type => 'string',
3013         optional => 1,
3014     },
3015     pid => {
3016         description => "PID of running qemu process.",
3017         type => 'integer',
3018         optional => 1,
3019     },
3020     uptime => {
3021         description => "Uptime.",
3022         type => 'integer',
3023         optional => 1,
3024         renderer => 'duration',
3025     },
3026     cpus => {
3027         description => "Maximum usable CPUs.",
3028         type => 'number',
3029         optional => 1,
3030     },
3031     lock => {
3032         description => "The current config lock, if any.",
3033         type => 'string',
3034         optional => 1,
3035     },
3036     tags => {
3037         description  => "The current configured tags, if any",
3038         type => 'string',
3039         optional => 1,
3040     },
3041 };
3042
3043 my $last_proc_pid_stat;
3044
3045 # get VM status information
3046 # This must be fast and should not block ($full == false)
3047 # We only query KVM using QMP if $full == true (this can be slow)
3048 sub vmstatus {
3049     my ($opt_vmid, $full) = @_;
3050
3051     my $res = {};
3052
3053     my $storecfg = PVE::Storage::config();
3054
3055     my $list = vzlist();
3056     my $defaults = load_defaults();
3057
3058     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
3059
3060     my $cpucount = $cpuinfo->{cpus} || 1;
3061
3062     foreach my $vmid (keys %$list) {
3063         next if $opt_vmid && ($vmid ne $opt_vmid);
3064
3065         my $conf = PVE::QemuConfig->load_config($vmid);
3066
3067         my $d = { vmid => $vmid };
3068         $d->{pid} = $list->{$vmid}->{pid};
3069
3070         # fixme: better status?
3071         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
3072
3073         my $size = disksize($storecfg, $conf);
3074         if (defined($size)) {
3075             $d->{disk} = 0; # no info available
3076             $d->{maxdisk} = $size;
3077         } else {
3078             $d->{disk} = 0;
3079             $d->{maxdisk} = 0;
3080         }
3081
3082         $d->{cpus} = ($conf->{sockets} || $defaults->{sockets})
3083             * ($conf->{cores} || $defaults->{cores});
3084         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
3085         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
3086
3087         $d->{name} = $conf->{name} || "VM $vmid";
3088         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024)
3089             : $defaults->{memory}*(1024*1024);
3090
3091         if ($conf->{balloon}) {
3092             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
3093             $d->{shares} = defined($conf->{shares}) ? $conf->{shares}
3094                 : $defaults->{shares};
3095         }
3096
3097         $d->{uptime} = 0;
3098         $d->{cpu} = 0;
3099         $d->{mem} = 0;
3100
3101         $d->{netout} = 0;
3102         $d->{netin} = 0;
3103
3104         $d->{diskread} = 0;
3105         $d->{diskwrite} = 0;
3106
3107         $d->{template} = PVE::QemuConfig->is_template($conf);
3108
3109         $d->{serial} = 1 if conf_has_serial($conf);
3110         $d->{lock} = $conf->{lock} if $conf->{lock};
3111         $d->{tags} = $conf->{tags} if defined($conf->{tags});
3112
3113         $res->{$vmid} = $d;
3114     }
3115
3116     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
3117     foreach my $dev (keys %$netdev) {
3118         next if $dev !~ m/^tap([1-9]\d*)i/;
3119         my $vmid = $1;
3120         my $d = $res->{$vmid};
3121         next if !$d;
3122
3123         $d->{netout} += $netdev->{$dev}->{receive};
3124         $d->{netin} += $netdev->{$dev}->{transmit};
3125
3126         if ($full) {
3127             $d->{nics}->{$dev}->{netout} = $netdev->{$dev}->{receive};
3128             $d->{nics}->{$dev}->{netin} = $netdev->{$dev}->{transmit};
3129         }
3130
3131     }
3132
3133     my $ctime = gettimeofday;
3134
3135     foreach my $vmid (keys %$list) {
3136
3137         my $d = $res->{$vmid};
3138         my $pid = $d->{pid};
3139         next if !$pid;
3140
3141         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
3142         next if !$pstat; # not running
3143
3144         my $used = $pstat->{utime} + $pstat->{stime};
3145
3146         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
3147
3148         if ($pstat->{vsize}) {
3149             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
3150         }
3151
3152         my $old = $last_proc_pid_stat->{$pid};
3153         if (!$old) {
3154             $last_proc_pid_stat->{$pid} = {
3155                 time => $ctime,
3156                 used => $used,
3157                 cpu => 0,
3158             };
3159             next;
3160         }
3161
3162         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
3163
3164         if ($dtime > 1000) {
3165             my $dutime = $used -  $old->{used};
3166
3167             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
3168             $last_proc_pid_stat->{$pid} = {
3169                 time => $ctime,
3170                 used => $used,
3171                 cpu => $d->{cpu},
3172             };
3173         } else {
3174             $d->{cpu} = $old->{cpu};
3175         }
3176     }
3177
3178     return $res if !$full;
3179
3180     my $qmpclient = PVE::QMPClient->new();
3181
3182     my $ballooncb = sub {
3183         my ($vmid, $resp) = @_;
3184
3185         my $info = $resp->{'return'};
3186         return if !$info->{max_mem};
3187
3188         my $d = $res->{$vmid};
3189
3190         # use memory assigned to VM
3191         $d->{maxmem} = $info->{max_mem};
3192         $d->{balloon} = $info->{actual};
3193
3194         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
3195             $d->{mem} = $info->{total_mem} - $info->{free_mem};
3196             $d->{freemem} = $info->{free_mem};
3197         }
3198
3199         $d->{ballooninfo} = $info;
3200     };
3201
3202     my $blockstatscb = sub {
3203         my ($vmid, $resp) = @_;
3204         my $data = $resp->{'return'} || [];
3205         my $totalrdbytes = 0;
3206         my $totalwrbytes = 0;
3207
3208         for my $blockstat (@$data) {
3209             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
3210             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
3211
3212             $blockstat->{device} =~ s/drive-//;
3213             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
3214         }
3215         $res->{$vmid}->{diskread} = $totalrdbytes;
3216         $res->{$vmid}->{diskwrite} = $totalwrbytes;
3217     };
3218
3219     my $statuscb = sub {
3220         my ($vmid, $resp) = @_;
3221
3222         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
3223         # this fails if ballon driver is not loaded, so this must be
3224         # the last commnand (following command are aborted if this fails).
3225         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
3226
3227         my $status = 'unknown';
3228         if (!defined($status = $resp->{'return'}->{status})) {
3229             warn "unable to get VM status\n";
3230             return;
3231         }
3232
3233         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
3234     };
3235
3236     foreach my $vmid (keys %$list) {
3237         next if $opt_vmid && ($vmid ne $opt_vmid);
3238         next if !$res->{$vmid}->{pid}; # not running
3239         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
3240     }
3241
3242     $qmpclient->queue_execute(undef, 2);
3243
3244     foreach my $vmid (keys %$list) {
3245         next if $opt_vmid && ($vmid ne $opt_vmid);
3246         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
3247     }
3248
3249     return $res;
3250 }
3251
3252 sub foreach_drive {
3253     my ($conf, $func, @param) = @_;
3254
3255     foreach my $ds (valid_drive_names()) {
3256         next if !defined($conf->{$ds});
3257
3258         my $drive = parse_drive($ds, $conf->{$ds});
3259         next if !$drive;
3260
3261         &$func($ds, $drive, @param);
3262     }
3263 }
3264
3265 sub foreach_volid {
3266     my ($conf, $func, @param) = @_;
3267
3268     my $volhash = {};
3269
3270     my $test_volid = sub {
3271         my ($volid, $is_cdrom, $replicate, $shared, $snapname, $size) = @_;
3272
3273         return if !$volid;
3274
3275         $volhash->{$volid}->{cdrom} //= 1;
3276         $volhash->{$volid}->{cdrom} = 0 if !$is_cdrom;
3277
3278         $volhash->{$volid}->{replicate} //= 0;
3279         $volhash->{$volid}->{replicate} = 1 if $replicate;
3280
3281         $volhash->{$volid}->{shared} //= 0;
3282         $volhash->{$volid}->{shared} = 1 if $shared;
3283
3284         $volhash->{$volid}->{referenced_in_config} //= 0;
3285         $volhash->{$volid}->{referenced_in_config} = 1 if !defined($snapname);
3286
3287         $volhash->{$volid}->{referenced_in_snapshot}->{$snapname} = 1
3288             if defined($snapname);
3289         $volhash->{$volid}->{size} = $size if $size;
3290     };
3291
3292     foreach_drive($conf, sub {
3293         my ($ds, $drive) = @_;
3294         $test_volid->($drive->{file}, drive_is_cdrom($drive), $drive->{replicate} // 1, $drive->{shared}, undef, $drive->{size});
3295     });
3296
3297     foreach my $snapname (keys %{$conf->{snapshots}}) {
3298         my $snap = $conf->{snapshots}->{$snapname};
3299         $test_volid->($snap->{vmstate}, 0, 1, $snapname);
3300         foreach_drive($snap, sub {
3301             my ($ds, $drive) = @_;
3302             $test_volid->($drive->{file}, drive_is_cdrom($drive), $drive->{replicate} // 1, $drive->{shared}, $snapname);
3303         });
3304     }
3305
3306     foreach my $volid (keys %$volhash) {
3307         &$func($volid, $volhash->{$volid}, @param);
3308     }
3309 }
3310
3311 sub conf_has_serial {
3312     my ($conf) = @_;
3313
3314     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3315         if ($conf->{"serial$i"}) {
3316             return 1;
3317         }
3318     }
3319
3320     return 0;
3321 }
3322
3323 sub conf_has_audio {
3324     my ($conf, $id) = @_;
3325
3326     $id //= 0;
3327     my $audio = $conf->{"audio$id"};
3328     return undef if !defined($audio);
3329
3330     my $audioproperties = PVE::JSONSchema::parse_property_string($audio_fmt, $audio);
3331     my $audiodriver = $audioproperties->{driver} // 'spice';
3332
3333     return {
3334         dev => $audioproperties->{device},
3335         dev_id => "audiodev$id",
3336         backend => $audiodriver,
3337         backend_id => "$audiodriver-backend${id}",
3338     };
3339 }
3340
3341 sub vga_conf_has_spice {
3342     my ($vga) = @_;
3343
3344     my $vgaconf = parse_vga($vga);
3345     my $vgatype = $vgaconf->{type};
3346     return 0 if !$vgatype || $vgatype !~ m/^qxl([234])?$/;
3347
3348     return $1 || 1;
3349 }
3350
3351 sub is_native($) {
3352     my ($arch) = @_;
3353     return get_host_arch() eq $arch;
3354 }
3355
3356 sub get_vm_arch {
3357     my ($conf) = @_;
3358     return $conf->{arch} // get_host_arch();
3359 }
3360
3361 my $default_machines = {
3362     x86_64 => 'pc',
3363     aarch64 => 'virt',
3364 };
3365
3366 sub get_vm_machine {
3367     my ($conf, $forcemachine, $arch) = @_;
3368
3369     my $machine = $forcemachine || $conf->{machine};
3370
3371     if (!$machine) {
3372         $arch //= 'x86_64';
3373         $machine ||= $default_machines->{$arch};
3374     }
3375
3376     return $machine;
3377 }
3378
3379 sub get_ovmf_files($) {
3380     my ($arch) = @_;
3381
3382     my $ovmf = $OVMF->{$arch}
3383         or die "no OVMF images known for architecture '$arch'\n";
3384
3385     return @$ovmf;
3386 }
3387
3388 my $Arch2Qemu = {
3389     aarch64 => '/usr/bin/qemu-system-aarch64',
3390     x86_64 => '/usr/bin/qemu-system-x86_64',
3391 };
3392 sub get_command_for_arch($) {
3393     my ($arch) = @_;
3394     return '/usr/bin/kvm' if is_native($arch);
3395
3396     my $cmd = $Arch2Qemu->{$arch}
3397         or die "don't know how to emulate architecture '$arch'\n";
3398     return $cmd;
3399 }
3400
3401 sub get_cpu_options {
3402     my ($conf, $arch, $kvm, $kvm_off, $machine_version, $winversion, $gpu_passthrough) = @_;
3403
3404     my $cpuFlags = [];
3405     my $ostype = $conf->{ostype};
3406
3407     my $cpu = $kvm ? "kvm64" : "qemu64";
3408     if ($arch eq 'aarch64') {
3409         $cpu = 'cortex-a57';
3410     }
3411     my $hv_vendor_id;
3412     if (my $cputype = $conf->{cpu}) {
3413         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
3414             or die "Cannot parse cpu description: $cputype\n";
3415         $cpu = $cpuconf->{cputype};
3416         $kvm_off = 1 if $cpuconf->{hidden};
3417         $hv_vendor_id = $cpuconf->{'hv-vendor-id'};
3418
3419         if (defined(my $flags = $cpuconf->{flags})) {
3420             push @$cpuFlags, split(";", $flags);
3421         }
3422     }
3423
3424     push @$cpuFlags , '+lahf_lm' if $cpu eq 'kvm64' && $arch eq 'x86_64';
3425
3426     push @$cpuFlags , '-x2apic' if $ostype && $ostype eq 'solaris';
3427
3428     push @$cpuFlags, '+sep' if $cpu eq 'kvm64' || $cpu eq 'kvm32';
3429
3430     push @$cpuFlags, '-rdtscp' if $cpu =~ m/^Opteron/;
3431
3432     if (min_version($machine_version, 2, 3) && $arch eq 'x86_64') {
3433
3434         push @$cpuFlags , '+kvm_pv_unhalt' if $kvm;
3435         push @$cpuFlags , '+kvm_pv_eoi' if $kvm;
3436     }
3437
3438     add_hyperv_enlightenments($cpuFlags, $winversion, $machine_version, $conf->{bios}, $gpu_passthrough, $hv_vendor_id) if $kvm;
3439
3440     push @$cpuFlags, 'enforce' if $cpu ne 'host' && $kvm && $arch eq 'x86_64';
3441
3442     push @$cpuFlags, 'kvm=off' if $kvm_off;
3443
3444     if (my $cpu_vendor = $cpu_vendor_list->{$cpu}) {
3445         push @$cpuFlags, "vendor=${cpu_vendor}"
3446             if $cpu_vendor ne 'default';
3447     } elsif ($arch ne 'aarch64') {
3448         die "internal error"; # should not happen
3449     }
3450
3451     $cpu .= "," . join(',', @$cpuFlags) if scalar(@$cpuFlags);
3452
3453     return ('-cpu', $cpu);
3454 }
3455
3456 sub config_to_command {
3457     my ($storecfg, $vmid, $conf, $defaults, $forcemachine) = @_;
3458
3459     my $cmd = [];
3460     my $globalFlags = [];
3461     my $machineFlags = [];
3462     my $rtcFlags = [];
3463     my $devices = [];
3464     my $pciaddr = '';
3465     my $bridges = {};
3466     my $vernum = 0; # unknown
3467     my $ostype = $conf->{ostype};
3468     my $winversion = windows_version($ostype);
3469     my $kvm = $conf->{kvm};
3470
3471     my $arch = get_vm_arch($conf);
3472     my $kvm_binary = get_command_for_arch($arch);
3473     my $kvmver = kvm_user_version($kvm_binary);
3474
3475     my $machine_type = get_vm_machine($conf, $forcemachine, $arch);
3476     my $machine_version = PVE::QemuServer::Machine::extract_version($machine_type) // $kvmver;
3477     $kvm //= 1 if is_native($arch);
3478
3479     if ($kvm) {
3480         die "KVM virtualisation configured, but not available. Either disable in VM configuration or enable in BIOS.\n"
3481             if !defined kvm_version();
3482     }
3483
3484     if ($kvmver =~ m/^(\d+)\.(\d+)$/) {
3485         $vernum = $1*1000000+$2*1000;
3486     } elsif ($kvmver =~ m/^(\d+)\.(\d+)\.(\d+)$/) {
3487         $vernum = $1*1000000+$2*1000+$3;
3488     }
3489
3490     die "detected old qemu-kvm binary ($kvmver)\n" if $vernum < 15000;
3491
3492     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
3493     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
3494     my $use_old_bios_files = undef;
3495     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3496
3497     my $cpuunits = defined($conf->{cpuunits}) ?
3498             $conf->{cpuunits} : $defaults->{cpuunits};
3499
3500     push @$cmd, $kvm_binary;
3501
3502     push @$cmd, '-id', $vmid;
3503
3504     my $vmname = $conf->{name} || "vm$vmid";
3505
3506     push @$cmd, '-name', $vmname;
3507
3508     my $use_virtio = 0;
3509
3510     my $qmpsocket = PVE::QemuServer::Helpers::qmp_socket($vmid);
3511     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server,nowait";
3512     push @$cmd, '-mon', "chardev=qmp,mode=control";
3513
3514     if (min_version($machine_version, 2, 12)) {
3515         push @$cmd, '-chardev', "socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5";
3516         push @$cmd, '-mon', "chardev=qmp-event,mode=control";
3517     }
3518
3519     push @$cmd, '-pidfile' , PVE::QemuServer::Helpers::pidfile_name($vmid);
3520
3521     push @$cmd, '-daemonize';
3522
3523     if ($conf->{smbios1}) {
3524         my $smbios_conf = parse_smbios1($conf->{smbios1});
3525         if ($smbios_conf->{base64}) {
3526             # Do not pass base64 flag to qemu
3527             delete $smbios_conf->{base64};
3528             my $smbios_string = "";
3529             foreach my $key (keys %$smbios_conf) {
3530                 my $value;
3531                 if ($key eq "uuid") {
3532                     $value = $smbios_conf->{uuid}
3533                 } else {
3534                     $value = decode_base64($smbios_conf->{$key});
3535                 }
3536                 # qemu accepts any binary data, only commas need escaping by double comma
3537                 $value =~ s/,/,,/g;
3538                 $smbios_string .= "," . $key . "=" . $value if $value;
3539             }
3540             push @$cmd, '-smbios', "type=1" . $smbios_string;
3541         } else {
3542             push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
3543         }
3544     }
3545
3546     if ($conf->{vmgenid}) {
3547         push @$devices, '-device', 'vmgenid,guid='.$conf->{vmgenid};
3548     }
3549
3550     my ($ovmf_code, $ovmf_vars) = get_ovmf_files($arch);
3551     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3552         die "uefi base image not found\n" if ! -f $ovmf_code;
3553
3554         my $path;
3555         my $format;
3556         if (my $efidisk = $conf->{efidisk0}) {
3557             my $d = PVE::JSONSchema::parse_property_string($efidisk_fmt, $efidisk);
3558             my ($storeid, $volname) = PVE::Storage::parse_volume_id($d->{file}, 1);
3559             $format = $d->{format};
3560             if ($storeid) {
3561                 $path = PVE::Storage::path($storecfg, $d->{file});
3562                 if (!defined($format)) {
3563                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
3564                     $format = qemu_img_format($scfg, $volname);
3565                 }
3566             } else {
3567                 $path = $d->{file};
3568                 die "efidisk format must be specified\n"
3569                     if !defined($format);
3570             }
3571         } else {
3572             warn "no efidisk configured! Using temporary efivars disk.\n";
3573             $path = "/tmp/$vmid-ovmf.fd";
3574             PVE::Tools::file_copy($ovmf_vars, $path, -s $ovmf_vars);
3575             $format = 'raw';
3576         }
3577
3578         push @$cmd, '-drive', "if=pflash,unit=0,format=raw,readonly,file=$ovmf_code";
3579         push @$cmd, '-drive', "if=pflash,unit=1,format=$format,id=drive-efidisk0,file=$path";
3580     }
3581
3582     # load q35 config
3583     if ($q35) {
3584         # we use different pcie-port hardware for qemu >= 4.0 for passthrough
3585         if (min_version($machine_version, 4, 0)) {
3586             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35-4.0.cfg';
3587         } else {
3588             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35.cfg';
3589         }
3590     }
3591
3592     # add usb controllers
3593     my @usbcontrollers = PVE::QemuServer::USB::get_usb_controllers($conf, $bridges, $arch, $machine_type, $usbdesc->{format}, $MAX_USB_DEVICES);
3594     push @$devices, @usbcontrollers if @usbcontrollers;
3595     my $vga = parse_vga($conf->{vga});
3596
3597     my $qxlnum = vga_conf_has_spice($conf->{vga});
3598     $vga->{type} = 'qxl' if $qxlnum;
3599
3600     if (!$vga->{type}) {
3601         if ($arch eq 'aarch64') {
3602             $vga->{type} = 'virtio';
3603         } elsif (min_version($machine_version, 2, 9)) {
3604             $vga->{type} = (!$winversion || $winversion >= 6) ? 'std' : 'cirrus';
3605         } else {
3606             $vga->{type} = ($winversion >= 6) ? 'std' : 'cirrus';
3607         }
3608     }
3609
3610     # enable absolute mouse coordinates (needed by vnc)
3611     my $tablet;
3612     if (defined($conf->{tablet})) {
3613         $tablet = $conf->{tablet};
3614     } else {
3615         $tablet = $defaults->{tablet};
3616         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
3617         $tablet = 0 if $vga->{type} =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
3618     }
3619
3620     if ($tablet) {
3621         push @$devices, '-device', print_tabletdevice_full($conf, $arch) if $tablet;
3622         my $kbd = print_keyboarddevice_full($conf, $arch);
3623         push @$devices, '-device', $kbd if defined($kbd);
3624     }
3625
3626     my $kvm_off = 0;
3627     my $gpu_passthrough;
3628
3629     # host pci devices
3630     for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
3631         my $id = "hostpci$i";
3632         my $d = parse_hostpci($conf->{$id});
3633         next if !$d;
3634
3635         if (my $pcie = $d->{pcie}) {
3636             die "q35 machine model is not enabled" if !$q35;
3637             # win7 wants to have the pcie devices directly on the pcie bus
3638             # instead of in the root port
3639             if ($winversion == 7) {
3640                 $pciaddr = print_pcie_addr("${id}bus0");
3641             } else {
3642                 # add more root ports if needed, 4 are present by default
3643                 # by pve-q35 cfgs, rest added here on demand.
3644                 if ($i > 3) {
3645                     push @$devices, '-device', print_pcie_root_port($i);
3646                 }
3647                 $pciaddr = print_pcie_addr($id);
3648             }
3649         } else {
3650             $pciaddr = print_pci_addr($id, $bridges, $arch, $machine_type);
3651         }
3652
3653         my $xvga = '';
3654         if ($d->{'x-vga'}) {
3655             $xvga = ',x-vga=on' if !($conf->{bios} && $conf->{bios} eq 'ovmf');
3656             $kvm_off = 1;
3657             $vga->{type} = 'none' if !defined($conf->{vga});
3658             $gpu_passthrough = 1;
3659         }
3660
3661         my $pcidevices = $d->{pciid};
3662         my $multifunction = 1 if @$pcidevices > 1;
3663
3664         my $sysfspath;
3665         if ($d->{mdev} && scalar(@$pcidevices) == 1) {
3666             my $pci_id = $pcidevices->[0]->{id};
3667             my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $i);
3668             $sysfspath = "/sys/bus/pci/devices/$pci_id/$uuid";
3669         } elsif ($d->{mdev}) {
3670             warn "ignoring mediated device '$id' with multifunction device\n";
3671         }
3672
3673         my $j=0;
3674         foreach my $pcidevice (@$pcidevices) {
3675             my $devicestr = "vfio-pci";
3676
3677             if ($sysfspath) {
3678                 $devicestr .= ",sysfsdev=$sysfspath";
3679             } else {
3680                 $devicestr .= ",host=$pcidevice->{id}";
3681             }
3682
3683             my $mf_addr = $multifunction ? ".$j" : '';
3684             $devicestr .= ",id=${id}${mf_addr}${pciaddr}${mf_addr}";
3685
3686             if ($j == 0) {
3687                 $devicestr .= ',rombar=0' if defined($d->{rombar}) && !$d->{rombar};
3688                 $devicestr .= "$xvga";
3689                 $devicestr .= ",multifunction=on" if $multifunction;
3690                 $devicestr .= ",romfile=/usr/share/kvm/$d->{romfile}" if $d->{romfile};
3691             }
3692
3693             push @$devices, '-device', $devicestr;
3694             $j++;
3695         }
3696     }
3697
3698     # usb devices
3699     my $usb_dev_features = {};
3700     $usb_dev_features->{spice_usb3} = 1 if min_version($machine_version, 4, 0);
3701
3702     my @usbdevices = PVE::QemuServer::USB::get_usb_devices($conf, $usbdesc->{format}, $MAX_USB_DEVICES, $usb_dev_features);
3703     push @$devices, @usbdevices if @usbdevices;
3704     # serial devices
3705     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3706         if (my $path = $conf->{"serial$i"}) {
3707             if ($path eq 'socket') {
3708                 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
3709                 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server,nowait";
3710                 # On aarch64, serial0 is the UART device. Qemu only allows
3711                 # connecting UART devices via the '-serial' command line, as
3712                 # the device has a fixed slot on the hardware...
3713                 if ($arch eq 'aarch64' && $i == 0) {
3714                     push @$devices, '-serial', "chardev:serial$i";
3715                 } else {
3716                     push @$devices, '-device', "isa-serial,chardev=serial$i";
3717                 }
3718             } else {
3719                 die "no such serial device\n" if ! -c $path;
3720                 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
3721                 push @$devices, '-device', "isa-serial,chardev=serial$i";
3722             }
3723         }
3724     }
3725
3726     # parallel devices
3727     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
3728         if (my $path = $conf->{"parallel$i"}) {
3729             die "no such parallel device\n" if ! -c $path;
3730             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
3731             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
3732             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
3733         }
3734     }
3735
3736     if (my $audio = conf_has_audio($conf)) {
3737
3738         my $audiopciaddr = print_pci_addr("audio0", $bridges, $arch, $machine_type);
3739
3740         my $id = $audio->{dev_id};
3741         if ($audio->{dev} eq 'AC97') {
3742             push @$devices, '-device', "AC97,id=${id}${audiopciaddr}";
3743         } elsif ($audio->{dev} =~ /intel\-hda$/) {
3744             push @$devices, '-device', "$audio->{dev},id=${id}${audiopciaddr}";
3745             push @$devices, '-device', "hda-micro,id=${id}-codec0,bus=${id}.0,cad=0";
3746             push @$devices, '-device', "hda-duplex,id=${id}-codec1,bus=${id}.0,cad=1";
3747         } else {
3748             die "unkown audio device '$audio->{dev}', implement me!";
3749         }
3750
3751         push @$devices, '-audiodev', "$audio->{backend},id=$audio->{backend_id}";
3752     }
3753
3754     my $sockets = 1;
3755     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3756     $sockets = $conf->{sockets} if  $conf->{sockets};
3757
3758     my $cores = $conf->{cores} || 1;
3759
3760     my $maxcpus = $sockets * $cores;
3761
3762     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
3763
3764     my $allowed_vcpus = $cpuinfo->{cpus};
3765
3766     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n"
3767         if ($allowed_vcpus < $maxcpus);
3768
3769     if($hotplug_features->{cpu} && min_version($machine_version, 2, 7)) {
3770
3771         push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3772         for (my $i = 2; $i <= $vcpus; $i++)  {
3773             my $cpustr = print_cpu_device($conf,$i);
3774             push @$cmd, '-device', $cpustr;
3775         }
3776
3777     } else {
3778
3779         push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3780     }
3781     push @$cmd, '-nodefaults';
3782
3783     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
3784
3785     my $bootindex_hash = {};
3786     my $i = 1;
3787     foreach my $o (split(//, $bootorder)) {
3788         $bootindex_hash->{$o} = $i*100;
3789         $i++;
3790     }
3791
3792     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
3793
3794     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
3795
3796     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
3797
3798     if ($vga->{type} && $vga->{type} !~ m/^serial\d+$/ && $vga->{type} ne 'none'){
3799         push @$devices, '-device', print_vga_device($conf, $vga, $arch, $machine_version, $machine_type, undef, $qxlnum, $bridges);
3800         my $socket = PVE::QemuServer::Helpers::vnc_socket($vmid);
3801         push @$cmd,  '-vnc', "unix:$socket,password";
3802     } else {
3803         push @$cmd, '-vga', 'none' if $vga->{type} eq 'none';
3804         push @$cmd, '-nographic';
3805     }
3806
3807     # time drift fix
3808     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
3809
3810     my $useLocaltime = $conf->{localtime};
3811
3812     if ($winversion >= 5) { # windows
3813         $useLocaltime = 1 if !defined($conf->{localtime});
3814
3815         # use time drift fix when acpi is enabled
3816         if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
3817             $tdf = 1 if !defined($conf->{tdf});
3818         }
3819     }
3820
3821     if ($winversion >= 6) {
3822         push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3823         push @$cmd, '-no-hpet';
3824     }
3825
3826     push @$rtcFlags, 'driftfix=slew' if $tdf;
3827
3828     if (!$kvm) {
3829         push @$machineFlags, 'accel=tcg';
3830     }
3831
3832     if ($machine_type) {
3833         push @$machineFlags, "type=${machine_type}";
3834     }
3835
3836     if (($conf->{startdate}) && ($conf->{startdate} ne 'now')) {
3837         push @$rtcFlags, "base=$conf->{startdate}";
3838     } elsif ($useLocaltime) {
3839         push @$rtcFlags, 'base=localtime';
3840     }
3841
3842     push @$cmd, get_cpu_options($conf, $arch, $kvm, $kvm_off, $machine_version, $winversion, $gpu_passthrough);
3843
3844     PVE::QemuServer::Memory::config($conf, $vmid, $sockets, $cores, $defaults, $hotplug_features, $cmd);
3845
3846     push @$cmd, '-S' if $conf->{freeze};
3847
3848     push @$cmd, '-k', $conf->{keyboard} if defined($conf->{keyboard});
3849
3850     my $guest_agent = parse_guest_agent($conf);
3851
3852     if ($guest_agent->{enabled}) {
3853         my $qgasocket = PVE::QemuServer::Helpers::qmp_socket($vmid, 1);
3854         push @$devices, '-chardev', "socket,path=$qgasocket,server,nowait,id=qga0";
3855
3856         if (!$guest_agent->{type} || $guest_agent->{type} eq 'virtio') {
3857             my $pciaddr = print_pci_addr("qga0", $bridges, $arch, $machine_type);
3858             push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3859             push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3860         } elsif ($guest_agent->{type} eq 'isa') {
3861             push @$devices, '-device', "isa-serial,chardev=qga0";
3862         }
3863     }
3864
3865     my $spice_port;
3866
3867     if ($qxlnum) {
3868         if ($qxlnum > 1) {
3869             if ($winversion){
3870                 for(my $i = 1; $i < $qxlnum; $i++){
3871                     push @$devices, '-device', print_vga_device($conf, $vga, $arch, $machine_version, $machine_type, $i, $qxlnum, $bridges);
3872                 }
3873             } else {
3874                 # assume other OS works like Linux
3875                 my ($ram, $vram) = ("134217728", "67108864");
3876                 if ($vga->{memory}) {
3877                     $ram = PVE::Tools::convert_size($qxlnum*4*$vga->{memory}, 'mb' => 'b');
3878                     $vram = PVE::Tools::convert_size($qxlnum*2*$vga->{memory}, 'mb' => 'b');
3879                 }
3880                 push @$cmd, '-global', "qxl-vga.ram_size=$ram";
3881                 push @$cmd, '-global', "qxl-vga.vram_size=$vram";
3882             }
3883         }
3884
3885         my $pciaddr = print_pci_addr("spice", $bridges, $arch, $machine_type);
3886
3887         my $nodename = PVE::INotify::nodename();
3888         my $pfamily = PVE::Tools::get_host_address_family($nodename);
3889         my @nodeaddrs = PVE::Tools::getaddrinfo_all('localhost', family => $pfamily);
3890         die "failed to get an ip address of type $pfamily for 'localhost'\n" if !@nodeaddrs;
3891
3892         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3893         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3894         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3895
3896         my $localhost = PVE::Network::addr_to_ip($nodeaddrs[0]->{addr});
3897         $spice_port = PVE::Tools::next_spice_port($pfamily, $localhost);
3898
3899         my $spice_enhancement = PVE::JSONSchema::parse_property_string($spice_enhancements_fmt, $conf->{spice_enhancements} // '');
3900         if ($spice_enhancement->{foldersharing}) {
3901             push @$devices, '-chardev', "spiceport,id=foldershare,name=org.spice-space.webdav.0";
3902             push @$devices, '-device', "virtserialport,chardev=foldershare,name=org.spice-space.webdav.0";
3903         }
3904
3905         my $spice_opts = "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";
3906         $spice_opts .= ",streaming-video=$spice_enhancement->{videostreaming}" if $spice_enhancement->{videostreaming};
3907         push @$devices, '-spice', "$spice_opts";
3908     }
3909
3910     # enable balloon by default, unless explicitly disabled
3911     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3912         $pciaddr = print_pci_addr("balloon0", $bridges, $arch, $machine_type);
3913         push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
3914     }
3915
3916     if ($conf->{watchdog}) {
3917         my $wdopts = parse_watchdog($conf->{watchdog});
3918         $pciaddr = print_pci_addr("watchdog", $bridges, $arch, $machine_type);
3919         my $watchdog = $wdopts->{model} || 'i6300esb';
3920         push @$devices, '-device', "$watchdog$pciaddr";
3921         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3922     }
3923
3924     my $vollist = [];
3925     my $scsicontroller = {};
3926     my $ahcicontroller = {};
3927     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3928
3929     # Add iscsi initiator name if available
3930     if (my $initiator = get_initiator_name()) {
3931         push @$devices, '-iscsi', "initiator-name=$initiator";
3932     }
3933
3934     foreach_drive($conf, sub {
3935         my ($ds, $drive) = @_;
3936
3937         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3938             push @$vollist, $drive->{file};
3939         }
3940
3941         # ignore efidisk here, already added in bios/fw handling code above
3942         return if $drive->{interface} eq 'efidisk';
3943
3944         $use_virtio = 1 if $ds =~ m/^virtio/;
3945
3946         if (drive_is_cdrom ($drive)) {
3947             if ($bootindex_hash->{d}) {
3948                 $drive->{bootindex} = $bootindex_hash->{d};
3949                 $bootindex_hash->{d} += 1;
3950             }
3951         } else {
3952             if ($bootindex_hash->{c}) {
3953                 $drive->{bootindex} = $bootindex_hash->{c} if $conf->{bootdisk} && ($conf->{bootdisk} eq $ds);
3954                 $bootindex_hash->{c} += 1;
3955             }
3956         }
3957
3958         if($drive->{interface} eq 'virtio'){
3959            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3960         }
3961
3962         if ($drive->{interface} eq 'scsi') {
3963
3964             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
3965
3966             $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges, $arch, $machine_type);
3967             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
3968
3969             my $iothread = '';
3970             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
3971                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3972                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3973             } elsif ($drive->{iothread}) {
3974                 warn "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n";
3975             }
3976
3977             my $queues = '';
3978             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
3979                 $queues = ",num_queues=$drive->{queues}";
3980             }
3981
3982             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues" if !$scsicontroller->{$controller};
3983             $scsicontroller->{$controller}=1;
3984         }
3985
3986         if ($drive->{interface} eq 'sata') {
3987            my $controller = int($drive->{index} / $MAX_SATA_DISKS);
3988            $pciaddr = print_pci_addr("ahci$controller", $bridges, $arch, $machine_type);
3989            push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr" if !$ahcicontroller->{$controller};
3990            $ahcicontroller->{$controller}=1;
3991         }
3992
3993         my $drive_cmd = print_drive_full($storecfg, $vmid, $drive);
3994         push @$devices, '-drive',$drive_cmd;
3995         push @$devices, '-device', print_drivedevice_full($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type);
3996     });
3997
3998     for (my $i = 0; $i < $MAX_NETS; $i++) {
3999          next if !$conf->{"net$i"};
4000          my $d = parse_net($conf->{"net$i"});
4001          next if !$d;
4002
4003          $use_virtio = 1 if $d->{model} eq 'virtio';
4004
4005          if ($bootindex_hash->{n}) {
4006             $d->{bootindex} = $bootindex_hash->{n};
4007             $bootindex_hash->{n} += 1;
4008          }
4009
4010          my $netdevfull = print_netdev_full($vmid, $conf, $arch, $d, "net$i");
4011          push @$devices, '-netdev', $netdevfull;
4012
4013          my $netdevicefull = print_netdevice_full($vmid, $conf, $d, "net$i", $bridges, $use_old_bios_files, $arch, $machine_type);
4014          push @$devices, '-device', $netdevicefull;
4015     }
4016
4017     if ($conf->{ivshmem}) {
4018         my $ivshmem = PVE::JSONSchema::parse_property_string($ivshmem_fmt, $conf->{ivshmem});
4019
4020         my $bus;
4021         if ($q35) {
4022             $bus = print_pcie_addr("ivshmem");
4023         } else {
4024             $bus = print_pci_addr("ivshmem", $bridges, $arch, $machine_type);
4025         }
4026
4027         my $ivshmem_name = $ivshmem->{name} // $vmid;
4028         my $path = '/dev/shm/pve-shm-' . $ivshmem_name;
4029
4030         push @$devices, '-device', "ivshmem-plain,memdev=ivshmem$bus,";
4031         push @$devices, '-object', "memory-backend-file,id=ivshmem,share=on,mem-path=$path,size=$ivshmem->{size}M";
4032     }
4033
4034     if (!$q35) {
4035         # add pci bridges
4036         if (min_version($machine_version, 2, 3)) {
4037            $bridges->{1} = 1;
4038            $bridges->{2} = 1;
4039         }
4040
4041         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
4042
4043         for my $k (sort {$b cmp $a} keys %$bridges) {
4044             $pciaddr = print_pci_addr("pci.$k", undef, $arch, $machine_type);
4045             unshift @$devices, '-device', "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr" if $k > 0;
4046         }
4047     }
4048
4049     push @$cmd, @$devices;
4050     push @$cmd, '-rtc', join(',', @$rtcFlags)
4051         if scalar(@$rtcFlags);
4052     push @$cmd, '-machine', join(',', @$machineFlags)
4053         if scalar(@$machineFlags);
4054     push @$cmd, '-global', join(',', @$globalFlags)
4055         if scalar(@$globalFlags);
4056
4057     if (my $vmstate = $conf->{vmstate}) {
4058         my $statepath = PVE::Storage::path($storecfg, $vmstate);
4059         push @$vollist, $vmstate;
4060         push @$cmd, '-loadstate', $statepath;
4061     }
4062
4063     # add custom args
4064     if ($conf->{args}) {
4065         my $aa = PVE::Tools::split_args($conf->{args});
4066         push @$cmd, @$aa;
4067     }
4068
4069     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
4070 }
4071
4072 sub spice_port {
4073     my ($vmid) = @_;
4074
4075     my $res = mon_cmd($vmid, 'query-spice');
4076
4077     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
4078 }
4079
4080 sub vm_devices_list {
4081     my ($vmid) = @_;
4082
4083     my $res = mon_cmd($vmid, 'query-pci');
4084     my $devices_to_check = [];
4085     my $devices = {};
4086     foreach my $pcibus (@$res) {
4087         push @$devices_to_check, @{$pcibus->{devices}},
4088     }
4089
4090     while (@$devices_to_check) {
4091         my $to_check = [];
4092         for my $d (@$devices_to_check) {
4093             $devices->{$d->{'qdev_id'}} = 1 if $d->{'qdev_id'};
4094             next if !$d->{'pci_bridge'};
4095
4096             $devices->{$d->{'qdev_id'}} += scalar(@{$d->{'pci_bridge'}->{devices}});
4097             push @$to_check, @{$d->{'pci_bridge'}->{devices}};
4098         }
4099         $devices_to_check = $to_check;
4100     }
4101
4102     my $resblock = mon_cmd($vmid, 'query-block');
4103     foreach my $block (@$resblock) {
4104         if($block->{device} =~ m/^drive-(\S+)/){
4105                 $devices->{$1} = 1;
4106         }
4107     }
4108
4109     my $resmice = mon_cmd($vmid, 'query-mice');
4110     foreach my $mice (@$resmice) {
4111         if ($mice->{name} eq 'QEMU HID Tablet') {
4112             $devices->{tablet} = 1;
4113             last;
4114         }
4115     }
4116
4117     # for usb devices there is no query-usb
4118     # but we can iterate over the entries in
4119     # qom-list path=/machine/peripheral
4120     my $resperipheral = mon_cmd($vmid, 'qom-list', path => '/machine/peripheral');
4121     foreach my $per (@$resperipheral) {
4122         if ($per->{name} =~ m/^usb\d+$/) {
4123             $devices->{$per->{name}} = 1;
4124         }
4125     }
4126
4127     return $devices;
4128 }
4129
4130 sub vm_deviceplug {
4131     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4132
4133     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
4134
4135     my $devices_list = vm_devices_list($vmid);
4136     return 1 if defined($devices_list->{$deviceid});
4137
4138     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid, $arch, $machine_type); # add PCI bridge if we need it for the device
4139
4140     if ($deviceid eq 'tablet') {
4141
4142         qemu_deviceadd($vmid, print_tabletdevice_full($conf, $arch));
4143
4144     } elsif ($deviceid eq 'keyboard') {
4145
4146         qemu_deviceadd($vmid, print_keyboarddevice_full($conf, $arch));
4147
4148     } elsif ($deviceid =~ m/^usb(\d+)$/) {
4149
4150         die "usb hotplug currently not reliable\n";
4151         # since we can't reliably hot unplug all added usb devices
4152         # and usb passthrough disables live migration
4153         # we disable usb hotplugging for now
4154         qemu_deviceadd($vmid, PVE::QemuServer::USB::print_usbdevice_full($conf, $deviceid, $device));
4155
4156     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4157
4158         qemu_iothread_add($vmid, $deviceid, $device);
4159
4160         qemu_driveadd($storecfg, $vmid, $device);
4161         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, $arch, $machine_type);
4162
4163         qemu_deviceadd($vmid, $devicefull);
4164         eval { qemu_deviceaddverify($vmid, $deviceid); };
4165         if (my $err = $@) {
4166             eval { qemu_drivedel($vmid, $deviceid); };
4167             warn $@ if $@;
4168             die $err;
4169         }
4170
4171     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4172
4173
4174         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
4175         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
4176         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
4177
4178         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
4179
4180         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
4181             qemu_iothread_add($vmid, $deviceid, $device);
4182             $devicefull .= ",iothread=iothread-$deviceid";
4183         }
4184
4185         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
4186             $devicefull .= ",num_queues=$device->{queues}";
4187         }
4188
4189         qemu_deviceadd($vmid, $devicefull);
4190         qemu_deviceaddverify($vmid, $deviceid);
4191
4192     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4193
4194         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device, $arch, $machine_type);
4195         qemu_driveadd($storecfg, $vmid, $device);
4196
4197         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, $arch, $machine_type);
4198         eval { qemu_deviceadd($vmid, $devicefull); };
4199         if (my $err = $@) {
4200             eval { qemu_drivedel($vmid, $deviceid); };
4201             warn $@ if $@;
4202             die $err;
4203         }
4204
4205     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4206
4207         return undef if !qemu_netdevadd($vmid, $conf, $arch, $device, $deviceid);
4208
4209         my $machine_type = PVE::QemuServer::Machine::qemu_machine_pxe($vmid, $conf);
4210         my $use_old_bios_files = undef;
4211         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
4212
4213         my $netdevicefull = print_netdevice_full($vmid, $conf, $device, $deviceid, undef, $use_old_bios_files, $arch, $machine_type);
4214         qemu_deviceadd($vmid, $netdevicefull);
4215         eval {
4216             qemu_deviceaddverify($vmid, $deviceid);
4217             qemu_set_link_status($vmid, $deviceid, !$device->{link_down});
4218         };
4219         if (my $err = $@) {
4220             eval { qemu_netdevdel($vmid, $deviceid); };
4221             warn $@ if $@;
4222             die $err;
4223         }
4224
4225     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
4226
4227         my $bridgeid = $2;
4228         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
4229         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
4230
4231         qemu_deviceadd($vmid, $devicefull);
4232         qemu_deviceaddverify($vmid, $deviceid);
4233
4234     } else {
4235         die "can't hotplug device '$deviceid'\n";
4236     }
4237
4238     return 1;
4239 }
4240
4241 # fixme: this should raise exceptions on error!
4242 sub vm_deviceunplug {
4243     my ($vmid, $conf, $deviceid) = @_;
4244
4245     my $devices_list = vm_devices_list($vmid);
4246     return 1 if !defined($devices_list->{$deviceid});
4247
4248     die "can't unplug bootdisk" if $conf->{bootdisk} && $conf->{bootdisk} eq $deviceid;
4249
4250     if ($deviceid eq 'tablet' || $deviceid eq 'keyboard') {
4251
4252         qemu_devicedel($vmid, $deviceid);
4253
4254     } elsif ($deviceid =~ m/^usb\d+$/) {
4255
4256         die "usb hotplug currently not reliable\n";
4257         # when unplugging usb devices this way,
4258         # there may be remaining usb controllers/hubs
4259         # so we disable it for now
4260         qemu_devicedel($vmid, $deviceid);
4261         qemu_devicedelverify($vmid, $deviceid);
4262
4263     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4264
4265         qemu_devicedel($vmid, $deviceid);
4266         qemu_devicedelverify($vmid, $deviceid);
4267         qemu_drivedel($vmid, $deviceid);
4268         qemu_iothread_del($conf, $vmid, $deviceid);
4269
4270     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4271
4272         qemu_devicedel($vmid, $deviceid);
4273         qemu_devicedelverify($vmid, $deviceid);
4274         qemu_iothread_del($conf, $vmid, $deviceid);
4275
4276     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4277
4278         qemu_devicedel($vmid, $deviceid);
4279         qemu_drivedel($vmid, $deviceid);
4280         qemu_deletescsihw($conf, $vmid, $deviceid);
4281
4282     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4283
4284         qemu_devicedel($vmid, $deviceid);
4285         qemu_devicedelverify($vmid, $deviceid);
4286         qemu_netdevdel($vmid, $deviceid);
4287
4288     } else {
4289         die "can't unplug device '$deviceid'\n";
4290     }
4291
4292     return 1;
4293 }
4294
4295 sub qemu_deviceadd {
4296     my ($vmid, $devicefull) = @_;
4297
4298     $devicefull = "driver=".$devicefull;
4299     my %options =  split(/[=,]/, $devicefull);
4300
4301     mon_cmd($vmid, "device_add" , %options);
4302 }
4303
4304 sub qemu_devicedel {
4305     my ($vmid, $deviceid) = @_;
4306
4307     my $ret = mon_cmd($vmid, "device_del", id => $deviceid);
4308 }
4309
4310 sub qemu_iothread_add {
4311     my($vmid, $deviceid, $device) = @_;
4312
4313     if ($device->{iothread}) {
4314         my $iothreads = vm_iothreads_list($vmid);
4315         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
4316     }
4317 }
4318
4319 sub qemu_iothread_del {
4320     my($conf, $vmid, $deviceid) = @_;
4321
4322     my $confid = $deviceid;
4323     if ($deviceid =~ m/^(?:virtioscsi|scsihw)(\d+)$/) {
4324         $confid = 'scsi' . $1;
4325     }
4326     my $device = parse_drive($confid, $conf->{$confid});
4327     if ($device->{iothread}) {
4328         my $iothreads = vm_iothreads_list($vmid);
4329         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
4330     }
4331 }
4332
4333 sub qemu_objectadd {
4334     my($vmid, $objectid, $qomtype) = @_;
4335
4336     mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
4337
4338     return 1;
4339 }
4340
4341 sub qemu_objectdel {
4342     my($vmid, $objectid) = @_;
4343
4344     mon_cmd($vmid, "object-del", id => $objectid);
4345
4346     return 1;
4347 }
4348
4349 sub qemu_driveadd {
4350     my ($storecfg, $vmid, $device) = @_;
4351
4352     my $drive = print_drive_full($storecfg, $vmid, $device);
4353     $drive =~ s/\\/\\\\/g;
4354     my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_add auto \"$drive\"");
4355
4356     # If the command succeeds qemu prints: "OK"
4357     return 1 if $ret =~ m/OK/s;
4358
4359     die "adding drive failed: $ret\n";
4360 }
4361
4362 sub qemu_drivedel {
4363     my($vmid, $deviceid) = @_;
4364
4365     my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_del drive-$deviceid");
4366     $ret =~ s/^\s+//;
4367
4368     return 1 if $ret eq "";
4369
4370     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
4371     return 1 if $ret =~ m/Device \'.*?\' not found/s;
4372
4373     die "deleting drive $deviceid failed : $ret\n";
4374 }
4375
4376 sub qemu_deviceaddverify {
4377     my ($vmid, $deviceid) = @_;
4378
4379     for (my $i = 0; $i <= 5; $i++) {
4380          my $devices_list = vm_devices_list($vmid);
4381          return 1 if defined($devices_list->{$deviceid});
4382          sleep 1;
4383     }
4384
4385     die "error on hotplug device '$deviceid'\n";
4386 }
4387
4388
4389 sub qemu_devicedelverify {
4390     my ($vmid, $deviceid) = @_;
4391
4392     # need to verify that the device is correctly removed as device_del
4393     # is async and empty return is not reliable
4394
4395     for (my $i = 0; $i <= 5; $i++) {
4396          my $devices_list = vm_devices_list($vmid);
4397          return 1 if !defined($devices_list->{$deviceid});
4398          sleep 1;
4399     }
4400
4401     die "error on hot-unplugging device '$deviceid'\n";
4402 }
4403
4404 sub qemu_findorcreatescsihw {
4405     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4406
4407     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4408
4409     my $scsihwid="$controller_prefix$controller";
4410     my $devices_list = vm_devices_list($vmid);
4411
4412     if(!defined($devices_list->{$scsihwid})) {
4413         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device, $arch, $machine_type);
4414     }
4415
4416     return 1;
4417 }
4418
4419 sub qemu_deletescsihw {
4420     my ($conf, $vmid, $opt) = @_;
4421
4422     my $device = parse_drive($opt, $conf->{$opt});
4423
4424     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
4425         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
4426         return 1;
4427     }
4428
4429     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4430
4431     my $devices_list = vm_devices_list($vmid);
4432     foreach my $opt (keys %{$devices_list}) {
4433         if (PVE::QemuServer::is_valid_drivename($opt)) {
4434             my $drive = PVE::QemuServer::parse_drive($opt, $conf->{$opt});
4435             if($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
4436                 return 1;
4437             }
4438         }
4439     }
4440
4441     my $scsihwid="scsihw$controller";
4442
4443     vm_deviceunplug($vmid, $conf, $scsihwid);
4444
4445     return 1;
4446 }
4447
4448 sub qemu_add_pci_bridge {
4449     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4450
4451     my $bridges = {};
4452
4453     my $bridgeid;
4454
4455     print_pci_addr($device, $bridges, $arch, $machine_type);
4456
4457     while (my ($k, $v) = each %$bridges) {
4458         $bridgeid = $k;
4459     }
4460     return 1 if !defined($bridgeid) || $bridgeid < 1;
4461
4462     my $bridge = "pci.$bridgeid";
4463     my $devices_list = vm_devices_list($vmid);
4464
4465     if (!defined($devices_list->{$bridge})) {
4466         vm_deviceplug($storecfg, $conf, $vmid, $bridge, $arch, $machine_type);
4467     }
4468
4469     return 1;
4470 }
4471
4472 sub qemu_set_link_status {
4473     my ($vmid, $device, $up) = @_;
4474
4475     mon_cmd($vmid, "set_link", name => $device,
4476                up => $up ? JSON::true : JSON::false);
4477 }
4478
4479 sub qemu_netdevadd {
4480     my ($vmid, $conf, $arch, $device, $deviceid) = @_;
4481
4482     my $netdev = print_netdev_full($vmid, $conf, $arch, $device, $deviceid, 1);
4483     my %options =  split(/[=,]/, $netdev);
4484
4485     mon_cmd($vmid, "netdev_add",  %options);
4486     return 1;
4487 }
4488
4489 sub qemu_netdevdel {
4490     my ($vmid, $deviceid) = @_;
4491
4492     mon_cmd($vmid, "netdev_del", id => $deviceid);
4493 }
4494
4495 sub qemu_usb_hotplug {
4496     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4497
4498     return if !$device;
4499
4500     # remove the old one first
4501     vm_deviceunplug($vmid, $conf, $deviceid);
4502
4503     # check if xhci controller is necessary and available
4504     if ($device->{usb3}) {
4505
4506         my $devicelist = vm_devices_list($vmid);
4507
4508         if (!$devicelist->{xhci}) {
4509             my $pciaddr = print_pci_addr("xhci", undef, $arch, $machine_type);
4510             qemu_deviceadd($vmid, "nec-usb-xhci,id=xhci$pciaddr");
4511         }
4512     }
4513     my $d = parse_usb_device($device->{host});
4514     $d->{usb3} = $device->{usb3};
4515
4516     # add the new one
4517     vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $d, $arch, $machine_type);
4518 }
4519
4520 sub qemu_cpu_hotplug {
4521     my ($vmid, $conf, $vcpus) = @_;
4522
4523     my $machine_type = PVE::QemuServer::Machine::get_current_qemu_machine($vmid);
4524
4525     my $sockets = 1;
4526     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
4527     $sockets = $conf->{sockets} if  $conf->{sockets};
4528     my $cores = $conf->{cores} || 1;
4529     my $maxcpus = $sockets * $cores;
4530
4531     $vcpus = $maxcpus if !$vcpus;
4532
4533     die "you can't add more vcpus than maxcpus\n"
4534         if $vcpus > $maxcpus;
4535
4536     my $currentvcpus = $conf->{vcpus} || $maxcpus;
4537
4538     if ($vcpus < $currentvcpus) {
4539
4540         if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4541
4542             for (my $i = $currentvcpus; $i > $vcpus; $i--) {
4543                 qemu_devicedel($vmid, "cpu$i");
4544                 my $retry = 0;
4545                 my $currentrunningvcpus = undef;
4546                 while (1) {
4547                     $currentrunningvcpus = mon_cmd($vmid, "query-cpus");
4548                     last if scalar(@{$currentrunningvcpus}) == $i-1;
4549                     raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
4550                     $retry++;
4551                     sleep 1;
4552                 }
4553                 #update conf after each succesfull cpu unplug
4554                 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4555                 PVE::QemuConfig->write_config($vmid, $conf);
4556             }
4557         } else {
4558             die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
4559         }
4560
4561         return;
4562     }
4563
4564     my $currentrunningvcpus = mon_cmd($vmid, "query-cpus");
4565     die "vcpus in running vm does not match its configuration\n"
4566         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
4567
4568     if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4569
4570         for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
4571             my $cpustr = print_cpu_device($conf, $i);
4572             qemu_deviceadd($vmid, $cpustr);
4573
4574             my $retry = 0;
4575             my $currentrunningvcpus = undef;
4576             while (1) {
4577                 $currentrunningvcpus = mon_cmd($vmid, "query-cpus");
4578                 last if scalar(@{$currentrunningvcpus}) == $i;
4579                 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
4580                 sleep 1;
4581                 $retry++;
4582             }
4583             #update conf after each succesfull cpu hotplug
4584             $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4585             PVE::QemuConfig->write_config($vmid, $conf);
4586         }
4587     } else {
4588
4589         for (my $i = $currentvcpus; $i < $vcpus; $i++) {
4590             mon_cmd($vmid, "cpu-add", id => int($i));
4591         }
4592     }
4593 }
4594
4595 sub qemu_block_set_io_throttle {
4596     my ($vmid, $deviceid,
4597         $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
4598         $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max,
4599         $bps_max_length, $bps_rd_max_length, $bps_wr_max_length,
4600         $iops_max_length, $iops_rd_max_length, $iops_wr_max_length) = @_;
4601
4602     return if !check_running($vmid) ;
4603
4604     mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
4605         bps => int($bps),
4606         bps_rd => int($bps_rd),
4607         bps_wr => int($bps_wr),
4608         iops => int($iops),
4609         iops_rd => int($iops_rd),
4610         iops_wr => int($iops_wr),
4611         bps_max => int($bps_max),
4612         bps_rd_max => int($bps_rd_max),
4613         bps_wr_max => int($bps_wr_max),
4614         iops_max => int($iops_max),
4615         iops_rd_max => int($iops_rd_max),
4616         iops_wr_max => int($iops_wr_max),
4617         bps_max_length => int($bps_max_length),
4618         bps_rd_max_length => int($bps_rd_max_length),
4619         bps_wr_max_length => int($bps_wr_max_length),
4620         iops_max_length => int($iops_max_length),
4621         iops_rd_max_length => int($iops_rd_max_length),
4622         iops_wr_max_length => int($iops_wr_max_length),
4623     );
4624
4625 }
4626
4627 # old code, only used to shutdown old VM after update
4628 sub __read_avail {
4629     my ($fh, $timeout) = @_;
4630
4631     my $sel = new IO::Select;
4632     $sel->add($fh);
4633
4634     my $res = '';
4635     my $buf;
4636
4637     my @ready;
4638     while (scalar (@ready = $sel->can_read($timeout))) {
4639         my $count;
4640         if ($count = $fh->sysread($buf, 8192)) {
4641             if ($buf =~ /^(.*)\(qemu\) $/s) {
4642                 $res .= $1;
4643                 last;
4644             } else {
4645                 $res .= $buf;
4646             }
4647         } else {
4648             if (!defined($count)) {
4649                 die "$!\n";
4650             }
4651             last;
4652         }
4653     }
4654
4655     die "monitor read timeout\n" if !scalar(@ready);
4656
4657     return $res;
4658 }
4659
4660 sub qemu_block_resize {
4661     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4662
4663     my $running = check_running($vmid);
4664
4665     $size = 0 if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4666
4667     return if !$running;
4668
4669     mon_cmd($vmid, "block_resize", device => $deviceid, size => int($size));
4670
4671 }
4672
4673 sub qemu_volume_snapshot {
4674     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4675
4676     my $running = check_running($vmid);
4677
4678     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4679         mon_cmd($vmid, 'blockdev-snapshot-internal-sync', device => $deviceid, name => $snap);
4680     } else {
4681         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4682     }
4683 }
4684
4685 sub qemu_volume_snapshot_delete {
4686     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4687
4688     my $running = check_running($vmid);
4689
4690     if($running) {
4691
4692         $running = undef;
4693         my $conf = PVE::QemuConfig->load_config($vmid);
4694         foreach_drive($conf, sub {
4695             my ($ds, $drive) = @_;
4696             $running = 1 if $drive->{file} eq $volid;
4697         });
4698     }
4699
4700     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4701         mon_cmd($vmid, 'blockdev-snapshot-delete-internal-sync', device => $deviceid, name => $snap);
4702     } else {
4703         PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4704     }
4705 }
4706
4707 sub set_migration_caps {
4708     my ($vmid) = @_;
4709
4710     my $cap_ref = [];
4711
4712     my $enabled_cap = {
4713         "auto-converge" => 1,
4714         "xbzrle" => 1,
4715         "x-rdma-pin-all" => 0,
4716         "zero-blocks" => 0,
4717         "compress" => 0
4718     };
4719
4720     my $supported_capabilities = mon_cmd($vmid, "query-migrate-capabilities");
4721
4722     for my $supported_capability (@$supported_capabilities) {
4723         push @$cap_ref, {
4724             capability => $supported_capability->{capability},
4725             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4726         };
4727     }
4728
4729     mon_cmd($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4730 }
4731
4732 my $fast_plug_option = {
4733     'lock' => 1,
4734     'name' => 1,
4735     'onboot' => 1,
4736     'shares' => 1,
4737     'startup' => 1,
4738     'description' => 1,
4739     'protection' => 1,
4740     'vmstatestorage' => 1,
4741     'hookscript' => 1,
4742     'tags' => 1,
4743 };
4744
4745 # hotplug changes in [PENDING]
4746 # $selection hash can be used to only apply specified options, for
4747 # example: { cores => 1 } (only apply changed 'cores')
4748 # $errors ref is used to return error messages
4749 sub vmconfig_hotplug_pending {
4750     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4751
4752     my $defaults = load_defaults();
4753     my $arch = get_vm_arch($conf);
4754     my $machine_type = get_vm_machine($conf, undef, $arch);
4755
4756     # commit values which do not have any impact on running VM first
4757     # Note: those option cannot raise errors, we we do not care about
4758     # $selection and always apply them.
4759
4760     my $add_error = sub {
4761         my ($opt, $msg) = @_;
4762         $errors->{$opt} = "hotplug problem - $msg";
4763     };
4764
4765     my $changes = 0;
4766     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4767         if ($fast_plug_option->{$opt}) {
4768             $conf->{$opt} = $conf->{pending}->{$opt};
4769             delete $conf->{pending}->{$opt};
4770             $changes = 1;
4771         }
4772     }
4773
4774     if ($changes) {
4775         PVE::QemuConfig->write_config($vmid, $conf);
4776         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4777     }
4778
4779     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4780
4781     my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4782     foreach my $opt (sort keys %$pending_delete_hash) {
4783         next if $selection && !$selection->{$opt};
4784         my $force = $pending_delete_hash->{$opt}->{force};
4785         eval {
4786             if ($opt eq 'hotplug') {
4787                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
4788             } elsif ($opt eq 'tablet') {
4789                 die "skip\n" if !$hotplug_features->{usb};
4790                 if ($defaults->{tablet}) {
4791                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4792                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4793                         if $arch eq 'aarch64';
4794                 } else {
4795                     vm_deviceunplug($vmid, $conf, 'tablet');
4796                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4797                 }
4798             } elsif ($opt =~ m/^usb\d+/) {
4799                 die "skip\n";
4800                 # since we cannot reliably hot unplug usb devices
4801                 # we are disabling it
4802                 die "skip\n" if !$hotplug_features->{usb} || $conf->{$opt} =~ m/spice/i;
4803                 vm_deviceunplug($vmid, $conf, $opt);
4804             } elsif ($opt eq 'vcpus') {
4805                 die "skip\n" if !$hotplug_features->{cpu};
4806                 qemu_cpu_hotplug($vmid, $conf, undef);
4807             } elsif ($opt eq 'balloon') {
4808                 # enable balloon device is not hotpluggable
4809                 die "skip\n" if defined($conf->{balloon}) && $conf->{balloon} == 0;
4810                 # here we reset the ballooning value to memory
4811                 my $balloon = $conf->{memory} || $defaults->{memory};
4812                 mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4813             } elsif ($fast_plug_option->{$opt}) {
4814                 # do nothing
4815             } elsif ($opt =~ m/^net(\d+)$/) {
4816                 die "skip\n" if !$hotplug_features->{network};
4817                 vm_deviceunplug($vmid, $conf, $opt);
4818             } elsif (is_valid_drivename($opt)) {
4819                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4820                 vm_deviceunplug($vmid, $conf, $opt);
4821                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4822             } elsif ($opt =~ m/^memory$/) {
4823                 die "skip\n" if !$hotplug_features->{memory};
4824                 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
4825             } elsif ($opt eq 'cpuunits') {
4826                 cgroups_write("cpu", $vmid, "cpu.shares", $defaults->{cpuunits});
4827             } elsif ($opt eq 'cpulimit') {
4828                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", -1);
4829             } else {
4830                 die "skip\n";
4831             }
4832         };
4833         if (my $err = $@) {
4834             &$add_error($opt, $err) if $err ne "skip\n";
4835         } else {
4836             # save new config if hotplug was successful
4837             delete $conf->{$opt};
4838             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4839             PVE::QemuConfig->write_config($vmid, $conf);
4840             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4841         }
4842     }
4843
4844     my ($apply_pending_cloudinit, $apply_pending_cloudinit_done);
4845     $apply_pending_cloudinit = sub {
4846         return if $apply_pending_cloudinit_done; # once is enough
4847         $apply_pending_cloudinit_done = 1; # once is enough
4848
4849         my ($key, $value) = @_;
4850
4851         my @cloudinit_opts = keys %$confdesc_cloudinit;
4852         foreach my $opt (keys %{$conf->{pending}}) {
4853             next if !grep { $_ eq $opt } @cloudinit_opts;
4854             $conf->{$opt} = delete $conf->{pending}->{$opt};
4855         }
4856
4857         my $new_conf = { %$conf };
4858         $new_conf->{$key} = $value;
4859         PVE::QemuServer::Cloudinit::generate_cloudinitconfig($new_conf, $vmid);
4860     };
4861
4862     foreach my $opt (keys %{$conf->{pending}}) {
4863         next if $selection && !$selection->{$opt};
4864         my $value = $conf->{pending}->{$opt};
4865         eval {
4866             if ($opt eq 'hotplug') {
4867                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
4868             } elsif ($opt eq 'tablet') {
4869                 die "skip\n" if !$hotplug_features->{usb};
4870                 if ($value == 1) {
4871                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4872                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4873                         if $arch eq 'aarch64';
4874                 } elsif ($value == 0) {
4875                     vm_deviceunplug($vmid, $conf, 'tablet');
4876                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4877                 }
4878             } elsif ($opt =~ m/^usb\d+$/) {
4879                 die "skip\n";
4880                 # since we cannot reliably hot unplug usb devices
4881                 # we are disabling it
4882                 die "skip\n" if !$hotplug_features->{usb} || $value =~ m/spice/i;
4883                 my $d = eval { PVE::JSONSchema::parse_property_string($usbdesc->{format}, $value) };
4884                 die "skip\n" if !$d;
4885                 qemu_usb_hotplug($storecfg, $conf, $vmid, $opt, $d, $arch, $machine_type);
4886             } elsif ($opt eq 'vcpus') {
4887                 die "skip\n" if !$hotplug_features->{cpu};
4888                 qemu_cpu_hotplug($vmid, $conf, $value);
4889             } elsif ($opt eq 'balloon') {
4890                 # enable/disable balloning device is not hotpluggable
4891                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
4892                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
4893                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
4894
4895                 # allow manual ballooning if shares is set to zero
4896                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
4897                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
4898                     mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4899                 }
4900             } elsif ($opt =~ m/^net(\d+)$/) {
4901                 # some changes can be done without hotplug
4902                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
4903                                     $vmid, $opt, $value, $arch, $machine_type);
4904             } elsif (is_valid_drivename($opt)) {
4905                 # some changes can be done without hotplug
4906                 my $drive = parse_drive($opt, $value);
4907                 if (drive_is_cloudinit($drive)) {
4908                     &$apply_pending_cloudinit($opt, $value);
4909                 }
4910                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
4911                                      $vmid, $opt, $value, 1, $arch, $machine_type);
4912             } elsif ($opt =~ m/^memory$/) { #dimms
4913                 die "skip\n" if !$hotplug_features->{memory};
4914                 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
4915             } elsif ($opt eq 'cpuunits') {
4916                 cgroups_write("cpu", $vmid, "cpu.shares", $conf->{pending}->{$opt});
4917             } elsif ($opt eq 'cpulimit') {
4918                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
4919                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", $cpulimit);
4920             } else {
4921                 die "skip\n";  # skip non-hot-pluggable options
4922             }
4923         };
4924         if (my $err = $@) {
4925             &$add_error($opt, $err) if $err ne "skip\n";
4926         } else {
4927             # save new config if hotplug was successful
4928             $conf->{$opt} = $value;
4929             delete $conf->{pending}->{$opt};
4930             PVE::QemuConfig->write_config($vmid, $conf);
4931             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4932         }
4933     }
4934 }
4935
4936 sub try_deallocate_drive {
4937     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
4938
4939     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
4940         my $volid = $drive->{file};
4941         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
4942             my $sid = PVE::Storage::parse_volume_id($volid);
4943             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
4944
4945             # check if the disk is really unused
4946             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
4947                 if is_volume_in_use($storecfg, $conf, $key, $volid);
4948             PVE::Storage::vdisk_free($storecfg, $volid);
4949             return 1;
4950         } else {
4951             # If vm is not owner of this disk remove from config
4952             return 1;
4953         }
4954     }
4955
4956     return undef;
4957 }
4958
4959 sub vmconfig_delete_or_detach_drive {
4960     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
4961
4962     my $drive = parse_drive($opt, $conf->{$opt});
4963
4964     my $rpcenv = PVE::RPCEnvironment::get();
4965     my $authuser = $rpcenv->get_user();
4966
4967     if ($force) {
4968         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
4969         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
4970     } else {
4971         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
4972     }
4973 }
4974
4975
4976
4977 sub vmconfig_apply_pending {
4978     my ($vmid, $conf, $storecfg) = @_;
4979
4980     # cold plug
4981
4982     my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4983     foreach my $opt (sort keys %$pending_delete_hash) {
4984         die "internal error" if $opt =~ m/^unused/;
4985         my $force = $pending_delete_hash->{$opt}->{force};
4986         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4987         if (!defined($conf->{$opt})) {
4988             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4989             PVE::QemuConfig->write_config($vmid, $conf);
4990         } elsif (is_valid_drivename($opt)) {
4991             vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4992             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4993             delete $conf->{$opt};
4994             PVE::QemuConfig->write_config($vmid, $conf);
4995         } else {
4996             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4997             delete $conf->{$opt};
4998             PVE::QemuConfig->write_config($vmid, $conf);
4999         }
5000     }
5001
5002     $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5003
5004     foreach my $opt (keys %{$conf->{pending}}) { # add/change
5005         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5006
5007         if (defined($conf->{$opt}) && ($conf->{$opt} eq $conf->{pending}->{$opt})) {
5008             # skip if nothing changed
5009         } elsif (is_valid_drivename($opt)) {
5010             vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
5011                 if defined($conf->{$opt});
5012             $conf->{$opt} = $conf->{pending}->{$opt};
5013         } else {
5014             $conf->{$opt} = $conf->{pending}->{$opt};
5015         }
5016
5017         delete $conf->{pending}->{$opt};
5018         PVE::QemuConfig->write_config($vmid, $conf);
5019     }
5020 }
5021
5022 my $safe_num_ne = sub {
5023     my ($a, $b) = @_;
5024
5025     return 0 if !defined($a) && !defined($b);
5026     return 1 if !defined($a);
5027     return 1 if !defined($b);
5028
5029     return $a != $b;
5030 };
5031
5032 my $safe_string_ne = sub {
5033     my ($a, $b) = @_;
5034
5035     return 0 if !defined($a) && !defined($b);
5036     return 1 if !defined($a);
5037     return 1 if !defined($b);
5038
5039     return $a ne $b;
5040 };
5041
5042 sub vmconfig_update_net {
5043     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
5044
5045     my $newnet = parse_net($value);
5046
5047     if ($conf->{$opt}) {
5048         my $oldnet = parse_net($conf->{$opt});
5049
5050         if (&$safe_string_ne($oldnet->{model}, $newnet->{model}) ||
5051             &$safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
5052             &$safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
5053             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
5054
5055             # for non online change, we try to hot-unplug
5056             die "skip\n" if !$hotplug;
5057             vm_deviceunplug($vmid, $conf, $opt);
5058         } else {
5059
5060             die "internal error" if $opt !~ m/net(\d+)/;
5061             my $iface = "tap${vmid}i$1";
5062
5063             if (&$safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
5064                 &$safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
5065                 &$safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
5066                 &$safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
5067                 PVE::Network::tap_unplug($iface);
5068                 PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
5069             } elsif (&$safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
5070                 # Rate can be applied on its own but any change above needs to
5071                 # include the rate in tap_plug since OVS resets everything.
5072                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
5073             }
5074
5075             if (&$safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
5076                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
5077             }
5078
5079             return 1;
5080         }
5081     }
5082
5083     if ($hotplug) {
5084         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet, $arch, $machine_type);
5085     } else {
5086         die "skip\n";
5087     }
5088 }
5089
5090 sub vmconfig_update_disk {
5091     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $force, $arch, $machine_type) = @_;
5092
5093     # fixme: do we need force?
5094
5095     my $drive = parse_drive($opt, $value);
5096
5097     if ($conf->{$opt}) {
5098
5099         if (my $old_drive = parse_drive($opt, $conf->{$opt}))  {
5100
5101             my $media = $drive->{media} || 'disk';
5102             my $oldmedia = $old_drive->{media} || 'disk';
5103             die "unable to change media type\n" if $media ne $oldmedia;
5104
5105             if (!drive_is_cdrom($old_drive)) {
5106
5107                 if ($drive->{file} ne $old_drive->{file}) {
5108
5109                     die "skip\n" if !$hotplug;
5110
5111                     # unplug and register as unused
5112                     vm_deviceunplug($vmid, $conf, $opt);
5113                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
5114
5115                 } else {
5116                     # update existing disk
5117
5118                     # skip non hotpluggable value
5119                     if (&$safe_string_ne($drive->{discard}, $old_drive->{discard}) ||
5120                         &$safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
5121                         &$safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
5122                         &$safe_string_ne($drive->{cache}, $old_drive->{cache})) {
5123                         die "skip\n";
5124                     }
5125
5126                     # apply throttle
5127                     if (&$safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
5128                         &$safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
5129                         &$safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
5130                         &$safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
5131                         &$safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
5132                         &$safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
5133                         &$safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
5134                         &$safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
5135                         &$safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
5136                         &$safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
5137                         &$safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
5138                         &$safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max}) ||
5139                         &$safe_num_ne($drive->{bps_max_length}, $old_drive->{bps_max_length}) ||
5140                         &$safe_num_ne($drive->{bps_rd_max_length}, $old_drive->{bps_rd_max_length}) ||
5141                         &$safe_num_ne($drive->{bps_wr_max_length}, $old_drive->{bps_wr_max_length}) ||
5142                         &$safe_num_ne($drive->{iops_max_length}, $old_drive->{iops_max_length}) ||
5143                         &$safe_num_ne($drive->{iops_rd_max_length}, $old_drive->{iops_rd_max_length}) ||
5144                         &$safe_num_ne($drive->{iops_wr_max_length}, $old_drive->{iops_wr_max_length})) {
5145
5146                         qemu_block_set_io_throttle($vmid,"drive-$opt",
5147                                                    ($drive->{mbps} || 0)*1024*1024,
5148                                                    ($drive->{mbps_rd} || 0)*1024*1024,
5149                                                    ($drive->{mbps_wr} || 0)*1024*1024,
5150                                                    $drive->{iops} || 0,
5151                                                    $drive->{iops_rd} || 0,
5152                                                    $drive->{iops_wr} || 0,
5153                                                    ($drive->{mbps_max} || 0)*1024*1024,
5154                                                    ($drive->{mbps_rd_max} || 0)*1024*1024,
5155                                                    ($drive->{mbps_wr_max} || 0)*1024*1024,
5156                                                    $drive->{iops_max} || 0,
5157                                                    $drive->{iops_rd_max} || 0,
5158                                                    $drive->{iops_wr_max} || 0,
5159                                                    $drive->{bps_max_length} || 1,
5160                                                    $drive->{bps_rd_max_length} || 1,
5161                                                    $drive->{bps_wr_max_length} || 1,
5162                                                    $drive->{iops_max_length} || 1,
5163                                                    $drive->{iops_rd_max_length} || 1,
5164                                                    $drive->{iops_wr_max_length} || 1);
5165
5166                     }
5167
5168                     return 1;
5169                 }
5170
5171             } else { # cdrom
5172
5173                 if ($drive->{file} eq 'none') {
5174                     mon_cmd($vmid, "eject",force => JSON::true,device => "drive-$opt");
5175                     if (drive_is_cloudinit($old_drive)) {
5176                         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive);
5177                     }
5178                 } else {
5179                     my $path = get_iso_path($storecfg, $vmid, $drive->{file});
5180                     mon_cmd($vmid, "eject", force => JSON::true,device => "drive-$opt"); # force eject if locked
5181                     mon_cmd($vmid, "change", device => "drive-$opt",target => "$path") if $path;
5182                 }
5183
5184                 return 1;
5185             }
5186         }
5187     }
5188
5189     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
5190     # hotplug new disks
5191     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
5192     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive, $arch, $machine_type);
5193 }
5194
5195 sub vm_start {
5196     my ($storecfg, $vmid, $statefile, $skiplock, $migratedfrom, $paused,
5197         $forcemachine, $spice_ticket, $migration_network, $migration_type, $targetstorage) = @_;
5198
5199     PVE::QemuConfig->lock_config($vmid, sub {
5200         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
5201
5202         die "you can't start a vm if it's a template\n" if PVE::QemuConfig->is_template($conf);
5203
5204         my $is_suspended = PVE::QemuConfig->has_lock($conf, 'suspended');
5205
5206         PVE::QemuConfig->check_lock($conf)
5207             if !($skiplock || $is_suspended);
5208
5209         die "VM $vmid already running\n" if check_running($vmid, undef, $migratedfrom);
5210
5211         # clean up leftover reboot request files
5212         eval { clear_reboot_request($vmid); };
5213         warn $@ if $@;
5214
5215         if (!$statefile && scalar(keys %{$conf->{pending}})) {
5216             vmconfig_apply_pending($vmid, $conf, $storecfg);
5217             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5218         }
5219
5220         PVE::QemuServer::Cloudinit::generate_cloudinitconfig($conf, $vmid);
5221
5222         my $defaults = load_defaults();
5223
5224         # set environment variable useful inside network script
5225         $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
5226
5227         my $local_volumes = {};
5228
5229         if ($targetstorage) {
5230             foreach_drive($conf, sub {
5231                 my ($ds, $drive) = @_;
5232
5233                 return if drive_is_cdrom($drive);
5234
5235                 my $volid = $drive->{file};
5236
5237                 return if !$volid;
5238
5239                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
5240
5241                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5242                 return if $scfg->{shared};
5243                 $local_volumes->{$ds} = [$volid, $storeid, $volname];
5244             });
5245
5246             my $format = undef;
5247
5248             foreach my $opt (sort keys %$local_volumes) {
5249
5250                 my ($volid, $storeid, $volname) = @{$local_volumes->{$opt}};
5251                 my $drive = parse_drive($opt, $conf->{$opt});
5252
5253                 #if remote storage is specified, use default format
5254                 if ($targetstorage && $targetstorage ne "1") {
5255                     $storeid = $targetstorage;
5256                     my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
5257                     $format = $defFormat;
5258                 } else {
5259                     #else we use same format than original
5260                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5261                     $format = qemu_img_format($scfg, $volid);
5262                 }
5263
5264                 my $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $format, undef, ($drive->{size}/1024));
5265                 my $newdrive = $drive;
5266                 $newdrive->{format} = $format;
5267                 $newdrive->{file} = $newvolid;
5268                 my $drivestr = PVE::QemuServer::print_drive($vmid, $newdrive);
5269                 $local_volumes->{$opt} = $drivestr;
5270                 #pass drive to conf for command line
5271                 $conf->{$opt} = $drivestr;
5272             }
5273         }
5274
5275         PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-start', 1);
5276
5277         if ($is_suspended) {
5278             # enforce machine type on suspended vm to ensure HW compatibility
5279             $forcemachine = $conf->{runningmachine};
5280             print "Resuming suspended VM\n";
5281         }
5282
5283         my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine);
5284
5285         my $migration_ip;
5286         my $get_migration_ip = sub {
5287             my ($cidr, $nodename) = @_;
5288
5289             return $migration_ip if defined($migration_ip);
5290
5291             if (!defined($cidr)) {
5292                 my $dc_conf = PVE::Cluster::cfs_read_file('datacenter.cfg');
5293                 $cidr = $dc_conf->{migration}->{network};
5294             }
5295
5296             if (defined($cidr)) {
5297                 my $ips = PVE::Network::get_local_ip_from_cidr($cidr);
5298
5299                 die "could not get IP: no address configured on local " .
5300                     "node for network '$cidr'\n" if scalar(@$ips) == 0;
5301
5302                 die "could not get IP: multiple addresses configured on local " .
5303                     "node for network '$cidr'\n" if scalar(@$ips) > 1;
5304
5305                 $migration_ip = @$ips[0];
5306             }
5307
5308             $migration_ip = PVE::Cluster::remote_node_ip($nodename, 1)
5309                 if !defined($migration_ip);
5310
5311             return $migration_ip;
5312         };
5313
5314         my $migrate_uri;
5315         if ($statefile) {
5316             if ($statefile eq 'tcp') {
5317                 my $localip = "localhost";
5318                 my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
5319                 my $nodename = PVE::INotify::nodename();
5320
5321                 if (!defined($migration_type)) {
5322                     if (defined($datacenterconf->{migration}->{type})) {
5323                         $migration_type = $datacenterconf->{migration}->{type};
5324                     } else {
5325                         $migration_type = 'secure';
5326                     }
5327                 }
5328
5329                 if ($migration_type eq 'insecure') {
5330                     $localip = $get_migration_ip->($migration_network, $nodename);
5331                     $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5332                 }
5333
5334                 my $pfamily = PVE::Tools::get_host_address_family($nodename);
5335                 my $migrate_port = PVE::Tools::next_migrate_port($pfamily);
5336                 $migrate_uri = "tcp:${localip}:${migrate_port}";
5337                 push @$cmd, '-incoming', $migrate_uri;
5338                 push @$cmd, '-S';
5339
5340             } elsif ($statefile eq 'unix') {
5341                 # should be default for secure migrations as a ssh TCP forward
5342                 # tunnel is not deterministic reliable ready and fails regurarly
5343                 # to set up in time, so use UNIX socket forwards
5344                 my $socket_addr = "/run/qemu-server/$vmid.migrate";
5345                 unlink $socket_addr;
5346
5347                 $migrate_uri = "unix:$socket_addr";
5348
5349                 push @$cmd, '-incoming', $migrate_uri;
5350                 push @$cmd, '-S';
5351
5352             } elsif (-e $statefile) {
5353                 push @$cmd, '-loadstate', $statefile;
5354             } else {
5355                 my $statepath = PVE::Storage::path($storecfg, $statefile);
5356                 push @$vollist, $statefile;
5357                 push @$cmd, '-loadstate', $statepath;
5358             }
5359         } elsif ($paused) {
5360             push @$cmd, '-S';
5361         }
5362
5363         # host pci devices
5364         for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
5365           my $d = parse_hostpci($conf->{"hostpci$i"});
5366           next if !$d;
5367           my $pcidevices = $d->{pciid};
5368           foreach my $pcidevice (@$pcidevices) {
5369                 my $pciid = $pcidevice->{id};
5370                 $pciid = "0000:$pciid" if $pciid !~ m/^[0-9a-f]{4}:/;
5371
5372                 my $info = PVE::SysFSTools::pci_device_info("$pciid");
5373                 die "IOMMU not present\n" if !PVE::SysFSTools::check_iommu_support();
5374                 die "no pci device info for device '$pciid'\n" if !$info;
5375
5376                 if ($d->{mdev}) {
5377                     my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $i);
5378                     PVE::SysFSTools::pci_create_mdev_device($pciid, $uuid, $d->{mdev});
5379                 } else {
5380                     die "can't unbind/bind pci group to vfio '$pciid'\n"
5381                         if !PVE::SysFSTools::pci_dev_group_bind_to_vfio($pciid);
5382                     die "can't reset pci device '$pciid'\n"
5383                         if $info->{has_fl_reset} and !PVE::SysFSTools::pci_dev_reset($info);
5384                 }
5385           }
5386         }
5387
5388         PVE::Storage::activate_volumes($storecfg, $vollist);
5389
5390         eval {
5391             run_command(['/bin/systemctl', 'stop', "$vmid.scope"],
5392                 outfunc => sub {}, errfunc => sub {});
5393         };
5394         # Issues with the above 'stop' not being fully completed are extremely rare, a very low
5395         # timeout should be more than enough here...
5396         PVE::Systemd::wait_for_unit_removed("$vmid.scope", 5);
5397
5398         my $cpuunits = defined($conf->{cpuunits}) ? $conf->{cpuunits}
5399                                                   : $defaults->{cpuunits};
5400
5401         my $start_timeout = ($conf->{hugepages} || $is_suspended) ? 300 : 30;
5402         my %run_params = (timeout => $statefile ? undef : $start_timeout, umask => 0077);
5403
5404         my %properties = (
5405             Slice => 'qemu.slice',
5406             KillMode => 'none',
5407             CPUShares => $cpuunits
5408         );
5409
5410         if (my $cpulimit = $conf->{cpulimit}) {
5411             $properties{CPUQuota} = int($cpulimit * 100);
5412         }
5413         $properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
5414
5415         my $run_qemu = sub {
5416             PVE::Tools::run_fork sub {
5417                 PVE::Systemd::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
5418                 run_command($cmd, %run_params);
5419             };
5420         };
5421
5422         if ($conf->{hugepages}) {
5423
5424             my $code = sub {
5425                 my $hugepages_topology = PVE::QemuServer::Memory::hugepages_topology($conf);
5426                 my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
5427
5428                 PVE::QemuServer::Memory::hugepages_mount();
5429                 PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
5430
5431                 eval { $run_qemu->() };
5432                 if (my $err = $@) {
5433                     PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology);
5434                     die $err;
5435                 }
5436
5437                 PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology);
5438             };
5439             eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
5440
5441         } else {
5442             eval { $run_qemu->() };
5443         }
5444
5445         if (my $err = $@) {
5446             # deactivate volumes if start fails
5447             eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
5448             die "start failed: $err";
5449         }
5450
5451         print "migration listens on $migrate_uri\n" if $migrate_uri;
5452
5453         if ($statefile && $statefile ne 'tcp' && $statefile ne 'unix')  {
5454             eval { mon_cmd($vmid, "cont"); };
5455             warn $@ if $@;
5456         }
5457
5458         #start nbd server for storage migration
5459         if ($targetstorage) {
5460             my $nodename = PVE::INotify::nodename();
5461             my $localip = $get_migration_ip->($migration_network, $nodename);
5462             my $pfamily = PVE::Tools::get_host_address_family($nodename);
5463             my $storage_migrate_port = PVE::Tools::next_migrate_port($pfamily);
5464
5465             mon_cmd($vmid, "nbd-server-start", addr => { type => 'inet', data => { host => "${localip}", port => "${storage_migrate_port}" } } );
5466
5467             $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5468
5469             foreach my $opt (sort keys %$local_volumes) {
5470                 my $volid = $local_volumes->{$opt};
5471                 mon_cmd($vmid, "nbd-server-add", device => "drive-$opt", writable => JSON::true );
5472                 my $migrate_storage_uri = "nbd:${localip}:${storage_migrate_port}:exportname=drive-$opt";
5473                 print "storage migration listens on $migrate_storage_uri volume:$volid\n";
5474             }
5475         }
5476
5477         if ($migratedfrom) {
5478             eval {
5479                 set_migration_caps($vmid);
5480             };
5481             warn $@ if $@;
5482
5483             if ($spice_port) {
5484                 print "spice listens on port $spice_port\n";
5485                 if ($spice_ticket) {
5486                     mon_cmd($vmid, "set_password", protocol => 'spice', password => $spice_ticket);
5487                     mon_cmd($vmid, "expire_password", protocol => 'spice', time => "+30");
5488                 }
5489             }
5490
5491         } else {
5492             mon_cmd($vmid, "balloon", value => $conf->{balloon}*1024*1024)
5493                 if !$statefile && $conf->{balloon};
5494
5495             foreach my $opt (keys %$conf) {
5496                 next if $opt !~  m/^net\d+$/;
5497                 my $nicconf = parse_net($conf->{$opt});
5498                 qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
5499             }
5500         }
5501
5502         mon_cmd($vmid, 'qom-set',
5503                     path => "machine/peripheral/balloon0",
5504                     property => "guest-stats-polling-interval",
5505                     value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
5506
5507         if ($is_suspended && (my $vmstate = $conf->{vmstate})) {
5508             print "Resumed VM, removing state\n";
5509             delete $conf->@{qw(lock vmstate runningmachine)};
5510             PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
5511             PVE::Storage::vdisk_free($storecfg, $vmstate);
5512             PVE::QemuConfig->write_config($vmid, $conf);
5513         }
5514
5515         PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'post-start');
5516     });
5517 }
5518
5519 sub vm_commandline {
5520     my ($storecfg, $vmid, $snapname) = @_;
5521
5522     my $conf = PVE::QemuConfig->load_config($vmid);
5523     my $forcemachine;
5524
5525     if ($snapname) {
5526         my $snapshot = $conf->{snapshots}->{$snapname};
5527         die "snapshot '$snapname' does not exist\n" if !defined($snapshot);
5528
5529         # check for a 'runningmachine' in snapshot
5530         $forcemachine = $snapshot->{runningmachine} if $snapshot->{runningmachine};
5531
5532         $snapshot->{digest} = $conf->{digest}; # keep file digest for API
5533
5534         $conf = $snapshot;
5535     }
5536
5537     my $defaults = load_defaults();
5538
5539     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine);
5540
5541     return PVE::Tools::cmd2string($cmd);
5542 }
5543
5544 sub vm_reset {
5545     my ($vmid, $skiplock) = @_;
5546
5547     PVE::QemuConfig->lock_config($vmid, sub {
5548
5549         my $conf = PVE::QemuConfig->load_config($vmid);
5550
5551         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5552
5553         mon_cmd($vmid, "system_reset");
5554     });
5555 }
5556
5557 sub get_vm_volumes {
5558     my ($conf) = @_;
5559
5560     my $vollist = [];
5561     foreach_volid($conf, sub {
5562         my ($volid, $attr) = @_;
5563
5564         return if $volid =~ m|^/|;
5565
5566         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5567         return if !$sid;
5568
5569         push @$vollist, $volid;
5570     });
5571
5572     return $vollist;
5573 }
5574
5575 sub vm_stop_cleanup {
5576     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
5577
5578     eval {
5579
5580         if (!$keepActive) {
5581             my $vollist = get_vm_volumes($conf);
5582             PVE::Storage::deactivate_volumes($storecfg, $vollist);
5583         }
5584
5585         foreach my $ext (qw(mon qmp pid vnc qga)) {
5586             unlink "/var/run/qemu-server/${vmid}.$ext";
5587         }
5588
5589         if ($conf->{ivshmem}) {
5590             my $ivshmem = PVE::JSONSchema::parse_property_string($ivshmem_fmt, $conf->{ivshmem});
5591             # just delete it for now, VMs which have this already open do not
5592             # are affected, but new VMs will get a separated one. If this
5593             # becomes an issue we either add some sort of ref-counting or just
5594             # add a "don't delete on stop" flag to the ivshmem format.
5595             unlink '/dev/shm/pve-shm-' . ($ivshmem->{name} // $vmid);
5596         }
5597
5598         foreach my $key (keys %$conf) {
5599             next if $key !~ m/^hostpci(\d+)$/;
5600             my $hostpciindex = $1;
5601             my $d = parse_hostpci($conf->{$key});
5602             my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $hostpciindex);
5603
5604             foreach my $pci (@{$d->{pciid}}) {
5605                 my $pciid = $pci->{id};
5606                 PVE::SysFSTools::pci_cleanup_mdev_device($pciid, $uuid);
5607             }
5608         }
5609
5610         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
5611     };
5612     warn $@ if $@; # avoid errors - just warn
5613 }
5614
5615 # call only in locked context
5616 sub _do_vm_stop {
5617     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive) = @_;
5618
5619     my $pid = check_running($vmid, $nocheck);
5620     return if !$pid;
5621
5622     my $conf;
5623     if (!$nocheck) {
5624         $conf = PVE::QemuConfig->load_config($vmid);
5625         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5626         if (!defined($timeout) && $shutdown && $conf->{startup}) {
5627             my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
5628             $timeout = $opts->{down} if $opts->{down};
5629         }
5630         PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-stop');
5631     }
5632
5633     eval {
5634         if ($shutdown) {
5635             if (defined($conf) && parse_guest_agent($conf)->{enabled}) {
5636                 mon_cmd($vmid, "guest-shutdown", timeout => $timeout);
5637             } else {
5638                 mon_cmd($vmid, "system_powerdown");
5639             }
5640         } else {
5641             mon_cmd($vmid, "quit");
5642         }
5643     };
5644     my $err = $@;
5645
5646     if (!$err) {
5647         $timeout = 60 if !defined($timeout);
5648
5649         my $count = 0;
5650         while (($count < $timeout) && check_running($vmid, $nocheck)) {
5651             $count++;
5652             sleep 1;
5653         }
5654
5655         if ($count >= $timeout) {
5656             if ($force) {
5657                 warn "VM still running - terminating now with SIGTERM\n";
5658                 kill 15, $pid;
5659             } else {
5660                 die "VM quit/powerdown failed - got timeout\n";
5661             }
5662         } else {
5663             vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5664             return;
5665         }
5666     } else {
5667         if ($force) {
5668             warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
5669             kill 15, $pid;
5670         } else {
5671             die "VM quit/powerdown failed\n";
5672         }
5673     }
5674
5675     # wait again
5676     $timeout = 10;
5677
5678     my $count = 0;
5679     while (($count < $timeout) && check_running($vmid, $nocheck)) {
5680         $count++;
5681         sleep 1;
5682     }
5683
5684     if ($count >= $timeout) {
5685         warn "VM still running - terminating now with SIGKILL\n";
5686         kill 9, $pid;
5687         sleep 1;
5688     }
5689
5690     vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5691 }
5692
5693 # Note: use $nocheck to skip tests if VM configuration file exists.
5694 # We need that when migration VMs to other nodes (files already moved)
5695 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
5696 sub vm_stop {
5697     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
5698
5699     $force = 1 if !defined($force) && !$shutdown;
5700
5701     if ($migratedfrom){
5702         my $pid = check_running($vmid, $nocheck, $migratedfrom);
5703         kill 15, $pid if $pid;
5704         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
5705         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
5706         return;
5707     }
5708
5709     PVE::QemuConfig->lock_config($vmid, sub {
5710         _do_vm_stop($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive);
5711    });
5712 }
5713
5714 sub vm_reboot {
5715     my ($vmid, $timeout) = @_;
5716
5717     PVE::QemuConfig->lock_config($vmid, sub {
5718         eval {
5719
5720             # only reboot if running, as qmeventd starts it again on a stop event
5721             return if !check_running($vmid);
5722
5723             create_reboot_request($vmid);
5724
5725             my $storecfg = PVE::Storage::config();
5726             _do_vm_stop($storecfg, $vmid, undef, undef, $timeout, 1);
5727
5728         };
5729         if (my $err = $@) {
5730             # avoid that the next normal shutdown will be confused for a reboot
5731             clear_reboot_request($vmid);
5732             die $err;
5733         }
5734    });
5735 }
5736
5737 sub vm_suspend {
5738     my ($vmid, $skiplock, $includestate, $statestorage) = @_;
5739
5740     my $conf;
5741     my $path;
5742     my $storecfg;
5743     my $vmstate;
5744
5745     PVE::QemuConfig->lock_config($vmid, sub {
5746
5747         $conf = PVE::QemuConfig->load_config($vmid);
5748
5749         my $is_backing_up = PVE::QemuConfig->has_lock($conf, 'backup');
5750         PVE::QemuConfig->check_lock($conf)
5751             if !($skiplock || $is_backing_up);
5752
5753         die "cannot suspend to disk during backup\n"
5754             if $is_backing_up && $includestate;
5755
5756         if ($includestate) {
5757             $conf->{lock} = 'suspending';
5758             my $date = strftime("%Y-%m-%d", localtime(time()));
5759             $storecfg = PVE::Storage::config();
5760             $vmstate = PVE::QemuConfig->__snapshot_save_vmstate($vmid, $conf, "suspend-$date", $storecfg, $statestorage, 1);
5761             $path = PVE::Storage::path($storecfg, $vmstate);
5762             PVE::QemuConfig->write_config($vmid, $conf);
5763         } else {
5764             mon_cmd($vmid, "stop");
5765         }
5766     });
5767
5768     if ($includestate) {
5769         # save vm state
5770         PVE::Storage::activate_volumes($storecfg, [$vmstate]);
5771
5772         eval {
5773             mon_cmd($vmid, "savevm-start", statefile => $path);
5774             for(;;) {
5775                 my $state = mon_cmd($vmid, "query-savevm");
5776                 if (!$state->{status}) {
5777                     die "savevm not active\n";
5778                 } elsif ($state->{status} eq 'active') {
5779                     sleep(1);
5780                     next;
5781                 } elsif ($state->{status} eq 'completed') {
5782                     print "State saved, quitting\n";
5783                     last;
5784                 } elsif ($state->{status} eq 'failed' && $state->{error}) {
5785                     die "query-savevm failed with error '$state->{error}'\n"
5786                 } else {
5787                     die "query-savevm returned status '$state->{status}'\n";
5788                 }
5789             }
5790         };
5791         my $err = $@;
5792
5793         PVE::QemuConfig->lock_config($vmid, sub {
5794             $conf = PVE::QemuConfig->load_config($vmid);
5795             if ($err) {
5796                 # cleanup, but leave suspending lock, to indicate something went wrong
5797                 eval {
5798                     mon_cmd($vmid, "savevm-end");
5799                     PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
5800                     PVE::Storage::vdisk_free($storecfg, $vmstate);
5801                     delete $conf->@{qw(vmstate runningmachine)};
5802                     PVE::QemuConfig->write_config($vmid, $conf);
5803                 };
5804                 warn $@ if $@;
5805                 die $err;
5806             }
5807
5808             die "lock changed unexpectedly\n"
5809                 if !PVE::QemuConfig->has_lock($conf, 'suspending');
5810
5811             mon_cmd($vmid, "quit");
5812             $conf->{lock} = 'suspended';
5813             PVE::QemuConfig->write_config($vmid, $conf);
5814         });
5815     }
5816 }
5817
5818 sub vm_resume {
5819     my ($vmid, $skiplock, $nocheck) = @_;
5820
5821     PVE::QemuConfig->lock_config($vmid, sub {
5822         my $res = mon_cmd($vmid, 'query-status');
5823         my $resume_cmd = 'cont';
5824
5825         if ($res->{status} && $res->{status} eq 'suspended') {
5826             $resume_cmd = 'system_wakeup';
5827         }
5828
5829         if (!$nocheck) {
5830
5831             my $conf = PVE::QemuConfig->load_config($vmid);
5832
5833             PVE::QemuConfig->check_lock($conf)
5834                 if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
5835         }
5836
5837         mon_cmd($vmid, $resume_cmd);
5838     });
5839 }
5840
5841 sub vm_sendkey {
5842     my ($vmid, $skiplock, $key) = @_;
5843
5844     PVE::QemuConfig->lock_config($vmid, sub {
5845
5846         my $conf = PVE::QemuConfig->load_config($vmid);
5847
5848         # there is no qmp command, so we use the human monitor command
5849         my $res = PVE::QemuServer::Monitor::hmp_cmd($vmid, "sendkey $key");
5850         die $res if $res ne '';
5851     });
5852 }
5853
5854 # vzdump restore implementaion
5855
5856 sub tar_archive_read_firstfile {
5857     my $archive = shift;
5858
5859     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
5860
5861     # try to detect archive type first
5862     my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
5863         die "unable to open file '$archive'\n";
5864     my $firstfile = <$fh>;
5865     kill 15, $pid;
5866     close $fh;
5867
5868     die "ERROR: archive contaions no data\n" if !$firstfile;
5869     chomp $firstfile;
5870
5871     return $firstfile;
5872 }
5873
5874 sub tar_restore_cleanup {
5875     my ($storecfg, $statfile) = @_;
5876
5877     print STDERR "starting cleanup\n";
5878
5879     if (my $fd = IO::File->new($statfile, "r")) {
5880         while (defined(my $line = <$fd>)) {
5881             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5882                 my $volid = $2;
5883                 eval {
5884                     if ($volid =~ m|^/|) {
5885                         unlink $volid || die 'unlink failed\n';
5886                     } else {
5887                         PVE::Storage::vdisk_free($storecfg, $volid);
5888                     }
5889                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
5890                 };
5891                 print STDERR "unable to cleanup '$volid' - $@" if $@;
5892             } else {
5893                 print STDERR "unable to parse line in statfile - $line";
5894             }
5895         }
5896         $fd->close();
5897     }
5898 }
5899
5900 sub restore_archive {
5901     my ($archive, $vmid, $user, $opts) = @_;
5902
5903     my $format = $opts->{format};
5904     my $comp;
5905
5906     if ($archive =~ m/\.tgz$/ || $archive =~ m/\.tar\.gz$/) {
5907         $format = 'tar' if !$format;
5908         $comp = 'gzip';
5909     } elsif ($archive =~ m/\.tar$/) {
5910         $format = 'tar' if !$format;
5911     } elsif ($archive =~ m/.tar.lzo$/) {
5912         $format = 'tar' if !$format;
5913         $comp = 'lzop';
5914     } elsif ($archive =~ m/\.vma$/) {
5915         $format = 'vma' if !$format;
5916     } elsif ($archive =~ m/\.vma\.gz$/) {
5917         $format = 'vma' if !$format;
5918         $comp = 'gzip';
5919     } elsif ($archive =~ m/\.vma\.lzo$/) {
5920         $format = 'vma' if !$format;
5921         $comp = 'lzop';
5922     } else {
5923         $format = 'vma' if !$format; # default
5924     }
5925
5926     # try to detect archive format
5927     if ($format eq 'tar') {
5928         return restore_tar_archive($archive, $vmid, $user, $opts);
5929     } else {
5930         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
5931     }
5932 }
5933
5934 sub restore_update_config_line {
5935     my ($outfd, $cookie, $vmid, $map, $line, $unique) = @_;
5936
5937     return if $line =~ m/^\#qmdump\#/;
5938     return if $line =~ m/^\#vzdump\#/;
5939     return if $line =~ m/^lock:/;
5940     return if $line =~ m/^unused\d+:/;
5941     return if $line =~ m/^parent:/;
5942
5943     my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
5944     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
5945         # try to convert old 1.X settings
5946         my ($id, $ind, $ethcfg) = ($1, $2, $3);
5947         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
5948             my ($model, $macaddr) = split(/\=/, $devconfig);
5949             $macaddr = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if !$macaddr || $unique;
5950             my $net = {
5951                 model => $model,
5952                 bridge => "vmbr$ind",
5953                 macaddr => $macaddr,
5954             };
5955             my $netstr = print_net($net);
5956
5957             print $outfd "net$cookie->{netcount}: $netstr\n";
5958             $cookie->{netcount}++;
5959         }
5960     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
5961         my ($id, $netstr) = ($1, $2);
5962         my $net = parse_net($netstr);
5963         $net->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if $net->{macaddr};
5964         $netstr = print_net($net);
5965         print $outfd "$id: $netstr\n";
5966     } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk)\d+):\s*(\S+)\s*$/) {
5967         my $virtdev = $1;
5968         my $value = $3;
5969         my $di = parse_drive($virtdev, $value);
5970         if (defined($di->{backup}) && !$di->{backup}) {
5971             print $outfd "#$line";
5972         } elsif ($map->{$virtdev}) {
5973             delete $di->{format}; # format can change on restore
5974             $di->{file} = $map->{$virtdev};
5975             $value = print_drive($vmid, $di);
5976             print $outfd "$virtdev: $value\n";
5977         } else {
5978             print $outfd $line;
5979         }
5980     } elsif (($line =~ m/^vmgenid: (.*)/)) {
5981         my $vmgenid = $1;
5982         if ($vmgenid ne '0') {
5983             # always generate a new vmgenid if there was a valid one setup
5984             $vmgenid = generate_uuid();
5985         }
5986         print $outfd "vmgenid: $vmgenid\n";
5987     } elsif (($line =~ m/^(smbios1: )(.*)/) && $unique) {
5988         my ($uuid, $uuid_str);
5989         UUID::generate($uuid);
5990         UUID::unparse($uuid, $uuid_str);
5991         my $smbios1 = parse_smbios1($2);
5992         $smbios1->{uuid} = $uuid_str;
5993         print $outfd $1.print_smbios1($smbios1)."\n";
5994     } else {
5995         print $outfd $line;
5996     }
5997 }
5998
5999 sub scan_volids {
6000     my ($cfg, $vmid) = @_;
6001
6002     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid);
6003
6004     my $volid_hash = {};
6005     foreach my $storeid (keys %$info) {
6006         foreach my $item (@{$info->{$storeid}}) {
6007             next if !($item->{volid} && $item->{size});
6008             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
6009             $volid_hash->{$item->{volid}} = $item;
6010         }
6011     }
6012
6013     return $volid_hash;
6014 }
6015
6016 sub is_volume_in_use {
6017     my ($storecfg, $conf, $skip_drive, $volid) = @_;
6018
6019     my $path = PVE::Storage::path($storecfg, $volid);
6020
6021     my $scan_config = sub {
6022         my ($cref, $snapname) = @_;
6023
6024         foreach my $key (keys %$cref) {
6025             my $value = $cref->{$key};
6026             if (is_valid_drivename($key)) {
6027                 next if $skip_drive && $key eq $skip_drive;
6028                 my $drive = parse_drive($key, $value);
6029                 next if !$drive || !$drive->{file} || drive_is_cdrom($drive);
6030                 return 1 if $volid eq $drive->{file};
6031                 if ($drive->{file} =~ m!^/!) {
6032                     return 1 if $drive->{file} eq $path;
6033                 } else {
6034                     my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file}, 1);
6035                     next if !$storeid;
6036                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid, 1);
6037                     next if !$scfg;
6038                     return 1 if $path eq PVE::Storage::path($storecfg, $drive->{file}, $snapname);
6039                 }
6040             }
6041         }
6042
6043         return 0;
6044     };
6045
6046     return 1 if &$scan_config($conf);
6047
6048     undef $skip_drive;
6049
6050     foreach my $snapname (keys %{$conf->{snapshots}}) {
6051         return 1 if &$scan_config($conf->{snapshots}->{$snapname}, $snapname);
6052     }
6053
6054     return 0;
6055 }
6056
6057 sub update_disksize {
6058     my ($vmid, $conf, $volid_hash) = @_;
6059
6060     my $changes;
6061     my $prefix = "VM $vmid:";
6062
6063     # used and unused disks
6064     my $referenced = {};
6065
6066     # Note: it is allowed to define multiple storages with same path (alias), so
6067     # we need to check both 'volid' and real 'path' (two different volid can point
6068     # to the same path).
6069
6070     my $referencedpath = {};
6071
6072     # update size info
6073     foreach my $opt (keys %$conf) {
6074         if (is_valid_drivename($opt)) {
6075             my $drive = parse_drive($opt, $conf->{$opt});
6076             my $volid = $drive->{file};
6077             next if !$volid;
6078
6079             $referenced->{$volid} = 1;
6080             if ($volid_hash->{$volid} &&
6081                 (my $path = $volid_hash->{$volid}->{path})) {
6082                 $referencedpath->{$path} = 1;
6083             }
6084
6085             next if drive_is_cdrom($drive);
6086             next if !$volid_hash->{$volid};
6087
6088             $drive->{size} = $volid_hash->{$volid}->{size};
6089             my $new = print_drive($vmid, $drive);
6090             if ($new ne $conf->{$opt}) {
6091                 $changes = 1;
6092                 $conf->{$opt} = $new;
6093                 print "$prefix update disk '$opt' information.\n";
6094             }
6095         }
6096     }
6097
6098     # remove 'unusedX' entry if volume is used
6099     foreach my $opt (keys %$conf) {
6100         next if $opt !~ m/^unused\d+$/;
6101         my $volid = $conf->{$opt};
6102         my $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
6103         if ($referenced->{$volid} || ($path && $referencedpath->{$path})) {
6104             print "$prefix remove entry '$opt', its volume '$volid' is in use.\n";
6105             $changes = 1;
6106             delete $conf->{$opt};
6107         }
6108
6109         $referenced->{$volid} = 1;
6110         $referencedpath->{$path} = 1 if $path;
6111     }
6112
6113     foreach my $volid (sort keys %$volid_hash) {
6114         next if $volid =~ m/vm-$vmid-state-/;
6115         next if $referenced->{$volid};
6116         my $path = $volid_hash->{$volid}->{path};
6117         next if !$path; # just to be sure
6118         next if $referencedpath->{$path};
6119         $changes = 1;
6120         my $key = PVE::QemuConfig->add_unused_volume($conf, $volid);
6121         print "$prefix add unreferenced volume '$volid' as '$key' to config.\n";
6122         $referencedpath->{$path} = 1; # avoid to add more than once (aliases)
6123     }
6124
6125     return $changes;
6126 }
6127
6128 sub rescan {
6129     my ($vmid, $nolock, $dryrun) = @_;
6130
6131     my $cfg = PVE::Storage::config();
6132
6133     # FIXME: Remove once our RBD plugin can handle CT and VM on a single storage
6134     # see: https://pve.proxmox.com/pipermail/pve-devel/2018-July/032900.html
6135     foreach my $stor (keys %{$cfg->{ids}}) {
6136         delete($cfg->{ids}->{$stor}) if ! $cfg->{ids}->{$stor}->{content}->{images};
6137     }
6138
6139     print "rescan volumes...\n";
6140     my $volid_hash = scan_volids($cfg, $vmid);
6141
6142     my $updatefn =  sub {
6143         my ($vmid) = @_;
6144
6145         my $conf = PVE::QemuConfig->load_config($vmid);
6146
6147         PVE::QemuConfig->check_lock($conf);
6148
6149         my $vm_volids = {};
6150         foreach my $volid (keys %$volid_hash) {
6151             my $info = $volid_hash->{$volid};
6152             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
6153         }
6154
6155         my $changes = update_disksize($vmid, $conf, $vm_volids);
6156
6157         PVE::QemuConfig->write_config($vmid, $conf) if $changes && !$dryrun;
6158     };
6159
6160     if (defined($vmid)) {
6161         if ($nolock) {
6162             &$updatefn($vmid);
6163         } else {
6164             PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6165         }
6166     } else {
6167         my $vmlist = config_list();
6168         foreach my $vmid (keys %$vmlist) {
6169             if ($nolock) {
6170                 &$updatefn($vmid);
6171             } else {
6172                 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6173             }
6174         }
6175     }
6176 }
6177
6178 sub restore_vma_archive {
6179     my ($archive, $vmid, $user, $opts, $comp) = @_;
6180
6181     my $readfrom = $archive;
6182
6183     my $cfg = PVE::Storage::config();
6184     my $commands = [];
6185     my $bwlimit = $opts->{bwlimit};
6186
6187     my $dbg_cmdstring = '';
6188     my $add_pipe = sub {
6189         my ($cmd) = @_;
6190         push @$commands, $cmd;
6191         $dbg_cmdstring .= ' | ' if length($dbg_cmdstring);
6192         $dbg_cmdstring .= PVE::Tools::cmd2string($cmd);
6193         $readfrom = '-';
6194     };
6195
6196     my $input = undef;
6197     if ($archive eq '-') {
6198         $input = '<&STDIN';
6199     } else {
6200         # If we use a backup from a PVE defined storage we also consider that
6201         # storage's rate limit:
6202         my (undef, $volid) = PVE::Storage::path_to_volume_id($cfg, $archive);
6203         if (defined($volid)) {
6204             my ($sid, undef) = PVE::Storage::parse_volume_id($volid);
6205             my $readlimit = PVE::Storage::get_bandwidth_limit('restore', [$sid], $bwlimit);
6206             if ($readlimit) {
6207                 print STDERR "applying read rate limit: $readlimit\n";
6208                 my $cstream = ['cstream', '-t', $readlimit*1024, '--', $readfrom];
6209                 $add_pipe->($cstream);
6210             }
6211         }
6212     }
6213
6214     if ($comp) {
6215         my $cmd;
6216         if ($comp eq 'gzip') {
6217             $cmd = ['zcat', $readfrom];
6218         } elsif ($comp eq 'lzop') {
6219             $cmd = ['lzop', '-d', '-c', $readfrom];
6220         } else {
6221             die "unknown compression method '$comp'\n";
6222         }
6223         $add_pipe->($cmd);
6224     }
6225
6226     my $tmpdir = "/var/tmp/vzdumptmp$$";
6227     rmtree $tmpdir;
6228
6229     # disable interrupts (always do cleanups)
6230     local $SIG{INT} =
6231         local $SIG{TERM} =
6232         local $SIG{QUIT} =
6233         local $SIG{HUP} = sub { warn "got interrupt - ignored\n"; };
6234
6235     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
6236     POSIX::mkfifo($mapfifo, 0600);
6237     my $fifofh;
6238
6239     my $openfifo = sub {
6240         open($fifofh, '>', $mapfifo) || die $!;
6241     };
6242
6243     $add_pipe->(['vma', 'extract', '-v', '-r', $mapfifo, $readfrom, $tmpdir]);
6244
6245     my $oldtimeout;
6246     my $timeout = 5;
6247
6248     my $devinfo = {};
6249
6250     my $rpcenv = PVE::RPCEnvironment::get();
6251
6252     my $conffile = PVE::QemuConfig->config_file($vmid);
6253     my $tmpfn = "$conffile.$$.tmp";
6254
6255     # Note: $oldconf is undef if VM does not exists
6256     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
6257     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
6258
6259     my %storage_limits;
6260
6261     my $print_devmap = sub {
6262         my $virtdev_hash = {};
6263
6264         my $cfgfn = "$tmpdir/qemu-server.conf";
6265
6266         # we can read the config - that is already extracted
6267         my $fh = IO::File->new($cfgfn, "r") ||
6268             "unable to read qemu-server.conf - $!\n";
6269
6270         my $fwcfgfn = "$tmpdir/qemu-server.fw";
6271         if (-f $fwcfgfn) {
6272             my $pve_firewall_dir = '/etc/pve/firewall';
6273             mkdir $pve_firewall_dir; # make sure the dir exists
6274             PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
6275         }
6276
6277         while (defined(my $line = <$fh>)) {
6278             if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
6279                 my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
6280                 die "archive does not contain data for drive '$virtdev'\n"
6281                     if !$devinfo->{$devname};
6282                 if (defined($opts->{storage})) {
6283                     $storeid = $opts->{storage} || 'local';
6284                 } elsif (!$storeid) {
6285                     $storeid = 'local';
6286                 }
6287                 $format = 'raw' if !$format;
6288                 $devinfo->{$devname}->{devname} = $devname;
6289                 $devinfo->{$devname}->{virtdev} = $virtdev;
6290                 $devinfo->{$devname}->{format} = $format;
6291                 $devinfo->{$devname}->{storeid} = $storeid;
6292
6293                 # check permission on storage
6294                 my $pool = $opts->{pool}; # todo: do we need that?
6295                 if ($user ne 'root@pam') {
6296                     $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
6297                 }
6298
6299                 $storage_limits{$storeid} = $bwlimit;
6300
6301                 $virtdev_hash->{$virtdev} = $devinfo->{$devname};
6302             } elsif ($line =~ m/^((?:ide|sata|scsi)\d+):\s*(.*)\s*$/) {
6303                 my $virtdev = $1;
6304                 my $drive = parse_drive($virtdev, $2);
6305                 if (drive_is_cloudinit($drive)) {
6306                     my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6307                     my $scfg = PVE::Storage::storage_config($cfg, $storeid);
6308                     my $format = qemu_img_format($scfg, $volname); # has 'raw' fallback
6309
6310                     my $d = {
6311                         format => $format,
6312                         storeid => $opts->{storage} // $storeid,
6313                         size => PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE,
6314                         file => $drive->{file}, # to make drive_is_cloudinit check possible
6315                         name => "vm-$vmid-cloudinit",
6316                         is_cloudinit => 1,
6317                     };
6318                     $virtdev_hash->{$virtdev} = $d;
6319                 }
6320             }
6321         }
6322
6323         foreach my $key (keys %storage_limits) {
6324             my $limit = PVE::Storage::get_bandwidth_limit('restore', [$key], $bwlimit);
6325             next if !$limit;
6326             print STDERR "rate limit for storage $key: $limit KiB/s\n";
6327             $storage_limits{$key} = $limit * 1024;
6328         }
6329
6330         foreach my $devname (keys %$devinfo) {
6331             die "found no device mapping information for device '$devname'\n"
6332                 if !$devinfo->{$devname}->{virtdev};
6333         }
6334
6335         # create empty/temp config
6336         if ($oldconf) {
6337             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
6338             foreach_drive($oldconf, sub {
6339                 my ($ds, $drive) = @_;
6340
6341                 return if drive_is_cdrom($drive, 1);
6342
6343                 my $volid = $drive->{file};
6344                 return if !$volid || $volid =~ m|^/|;
6345
6346                 my ($path, $owner) = PVE::Storage::path($cfg, $volid);
6347                 return if !$path || !$owner || ($owner != $vmid);
6348
6349                 # Note: only delete disk we want to restore
6350                 # other volumes will become unused
6351                 if ($virtdev_hash->{$ds}) {
6352                     eval { PVE::Storage::vdisk_free($cfg, $volid); };
6353                     if (my $err = $@) {
6354                         warn $err;
6355                     }
6356                 }
6357             });
6358
6359             # delete vmstate files, after the restore we have no snapshots anymore
6360             foreach my $snapname (keys %{$oldconf->{snapshots}}) {
6361                 my $snap = $oldconf->{snapshots}->{$snapname};
6362                 if ($snap->{vmstate}) {
6363                     eval { PVE::Storage::vdisk_free($cfg, $snap->{vmstate}); };
6364                     if (my $err = $@) {
6365                         warn $err;
6366                     }
6367                 }
6368             }
6369         }
6370
6371         my $map = {};
6372         foreach my $virtdev (sort keys %$virtdev_hash) {
6373             my $d = $virtdev_hash->{$virtdev};
6374             my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
6375             my $storeid = $d->{storeid};
6376             my $scfg = PVE::Storage::storage_config($cfg, $storeid);
6377
6378             my $map_opts = '';
6379             if (my $limit = $storage_limits{$storeid}) {
6380                 $map_opts .= "throttling.bps=$limit:throttling.group=$storeid:";
6381             }
6382
6383             # test if requested format is supported
6384             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($cfg, $storeid);
6385             my $supported = grep { $_ eq $d->{format} } @$validFormats;
6386             $d->{format} = $defFormat if !$supported;
6387
6388             my $name;
6389             if ($d->{is_cloudinit}) {
6390                 $name = $d->{name};
6391                 $name .= ".$d->{format}" if $d->{format} ne 'raw';
6392             }
6393
6394             my $volid = PVE::Storage::vdisk_alloc($cfg, $storeid, $vmid, $d->{format}, $name, $alloc_size);
6395             print STDERR "new volume ID is '$volid'\n";
6396             $d->{volid} = $volid;
6397
6398             PVE::Storage::activate_volumes($cfg, [$volid]);
6399
6400             my $write_zeros = 1;
6401             if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
6402                 $write_zeros = 0;
6403             }
6404
6405             if (!$d->{is_cloudinit}) {
6406                 my $path = PVE::Storage::path($cfg, $volid);
6407
6408                 print $fifofh "${map_opts}format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
6409
6410                 print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
6411             }
6412             $map->{$virtdev} = $volid;
6413         }
6414
6415         $fh->seek(0, 0) || die "seek failed - $!\n";
6416
6417         my $outfd = new IO::File ($tmpfn, "w") ||
6418             die "unable to write config for VM $vmid\n";
6419
6420         my $cookie = { netcount => 0 };
6421         while (defined(my $line = <$fh>)) {
6422             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
6423         }
6424
6425         $fh->close();
6426         $outfd->close();
6427     };
6428
6429     eval {
6430         # enable interrupts
6431         local $SIG{INT} =
6432             local $SIG{TERM} =
6433             local $SIG{QUIT} =
6434             local $SIG{HUP} =
6435             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6436         local $SIG{ALRM} = sub { die "got timeout\n"; };
6437
6438         $oldtimeout = alarm($timeout);
6439
6440         my $parser = sub {
6441             my $line = shift;
6442
6443             print "$line\n";
6444
6445             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
6446                 my ($dev_id, $size, $devname) = ($1, $2, $3);
6447                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
6448             } elsif ($line =~ m/^CTIME: /) {
6449                 # we correctly received the vma config, so we can disable
6450                 # the timeout now for disk allocation (set to 10 minutes, so
6451                 # that we always timeout if something goes wrong)
6452                 alarm(600);
6453                 &$print_devmap();
6454                 print $fifofh "done\n";
6455                 my $tmp = $oldtimeout || 0;
6456                 $oldtimeout = undef;
6457                 alarm($tmp);
6458                 close($fifofh);
6459             }
6460         };
6461
6462         print "restore vma archive: $dbg_cmdstring\n";
6463         run_command($commands, input => $input, outfunc => $parser, afterfork => $openfifo);
6464     };
6465     my $err = $@;
6466
6467     alarm($oldtimeout) if $oldtimeout;
6468
6469     my $vollist = [];
6470     foreach my $devname (keys %$devinfo) {
6471         my $volid = $devinfo->{$devname}->{volid};
6472         push @$vollist, $volid if $volid;
6473     }
6474
6475     PVE::Storage::deactivate_volumes($cfg, $vollist);
6476
6477     unlink $mapfifo;
6478
6479     if ($err) {
6480         rmtree $tmpdir;
6481         unlink $tmpfn;
6482
6483         foreach my $devname (keys %$devinfo) {
6484             my $volid = $devinfo->{$devname}->{volid};
6485             next if !$volid;
6486             eval {
6487                 if ($volid =~ m|^/|) {
6488                     unlink $volid || die 'unlink failed\n';
6489                 } else {
6490                     PVE::Storage::vdisk_free($cfg, $volid);
6491                 }
6492                 print STDERR "temporary volume '$volid' sucessfuly removed\n";
6493             };
6494             print STDERR "unable to cleanup '$volid' - $@" if $@;
6495         }
6496         die $err;
6497     }
6498
6499     rmtree $tmpdir;
6500
6501     rename($tmpfn, $conffile) ||
6502         die "unable to commit configuration file '$conffile'\n";
6503
6504     PVE::Cluster::cfs_update(); # make sure we read new file
6505
6506     eval { rescan($vmid, 1); };
6507     warn $@ if $@;
6508 }
6509
6510 sub restore_tar_archive {
6511     my ($archive, $vmid, $user, $opts) = @_;
6512
6513     if ($archive ne '-') {
6514         my $firstfile = tar_archive_read_firstfile($archive);
6515         die "ERROR: file '$archive' dos not lock like a QemuServer vzdump backup\n"
6516             if $firstfile ne 'qemu-server.conf';
6517     }
6518
6519     my $storecfg = PVE::Storage::config();
6520
6521     # avoid zombie disks when restoring over an existing VM -> cleanup first
6522     # pass keep_empty_config=1 to keep the config (thus VMID) reserved for us
6523     # skiplock=1 because qmrestore has set the 'create' lock itself already
6524     my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
6525     destroy_vm($storecfg, $vmid, 1, { lock => 'restore' }) if -f $vmcfgfn;
6526
6527     my $tocmd = "/usr/lib/qemu-server/qmextract";
6528
6529     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
6530     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
6531     $tocmd .= ' --prealloc' if $opts->{prealloc};
6532     $tocmd .= ' --info' if $opts->{info};
6533
6534     # tar option "xf" does not autodetect compression when read from STDIN,
6535     # so we pipe to zcat
6536     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
6537         PVE::Tools::shellquote("--to-command=$tocmd");
6538
6539     my $tmpdir = "/var/tmp/vzdumptmp$$";
6540     mkpath $tmpdir;
6541
6542     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
6543     local $ENV{VZDUMP_VMID} = $vmid;
6544     local $ENV{VZDUMP_USER} = $user;
6545
6546     my $conffile = PVE::QemuConfig->config_file($vmid);
6547     my $tmpfn = "$conffile.$$.tmp";
6548
6549     # disable interrupts (always do cleanups)
6550     local $SIG{INT} =
6551         local $SIG{TERM} =
6552         local $SIG{QUIT} =
6553         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
6554
6555     eval {
6556         # enable interrupts
6557         local $SIG{INT} =
6558             local $SIG{TERM} =
6559             local $SIG{QUIT} =
6560             local $SIG{HUP} =
6561             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6562
6563         if ($archive eq '-') {
6564             print "extracting archive from STDIN\n";
6565             run_command($cmd, input => "<&STDIN");
6566         } else {
6567             print "extracting archive '$archive'\n";
6568             run_command($cmd);
6569         }
6570
6571         return if $opts->{info};
6572
6573         # read new mapping
6574         my $map = {};
6575         my $statfile = "$tmpdir/qmrestore.stat";
6576         if (my $fd = IO::File->new($statfile, "r")) {
6577             while (defined (my $line = <$fd>)) {
6578                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
6579                     $map->{$1} = $2 if $1;
6580                 } else {
6581                     print STDERR "unable to parse line in statfile - $line\n";
6582                 }
6583             }
6584             $fd->close();
6585         }
6586
6587         my $confsrc = "$tmpdir/qemu-server.conf";
6588
6589         my $srcfd = new IO::File($confsrc, "r") ||
6590             die "unable to open file '$confsrc'\n";
6591
6592         my $outfd = new IO::File ($tmpfn, "w") ||
6593             die "unable to write config for VM $vmid\n";
6594
6595         my $cookie = { netcount => 0 };
6596         while (defined (my $line = <$srcfd>)) {
6597             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
6598         }
6599
6600         $srcfd->close();
6601         $outfd->close();
6602     };
6603     if (my $err = $@) {
6604         unlink $tmpfn;
6605         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
6606         die $err;
6607     }
6608
6609     rmtree $tmpdir;
6610
6611     rename $tmpfn, $conffile ||
6612         die "unable to commit configuration file '$conffile'\n";
6613
6614     PVE::Cluster::cfs_update(); # make sure we read new file
6615
6616     eval { rescan($vmid, 1); };
6617     warn $@ if $@;
6618 };
6619
6620 sub foreach_storage_used_by_vm {
6621     my ($conf, $func) = @_;
6622
6623     my $sidhash = {};
6624
6625     foreach_drive($conf, sub {
6626         my ($ds, $drive) = @_;
6627         return if drive_is_cdrom($drive);
6628
6629         my $volid = $drive->{file};
6630
6631         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
6632         $sidhash->{$sid} = $sid if $sid;
6633     });
6634
6635     foreach my $sid (sort keys %$sidhash) {
6636         &$func($sid);
6637     }
6638 }
6639
6640 my $qemu_snap_storage = {
6641     rbd => 1,
6642 };
6643 sub do_snapshots_with_qemu {
6644     my ($storecfg, $volid) = @_;
6645
6646     my $storage_name = PVE::Storage::parse_volume_id($volid);
6647     my $scfg = $storecfg->{ids}->{$storage_name};
6648
6649     if ($qemu_snap_storage->{$scfg->{type}} && !$scfg->{krbd}){
6650         return 1;
6651     }
6652
6653     if ($volid =~ m/\.(qcow2|qed)$/){
6654         return 1;
6655     }
6656
6657     return undef;
6658 }
6659
6660 sub qga_check_running {
6661     my ($vmid, $nowarn) = @_;
6662
6663     eval { mon_cmd($vmid, "guest-ping", timeout => 3); };
6664     if ($@) {
6665         warn "Qemu Guest Agent is not running - $@" if !$nowarn;
6666         return 0;
6667     }
6668     return 1;
6669 }
6670
6671 sub template_create {
6672     my ($vmid, $conf, $disk) = @_;
6673
6674     my $storecfg = PVE::Storage::config();
6675
6676     foreach_drive($conf, sub {
6677         my ($ds, $drive) = @_;
6678
6679         return if drive_is_cdrom($drive);
6680         return if $disk && $ds ne $disk;
6681
6682         my $volid = $drive->{file};
6683         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
6684
6685         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
6686         $drive->{file} = $voliddst;
6687         $conf->{$ds} = print_drive($vmid, $drive);
6688         PVE::QemuConfig->write_config($vmid, $conf);
6689     });
6690 }
6691
6692 sub convert_iscsi_path {
6693     my ($path) = @_;
6694
6695     if ($path =~ m|^iscsi://([^/]+)/([^/]+)/(.+)$|) {
6696         my $portal = $1;
6697         my $target = $2;
6698         my $lun = $3;
6699
6700         my $initiator_name = get_initiator_name();
6701
6702         return "file.driver=iscsi,file.transport=tcp,file.initiator-name=$initiator_name,".
6703                "file.portal=$portal,file.target=$target,file.lun=$lun,driver=raw";
6704     }
6705
6706     die "cannot convert iscsi path '$path', unkown format\n";
6707 }
6708
6709 sub qemu_img_convert {
6710     my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized) = @_;
6711
6712     my $storecfg = PVE::Storage::config();
6713     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
6714     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
6715
6716     die "destination '$dst_volid' is not a valid volid form qemu-img convert\n" if !$dst_storeid;
6717
6718     my $cachemode;
6719     my $src_path;
6720     my $src_is_iscsi = 0;
6721     my $src_format = 'raw';
6722
6723     if ($src_storeid) {
6724         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
6725         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
6726         $src_format = qemu_img_format($src_scfg, $src_volname);
6727         $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
6728         $src_is_iscsi = ($src_path =~ m|^iscsi://|);
6729         $cachemode = 'none' if $src_scfg->{type} eq 'zfspool';
6730     } elsif (-f $src_volid) {
6731         $src_path = $src_volid;
6732         if ($src_path =~ m/\.($QEMU_FORMAT_RE)$/) {
6733             $src_format = $1;
6734         }
6735     }
6736
6737     die "source '$src_volid' is not a valid volid nor path for qemu-img convert\n" if !$src_path;
6738
6739     my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6740     my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
6741     my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6742     my $dst_is_iscsi = ($dst_path =~ m|^iscsi://|);
6743
6744     my $cmd = [];
6745     push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
6746     push @$cmd, '-l', "snapshot.name=$snapname" if($snapname && $src_format eq "qcow2");
6747     push @$cmd, '-t', 'none' if $dst_scfg->{type} eq 'zfspool';
6748     push @$cmd, '-T', $cachemode if defined($cachemode);
6749
6750     if ($src_is_iscsi) {
6751         push @$cmd, '--image-opts';
6752         $src_path = convert_iscsi_path($src_path);
6753     } else {
6754         push @$cmd, '-f', $src_format;
6755     }
6756
6757     if ($dst_is_iscsi) {
6758         push @$cmd, '--target-image-opts';
6759         $dst_path = convert_iscsi_path($dst_path);
6760     } else {
6761         push @$cmd, '-O', $dst_format;
6762     }
6763
6764     push @$cmd, $src_path;
6765
6766     if (!$dst_is_iscsi && $is_zero_initialized) {
6767         push @$cmd, "zeroinit:$dst_path";
6768     } else {
6769         push @$cmd, $dst_path;
6770     }
6771
6772     my $parser = sub {
6773         my $line = shift;
6774         if($line =~ m/\((\S+)\/100\%\)/){
6775             my $percent = $1;
6776             my $transferred = int($size * $percent / 100);
6777             my $remaining = $size - $transferred;
6778
6779             print "transferred: $transferred bytes remaining: $remaining bytes total: $size bytes progression: $percent %\n";
6780         }
6781
6782     };
6783
6784     eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
6785     my $err = $@;
6786     die "copy failed: $err" if $err;
6787 }
6788
6789 sub qemu_img_format {
6790     my ($scfg, $volname) = @_;
6791
6792     if ($scfg->{path} && $volname =~ m/\.($QEMU_FORMAT_RE)$/) {
6793         return $1;
6794     } else {
6795         return "raw";
6796     }
6797 }
6798
6799 sub qemu_drive_mirror {
6800     my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $skipcomplete, $qga, $bwlimit) = @_;
6801
6802     $jobs = {} if !$jobs;
6803
6804     my $qemu_target;
6805     my $format;
6806     $jobs->{"drive-$drive"} = {};
6807
6808     if ($dst_volid =~ /^nbd:/) {
6809         $qemu_target = $dst_volid;
6810         $format = "nbd";
6811     } else {
6812         my $storecfg = PVE::Storage::config();
6813         my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
6814
6815         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6816
6817         $format = qemu_img_format($dst_scfg, $dst_volname);
6818
6819         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6820
6821         $qemu_target = $is_zero_initialized ? "zeroinit:$dst_path" : $dst_path;
6822     }
6823
6824     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $qemu_target };
6825     $opts->{format} = $format if $format;
6826
6827     if (defined($bwlimit)) {
6828         $opts->{speed} = $bwlimit * 1024;
6829         print "drive mirror is starting for drive-$drive with bandwidth limit: ${bwlimit} KB/s\n";
6830     } else {
6831         print "drive mirror is starting for drive-$drive\n";
6832     }
6833
6834     # if a job already runs for this device we get an error, catch it for cleanup
6835     eval { mon_cmd($vmid, "drive-mirror", %$opts); };
6836     if (my $err = $@) {
6837         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6838         warn "$@\n" if $@;
6839         die "mirroring error: $err\n";
6840     }
6841
6842     qemu_drive_mirror_monitor ($vmid, $vmiddst, $jobs, $skipcomplete, $qga);
6843 }
6844
6845 sub qemu_drive_mirror_monitor {
6846     my ($vmid, $vmiddst, $jobs, $skipcomplete, $qga) = @_;
6847
6848     eval {
6849         my $err_complete = 0;
6850
6851         while (1) {
6852             die "storage migration timed out\n" if $err_complete > 300;
6853
6854             my $stats = mon_cmd($vmid, "query-block-jobs");
6855
6856             my $running_mirror_jobs = {};
6857             foreach my $stat (@$stats) {
6858                 next if $stat->{type} ne 'mirror';
6859                 $running_mirror_jobs->{$stat->{device}} = $stat;
6860             }
6861
6862             my $readycounter = 0;
6863
6864             foreach my $job (keys %$jobs) {
6865
6866                 if(defined($jobs->{$job}->{complete}) && !defined($running_mirror_jobs->{$job})) {
6867                     print "$job : finished\n";
6868                     delete $jobs->{$job};
6869                     next;
6870                 }
6871
6872                 die "$job: mirroring has been cancelled\n" if !defined($running_mirror_jobs->{$job});
6873
6874                 my $busy = $running_mirror_jobs->{$job}->{busy};
6875                 my $ready = $running_mirror_jobs->{$job}->{ready};
6876                 if (my $total = $running_mirror_jobs->{$job}->{len}) {
6877                     my $transferred = $running_mirror_jobs->{$job}->{offset} || 0;
6878                     my $remaining = $total - $transferred;
6879                     my $percent = sprintf "%.2f", ($transferred * 100 / $total);
6880
6881                     print "$job: transferred: $transferred bytes remaining: $remaining bytes total: $total bytes progression: $percent % busy: $busy ready: $ready \n";
6882                 }
6883
6884                 $readycounter++ if $running_mirror_jobs->{$job}->{ready};
6885             }
6886
6887             last if scalar(keys %$jobs) == 0;
6888
6889             if ($readycounter == scalar(keys %$jobs)) {
6890                 print "all mirroring jobs are ready \n";
6891                 last if $skipcomplete; #do the complete later
6892
6893                 if ($vmiddst && $vmiddst != $vmid) {
6894                     my $agent_running = $qga && qga_check_running($vmid);
6895                     if ($agent_running) {
6896                         print "freeze filesystem\n";
6897                         eval { mon_cmd($vmid, "guest-fsfreeze-freeze"); };
6898                     } else {
6899                         print "suspend vm\n";
6900                         eval { PVE::QemuServer::vm_suspend($vmid, 1); };
6901                     }
6902
6903                     # if we clone a disk for a new target vm, we don't switch the disk
6904                     PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs);
6905
6906                     if ($agent_running) {
6907                         print "unfreeze filesystem\n";
6908                         eval { mon_cmd($vmid, "guest-fsfreeze-thaw"); };
6909                     } else {
6910                         print "resume vm\n";
6911                         eval {  PVE::QemuServer::vm_resume($vmid, 1, 1); };
6912                     }
6913
6914                     last;
6915                 } else {
6916
6917                     foreach my $job (keys %$jobs) {
6918                         # try to switch the disk if source and destination are on the same guest
6919                         print "$job: Completing block job...\n";
6920
6921                         eval { mon_cmd($vmid, "block-job-complete", device => $job) };
6922                         if ($@ =~ m/cannot be completed/) {
6923                             print "$job: Block job cannot be completed, try again.\n";
6924                             $err_complete++;
6925                         }else {
6926                             print "$job: Completed successfully.\n";
6927                             $jobs->{$job}->{complete} = 1;
6928                         }
6929                     }
6930                 }
6931             }
6932             sleep 1;
6933         }
6934     };
6935     my $err = $@;
6936
6937     if ($err) {
6938         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6939         die "mirroring error: $err";
6940     }
6941
6942 }
6943
6944 sub qemu_blockjobs_cancel {
6945     my ($vmid, $jobs) = @_;
6946
6947     foreach my $job (keys %$jobs) {
6948         print "$job: Cancelling block job\n";
6949         eval { mon_cmd($vmid, "block-job-cancel", device => $job); };
6950         $jobs->{$job}->{cancel} = 1;
6951     }
6952
6953     while (1) {
6954         my $stats = mon_cmd($vmid, "query-block-jobs");
6955
6956         my $running_jobs = {};
6957         foreach my $stat (@$stats) {
6958             $running_jobs->{$stat->{device}} = $stat;
6959         }
6960
6961         foreach my $job (keys %$jobs) {
6962
6963             if (defined($jobs->{$job}->{cancel}) && !defined($running_jobs->{$job})) {
6964                 print "$job: Done.\n";
6965                 delete $jobs->{$job};
6966             }
6967         }
6968
6969         last if scalar(keys %$jobs) == 0;
6970
6971         sleep 1;
6972     }
6973 }
6974
6975 sub clone_disk {
6976     my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
6977         $newvmid, $storage, $format, $full, $newvollist, $jobs, $skipcomplete, $qga, $bwlimit) = @_;
6978
6979     my $newvolid;
6980
6981     if (!$full) {
6982         print "create linked clone of drive $drivename ($drive->{file})\n";
6983         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
6984         push @$newvollist, $newvolid;
6985     } else {
6986
6987         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6988         $storeid = $storage if $storage;
6989
6990         my $dst_format = resolve_dst_disk_format($storecfg, $storeid, $volname, $format);
6991         my ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 3);
6992
6993         print "create full clone of drive $drivename ($drive->{file})\n";
6994         my $name = undef;
6995         if (drive_is_cloudinit($drive)) {
6996             $name = "vm-$newvmid-cloudinit";
6997             $name .= ".$dst_format" if $dst_format ne 'raw';
6998             $snapname = undef;
6999             $size = PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE;
7000         }
7001         $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $newvmid, $dst_format, $name, ($size/1024));
7002         push @$newvollist, $newvolid;
7003
7004         PVE::Storage::activate_volumes($storecfg, [$newvolid]);
7005
7006         if (drive_is_cloudinit($drive)) {
7007             goto no_data_clone;
7008         }
7009
7010         my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
7011         if (!$running || $snapname) {
7012             # TODO: handle bwlimits
7013             qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit);
7014         } else {
7015
7016             my $kvmver = get_running_qemu_version ($vmid);
7017             if (!min_version($kvmver, 2, 7)) {
7018                 die "drive-mirror with iothread requires qemu version 2.7 or higher\n"
7019                     if $drive->{iothread};
7020             }
7021
7022             qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid, $sparseinit, $jobs, $skipcomplete, $qga, $bwlimit);
7023         }
7024     }
7025
7026 no_data_clone:
7027     my ($size) = PVE::Storage::volume_size_info($storecfg, $newvolid, 3);
7028
7029     my $disk = $drive;
7030     $disk->{format} = undef;
7031     $disk->{file} = $newvolid;
7032     $disk->{size} = $size;
7033
7034     return $disk;
7035 }
7036
7037 sub get_running_qemu_version {
7038     my ($vmid) = @_;
7039     my $res = mon_cmd($vmid, "query-version");
7040     return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
7041 }
7042
7043 sub qemu_use_old_bios_files {
7044     my ($machine_type) = @_;
7045
7046     return if !$machine_type;
7047
7048     my $use_old_bios_files = undef;
7049
7050     if ($machine_type =~ m/^(\S+)\.pxe$/) {
7051         $machine_type = $1;
7052         $use_old_bios_files = 1;
7053     } else {
7054         my $version = PVE::QemuServer::Machine::extract_version($machine_type) // kvm_user_version();
7055         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
7056         # load new efi bios files on migration. So this hack is required to allow
7057         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
7058         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
7059         $use_old_bios_files = !min_version($version, 2, 4);
7060     }
7061
7062     return ($use_old_bios_files, $machine_type);
7063 }
7064
7065 sub create_efidisk($$$$$) {
7066     my ($storecfg, $storeid, $vmid, $fmt, $arch) = @_;
7067
7068     my (undef, $ovmf_vars) = get_ovmf_files($arch);
7069     die "EFI vars default image not found\n" if ! -f $ovmf_vars;
7070
7071     my $vars_size_b = -s $ovmf_vars;
7072     my $vars_size = PVE::Tools::convert_size($vars_size_b, 'b' => 'kb');
7073     my $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $fmt, undef, $vars_size);
7074     PVE::Storage::activate_volumes($storecfg, [$volid]);
7075
7076     qemu_img_convert($ovmf_vars, $volid, $vars_size_b, undef, 0);
7077
7078     return ($volid, $vars_size);
7079 }
7080
7081 sub vm_iothreads_list {
7082     my ($vmid) = @_;
7083
7084     my $res = mon_cmd($vmid, 'query-iothreads');
7085
7086     my $iothreads = {};
7087     foreach my $iothread (@$res) {
7088         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
7089     }
7090
7091     return $iothreads;
7092 }
7093
7094 sub scsihw_infos {
7095     my ($conf, $drive) = @_;
7096
7097     my $maxdev = 0;
7098
7099     if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)) {
7100         $maxdev = 7;
7101     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
7102         $maxdev = 1;
7103     } else {
7104         $maxdev = 256;
7105     }
7106
7107     my $controller = int($drive->{index} / $maxdev);
7108     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single') ? "virtioscsi" : "scsihw";
7109
7110     return ($maxdev, $controller, $controller_prefix);
7111 }
7112
7113 sub add_hyperv_enlightenments {
7114     my ($cpuFlags, $winversion, $machine_version, $bios, $gpu_passthrough, $hv_vendor_id) = @_;
7115
7116     return if $winversion < 6;
7117     return if $bios && $bios eq 'ovmf' && $winversion < 8;
7118
7119     if ($gpu_passthrough || defined($hv_vendor_id)) {
7120         $hv_vendor_id //= 'proxmox';
7121         push @$cpuFlags , "hv_vendor_id=$hv_vendor_id";
7122     }
7123
7124     if (min_version($machine_version, 2, 3)) {
7125         push @$cpuFlags , 'hv_spinlocks=0x1fff';
7126         push @$cpuFlags , 'hv_vapic';
7127         push @$cpuFlags , 'hv_time';
7128     } else {
7129         push @$cpuFlags , 'hv_spinlocks=0xffff';
7130     }
7131
7132     if (min_version($machine_version, 2, 6)) {
7133         push @$cpuFlags , 'hv_reset';
7134         push @$cpuFlags , 'hv_vpindex';
7135         push @$cpuFlags , 'hv_runtime';
7136     }
7137
7138     if ($winversion >= 7) {
7139         push @$cpuFlags , 'hv_relaxed';
7140
7141         if (min_version($machine_version, 2, 12)) {
7142             push @$cpuFlags , 'hv_synic';
7143             push @$cpuFlags , 'hv_stimer';
7144         }
7145
7146         if (min_version($machine_version, 3, 1)) {
7147             push @$cpuFlags , 'hv_ipi';
7148         }
7149     }
7150 }
7151
7152 sub windows_version {
7153     my ($ostype) = @_;
7154
7155     return 0 if !$ostype;
7156
7157     my $winversion = 0;
7158
7159     if($ostype eq 'wxp' || $ostype eq 'w2k3' || $ostype eq 'w2k') {
7160         $winversion = 5;
7161     } elsif($ostype eq 'w2k8' || $ostype eq 'wvista') {
7162         $winversion = 6;
7163     } elsif ($ostype =~ m/^win(\d+)$/) {
7164         $winversion = $1;
7165     }
7166
7167     return $winversion;
7168 }
7169
7170 sub resolve_dst_disk_format {
7171         my ($storecfg, $storeid, $src_volname, $format) = @_;
7172         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
7173
7174         if (!$format) {
7175             # if no target format is specified, use the source disk format as hint
7176             if ($src_volname) {
7177                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
7178                 $format = qemu_img_format($scfg, $src_volname);
7179             } else {
7180                 return $defFormat;
7181             }
7182         }
7183
7184         # test if requested format is supported - else use default
7185         my $supported = grep { $_ eq $format } @$validFormats;
7186         $format = $defFormat if !$supported;
7187         return $format;
7188 }
7189
7190 sub resolve_first_disk {
7191     my $conf = shift;
7192     my @disks = PVE::QemuServer::valid_drive_names();
7193     my $firstdisk;
7194     foreach my $ds (reverse @disks) {
7195         next if !$conf->{$ds};
7196         my $disk = PVE::QemuServer::parse_drive($ds, $conf->{$ds});
7197         next if PVE::QemuServer::drive_is_cdrom($disk);
7198         $firstdisk = $ds;
7199     }
7200     return $firstdisk;
7201 }
7202
7203 sub generate_uuid {
7204     my ($uuid, $uuid_str);
7205     UUID::generate($uuid);
7206     UUID::unparse($uuid, $uuid_str);
7207     return $uuid_str;
7208 }
7209
7210 sub generate_smbios1_uuid {
7211     return "uuid=".generate_uuid();
7212 }
7213
7214 sub nbd_stop {
7215     my ($vmid) = @_;
7216
7217     mon_cmd($vmid, 'nbd-server-stop');
7218 }
7219
7220 sub create_reboot_request {
7221     my ($vmid) = @_;
7222     open(my $fh, '>', "/run/qemu-server/$vmid.reboot")
7223         or die "failed to create reboot trigger file: $!\n";
7224     close($fh);
7225 }
7226
7227 sub clear_reboot_request {
7228     my ($vmid) = @_;
7229     my $path = "/run/qemu-server/$vmid.reboot";
7230     my $res = 0;
7231
7232     $res = unlink($path);
7233     die "could not remove reboot request for $vmid: $!"
7234         if !$res && $! != POSIX::ENOENT;
7235
7236     return $res;
7237 }
7238
7239 # bash completion helper
7240
7241 sub complete_backup_archives {
7242     my ($cmdname, $pname, $cvalue) = @_;
7243
7244     my $cfg = PVE::Storage::config();
7245
7246     my $storeid;
7247
7248     if ($cvalue =~ m/^([^:]+):/) {
7249         $storeid = $1;
7250     }
7251
7252     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
7253
7254     my $res = [];
7255     foreach my $id (keys %$data) {
7256         foreach my $item (@{$data->{$id}}) {
7257             next if $item->{format} !~ m/^vma\.(gz|lzo)$/;
7258             push @$res, $item->{volid} if defined($item->{volid});
7259         }
7260     }
7261
7262     return $res;
7263 }
7264
7265 my $complete_vmid_full = sub {
7266     my ($running) = @_;
7267
7268     my $idlist = vmstatus();
7269
7270     my $res = [];
7271
7272     foreach my $id (keys %$idlist) {
7273         my $d = $idlist->{$id};
7274         if (defined($running)) {
7275             next if $d->{template};
7276             next if $running && $d->{status} ne 'running';
7277             next if !$running && $d->{status} eq 'running';
7278         }
7279         push @$res, $id;
7280
7281     }
7282     return $res;
7283 };
7284
7285 sub complete_vmid {
7286     return &$complete_vmid_full();
7287 }
7288
7289 sub complete_vmid_stopped {
7290     return &$complete_vmid_full(0);
7291 }
7292
7293 sub complete_vmid_running {
7294     return &$complete_vmid_full(1);
7295 }
7296
7297 sub complete_storage {
7298
7299     my $cfg = PVE::Storage::config();
7300     my $ids = $cfg->{ids};
7301
7302     my $res = [];
7303     foreach my $sid (keys %$ids) {
7304         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
7305         next if !$ids->{$sid}->{content}->{images};
7306         push @$res, $sid;
7307     }
7308
7309     return $res;
7310 }
7311
7312 sub complete_migration_storage {
7313     my ($cmd, $param, $current_value, $all_args) = @_;
7314
7315     my $targetnode = @$all_args[1];
7316
7317     my $cfg = PVE::Storage::config();
7318     my $ids = $cfg->{ids};
7319
7320     my $res = [];
7321     foreach my $sid (keys %$ids) {
7322         next if !PVE::Storage::storage_check_enabled($cfg, $sid, $targetnode, 1);
7323         next if !$ids->{$sid}->{content}->{images};
7324         push @$res, $sid;
7325     }
7326
7327     return $res;
7328 }
7329
7330 1;