PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5
   6 use Cwd 'abs_path';
   7 use Digest::SHA;
   8 use Fcntl ':flock';
   9 use Fcntl;
  10 use File::Basename;
  11 use File::Copy qw(copy);
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use IO::Dir;
  16 use IO::File;
  17 use IO::Handle;
  18 use IO::Select;
  19 use IO::Socket::UNIX;
  20 use IPC::Open3;
  21 use JSON;
  22 use MIME::Base64;
  23 use POSIX;
  24 use Storable qw(dclone);
  25 use Time::HiRes qw(gettimeofday);
  26 use URI::Escape;
  27 use UUID;
  28
  29 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file cfs_lock_file);
  30 use PVE::DataCenterConfig;
  31 use PVE::Exception qw(raise raise_param_exc);
  32 use PVE::GuestHelpers;
  33 use PVE::INotify;
  34 use PVE::JSONSchema qw(get_standard_option);
  35 use PVE::ProcFSTools;
  36 use PVE::RPCEnvironment;
  37 use PVE::Storage;
  38 use PVE::SysFSTools;
  39 use PVE::Systemd;
  40 use PVE::Tools qw(run_command lock_file lock_file_full file_read_firstline dir_glob_foreach get_host_arch $IPV6RE);
  41
  42 use PVE::QMPClient;
  43 use PVE::QemuConfig;
  44 use PVE::QemuServer::Helpers qw(min_version);
  45 use PVE::QemuServer::Cloudinit;
  46 use PVE::QemuServer::Machine;
  47 use PVE::QemuServer::Memory;
  48 use PVE::QemuServer::Monitor qw(mon_cmd);
  49 use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr print_pcie_root_port);
  50 use PVE::QemuServer::USB qw(parse_usb_device);
  51
  52 my $EDK2_FW_BASE = '/usr/share/pve-edk2-firmware/';
  53 my $OVMF = {
  54     x86_64 => [
  55         "$EDK2_FW_BASE/OVMF_CODE.fd",
  56         "$EDK2_FW_BASE/OVMF_VARS.fd"
  57     ],
  58     aarch64 => [
  59         "$EDK2_FW_BASE/AAVMF_CODE.fd",
  60         "$EDK2_FW_BASE/AAVMF_VARS.fd"
  61     ],
  62 };
  63
  64 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  65
  66 my $QEMU_FORMAT_RE = qr/raw|cow|qcow|qcow2|qed|vmdk|cloop/;
  67
  68 # Note about locking: we use flock on the config file protect
  69 # against concurent actions.
  70 # Aditionaly, we have a 'lock' setting in the config file. This
  71 # can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
  72 # allowed when such lock is set. But you can ignore this kind of
  73 # lock with the --skiplock flag.
  74
  75 cfs_register_file('/qemu-server/',
  76                   \&parse_vm_config,
  77                   \&write_vm_config);
  78
  79 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
  80     description => "Some command save/restore state from this location.",
  81     type => 'string',
  82     maxLength => 128,
  83     optional => 1,
  84 });
  85
  86 PVE::JSONSchema::register_standard_option('pve-qm-image-format', {
  87     type => 'string',
  88     enum => [qw(raw cow qcow qed qcow2 vmdk cloop)],
  89     description => "The drive's backing file's data format.",
  90     optional => 1,
  91 });
  92
  93 PVE::JSONSchema::register_standard_option('pve-qemu-machine', {
  94         description => "Specifies the Qemu machine type.",
  95         type => 'string',
  96         pattern => '(pc|pc(-i440fx)?-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|q35|pc-q35-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|virt(?:-\d+(\.\d+)+)?(\+pve\d+)?)',
  97         maxLength => 40,
  98         optional => 1,
  99 });
 100
 101 #no warnings 'redefine';
 102
 103 sub cgroups_write {
 104    my ($controller, $vmid, $option, $value) = @_;
 105
 106    my $path = "/sys/fs/cgroup/$controller/qemu.slice/$vmid.scope/$option";
 107    PVE::ProcFSTools::write_proc_entry($path, $value);
 108
 109 }
 110
 111 my $nodename = PVE::INotify::nodename();
 112
 113 my $cpu_vendor_list = {
 114     # Intel CPUs
 115     486 => 'GenuineIntel',
 116     pentium => 'GenuineIntel',
 117     pentium2  => 'GenuineIntel',
 118     pentium3  => 'GenuineIntel',
 119     coreduo => 'GenuineIntel',
 120     core2duo => 'GenuineIntel',
 121     Conroe  => 'GenuineIntel',
 122     Penryn  => 'GenuineIntel',
 123     Nehalem  => 'GenuineIntel',
 124     'Nehalem-IBRS'  => 'GenuineIntel',
 125     Westmere => 'GenuineIntel',
 126     'Westmere-IBRS' => 'GenuineIntel',
 127     SandyBridge => 'GenuineIntel',
 128     'SandyBridge-IBRS' => 'GenuineIntel',
 129     IvyBridge => 'GenuineIntel',
 130     'IvyBridge-IBRS' => 'GenuineIntel',
 131     Haswell => 'GenuineIntel',
 132     'Haswell-IBRS' => 'GenuineIntel',
 133     'Haswell-noTSX' => 'GenuineIntel',
 134     'Haswell-noTSX-IBRS' => 'GenuineIntel',
 135     Broadwell => 'GenuineIntel',
 136     'Broadwell-IBRS' => 'GenuineIntel',
 137     'Broadwell-noTSX' => 'GenuineIntel',
 138     'Broadwell-noTSX-IBRS' => 'GenuineIntel',
 139     'Skylake-Client' => 'GenuineIntel',
 140     'Skylake-Client-IBRS' => 'GenuineIntel',
 141     'Skylake-Server' => 'GenuineIntel',
 142     'Skylake-Server-IBRS' => 'GenuineIntel',
 143     'Cascadelake-Server' => 'GenuineIntel',
 144     KnightsMill => 'GenuineIntel',
 145
 146
 147     # AMD CPUs
 148     athlon => 'AuthenticAMD',
 149     phenom  => 'AuthenticAMD',
 150     Opteron_G1  => 'AuthenticAMD',
 151     Opteron_G2  => 'AuthenticAMD',
 152     Opteron_G3  => 'AuthenticAMD',
 153     Opteron_G4  => 'AuthenticAMD',
 154     Opteron_G5  => 'AuthenticAMD',
 155     EPYC => 'AuthenticAMD',
 156     'EPYC-IBPB' => 'AuthenticAMD',
 157
 158     # generic types, use vendor from host node
 159     host => 'default',
 160     kvm32 => 'default',
 161     kvm64 => 'default',
 162     qemu32 => 'default',
 163     qemu64 => 'default',
 164     max => 'default',
 165 };
 166
 167 my @supported_cpu_flags = (
 168     'pcid',
 169     'spec-ctrl',
 170     'ibpb',
 171     'ssbd',
 172     'virt-ssbd',
 173     'amd-ssbd',
 174     'amd-no-ssb',
 175     'pdpe1gb',
 176     'md-clear',
 177     'hv-tlbflush',
 178     'hv-evmcs',
 179     'aes'
 180 );
 181 my $cpu_flag = qr/[+-](@{[join('|', @supported_cpu_flags)]})/;
 182
 183 my $cpu_fmt = {
 184     cputype => {
 185         description => "Emulated CPU type.",
 186         type => 'string',
 187         enum => [ sort { "\L$a" cmp "\L$b" } keys %$cpu_vendor_list ],
 188         default => 'kvm64',
 189         default_key => 1,
 190     },
 191     hidden => {
 192         description => "Do not identify as a KVM virtual machine.",
 193         type => 'boolean',
 194         optional => 1,
 195         default => 0
 196     },
 197     'hv-vendor-id' => {
 198         type => 'string',
 199         pattern => qr/[a-zA-Z0-9]{1,12}/,
 200         format_description => 'vendor-id',
 201         description => 'The Hyper-V vendor ID. Some drivers or programs inside Windows guests need a specific ID.',
 202         optional => 1,
 203     },
 204     flags => {
 205         description => "List of additional CPU flags separated by ';'."
 206                      . " Use '+FLAG' to enable, '-FLAG' to disable a flag."
 207                      . " Currently supported flags: @{[join(', ', @supported_cpu_flags)]}.",
 208         format_description => '+FLAG[;-FLAG...]',
 209         type => 'string',
 210         pattern => qr/$cpu_flag(;$cpu_flag)*/,
 211         optional => 1,
 212     },
 213 };
 214
 215 my $watchdog_fmt = {
 216     model => {
 217         default_key => 1,
 218         type => 'string',
 219         enum => [qw(i6300esb ib700)],
 220         description => "Watchdog type to emulate.",
 221         default => 'i6300esb',
 222         optional => 1,
 223     },
 224     action => {
 225         type => 'string',
 226         enum => [qw(reset shutdown poweroff pause debug none)],
 227         description => "The action to perform if after activation the guest fails to poll the watchdog in time.",
 228         optional => 1,
 229     },
 230 };
 231 PVE::JSONSchema::register_format('pve-qm-watchdog', $watchdog_fmt);
 232
 233 my $agent_fmt = {
 234     enabled => {
 235         description => "Enable/disable Qemu GuestAgent.",
 236         type => 'boolean',
 237         default => 0,
 238         default_key => 1,
 239     },
 240     fstrim_cloned_disks => {
 241         description => "Run fstrim after cloning/moving a disk.",
 242         type => 'boolean',
 243         optional => 1,
 244         default => 0
 245     },
 246     type => {
 247         description => "Select the agent type",
 248         type => 'string',
 249         default => 'virtio',
 250         optional => 1,
 251         enum => [qw(virtio isa)],
 252     },
 253 };
 254
 255 my $vga_fmt = {
 256     type => {
 257         description => "Select the VGA type.",
 258         type => 'string',
 259         default => 'std',
 260         optional => 1,
 261         default_key => 1,
 262         enum => [qw(cirrus qxl qxl2 qxl3 qxl4 none serial0 serial1 serial2 serial3 std virtio vmware)],
 263     },
 264     memory => {
 265         description => "Sets the VGA memory (in MiB). Has no effect with serial display.",
 266         type => 'integer',
 267         optional => 1,
 268         minimum => 4,
 269         maximum => 512,
 270     },
 271 };
 272
 273 my $ivshmem_fmt = {
 274     size => {
 275         type => 'integer',
 276         minimum => 1,
 277         description => "The size of the file in MB.",
 278     },
 279     name => {
 280         type => 'string',
 281         pattern => '[a-zA-Z0-9\-]+',
 282         optional => 1,
 283         format_description => 'string',
 284         description => "The name of the file. Will be prefixed with 'pve-shm-'. Default is the VMID. Will be deleted when the VM is stopped.",
 285     },
 286 };
 287
 288 my $audio_fmt = {
 289     device => {
 290         type => 'string',
 291         enum => [qw(ich9-intel-hda intel-hda AC97)],
 292         description =>  "Configure an audio device."
 293     },
 294     driver =>  {
 295         type => 'string',
 296         enum => ['spice'],
 297         default => 'spice',
 298         optional => 1,
 299         description => "Driver backend for the audio device."
 300     },
 301 };
 302
 303 my $spice_enhancements_fmt = {
 304     foldersharing => {
 305         type => 'boolean',
 306         optional => 1,
 307         default => '0',
 308         description =>  "Enable folder sharing via SPICE. Needs Spice-WebDAV daemon installed in the VM."
 309     },
 310     videostreaming =>  {
 311         type => 'string',
 312         enum => ['off', 'all', 'filter'],
 313         default => 'off',
 314         optional => 1,
 315         description => "Enable video streaming. Uses compression for detected video streams."
 316     },
 317 };
 318
 319 my $confdesc = {
 320     onboot => {
 321         optional => 1,
 322         type => 'boolean',
 323         description => "Specifies whether a VM will be started during system bootup.",
 324         default => 0,
 325     },
 326     autostart => {
 327         optional => 1,
 328         type => 'boolean',
 329         description => "Automatic restart after crash (currently ignored).",
 330         default => 0,
 331     },
 332     hotplug => {
 333         optional => 1,
 334         type => 'string', format => 'pve-hotplug-features',
 335         description => "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
 336         default => 'network,disk,usb',
 337     },
 338     reboot => {
 339         optional => 1,
 340         type => 'boolean',
 341         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 342         default => 1,
 343     },
 344     lock => {
 345         optional => 1,
 346         type => 'string',
 347         description => "Lock/unlock the VM.",
 348         enum => [qw(backup clone create migrate rollback snapshot snapshot-delete suspending suspended)],
 349     },
 350     cpulimit => {
 351         optional => 1,
 352         type => 'number',
 353         description => "Limit of CPU usage.",
 354         verbose_description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
 355         minimum => 0,
 356         maximum => 128,
 357         default => 0,
 358     },
 359     cpuunits => {
 360         optional => 1,
 361         type => 'integer',
 362         description => "CPU weight for a VM.",
 363         verbose_description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.",
 364         minimum => 2,
 365         maximum => 262144,
 366         default => 1024,
 367     },
 368     memory => {
 369         optional => 1,
 370         type => 'integer',
 371         description => "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
 372         minimum => 16,
 373         default => 512,
 374     },
 375     balloon => {
 376         optional => 1,
 377         type => 'integer',
 378         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 379         minimum => 0,
 380     },
 381     shares => {
 382         optional => 1,
 383         type => 'integer',
 384         description => "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning. Auto-ballooning is done by pvestatd.",
 385         minimum => 0,
 386         maximum => 50000,
 387         default => 1000,
 388     },
 389     keyboard => {
 390         optional => 1,
 391         type => 'string',
 392         description => "Keybord layout for vnc server. Default is read from the '/etc/pve/datacenter.cfg' configuration file.".
 393                        "It should not be necessary to set it.",
 394         enum => PVE::Tools::kvmkeymaplist(),
 395         default => undef,
 396     },
 397     name => {
 398         optional => 1,
 399         type => 'string', format => 'dns-name',
 400         description => "Set a name for the VM. Only used on the configuration web interface.",
 401     },
 402     scsihw => {
 403         optional => 1,
 404         type => 'string',
 405         description => "SCSI controller model",
 406         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 407         default => 'lsi',
 408     },
 409     description => {
 410         optional => 1,
 411         type => 'string',
 412         description => "Description for the VM. Only used on the configuration web interface. This is saved as comment inside the configuration file.",
 413     },
 414     ostype => {
 415         optional => 1,
 416         type => 'string',
 417         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 l24 l26 solaris)],
 418         description => "Specify guest operating system.",
 419         verbose_description => <<EODESC,
 420 Specify guest operating system. This is used to enable special
 421 optimization/features for specific operating systems:
 422
 423 [horizontal]
 424 other;; unspecified OS
 425 wxp;; Microsoft Windows XP
 426 w2k;; Microsoft Windows 2000
 427 w2k3;; Microsoft Windows 2003
 428 w2k8;; Microsoft Windows 2008
 429 wvista;; Microsoft Windows Vista
 430 win7;; Microsoft Windows 7
 431 win8;; Microsoft Windows 8/2012/2012r2
 432 win10;; Microsoft Windows 10/2016
 433 l24;; Linux 2.4 Kernel
 434 l26;; Linux 2.6 - 5.X Kernel
 435 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
 436 EODESC
 437     },
 438     boot => {
 439         optional => 1,
 440         type => 'string',
 441         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n).",
 442         pattern => '[acdn]{1,4}',
 443         default => 'cdn',
 444     },
 445     bootdisk => {
 446         optional => 1,
 447         type => 'string', format => 'pve-qm-bootdisk',
 448         description => "Enable booting from specified disk.",
 449         pattern => '(ide|sata|scsi|virtio)\d+',
 450     },
 451     smp => {
 452         optional => 1,
 453         type => 'integer',
 454         description => "The number of CPUs. Please use option -sockets instead.",
 455         minimum => 1,
 456         default => 1,
 457     },
 458     sockets => {
 459         optional => 1,
 460         type => 'integer',
 461         description => "The number of CPU sockets.",
 462         minimum => 1,
 463         default => 1,
 464     },
 465     cores => {
 466         optional => 1,
 467         type => 'integer',
 468         description => "The number of cores per socket.",
 469         minimum => 1,
 470         default => 1,
 471     },
 472     numa => {
 473         optional => 1,
 474         type => 'boolean',
 475         description => "Enable/disable NUMA.",
 476         default => 0,
 477     },
 478     hugepages => {
 479         optional => 1,
 480         type => 'string',
 481         description => "Enable/disable hugepages memory.",
 482         enum => [qw(any 2 1024)],
 483     },
 484     vcpus => {
 485         optional => 1,
 486         type => 'integer',
 487         description => "Number of hotplugged vcpus.",
 488         minimum => 1,
 489         default => 0,
 490     },
 491     acpi => {
 492         optional => 1,
 493         type => 'boolean',
 494         description => "Enable/disable ACPI.",
 495         default => 1,
 496     },
 497     agent => {
 498         optional => 1,
 499         description => "Enable/disable Qemu GuestAgent and its properties.",
 500         type => 'string',
 501         format => $agent_fmt,
 502     },
 503     kvm => {
 504         optional => 1,
 505         type => 'boolean',
 506         description => "Enable/disable KVM hardware virtualization.",
 507         default => 1,
 508     },
 509     tdf => {
 510         optional => 1,
 511         type => 'boolean',
 512         description => "Enable/disable time drift fix.",
 513         default => 0,
 514     },
 515     localtime => {
 516         optional => 1,
 517         type => 'boolean',
 518         description => "Set the real time clock to local time. This is enabled by default if ostype indicates a Microsoft OS.",
 519     },
 520     freeze => {
 521         optional => 1,
 522         type => 'boolean',
 523         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 524     },
 525     vga => {
 526         optional => 1,
 527         type => 'string', format => $vga_fmt,
 528         description => "Configure the VGA hardware.",
 529         verbose_description => "Configure the VGA Hardware. If you want to use ".
 530             "high resolution modes (>= 1280x1024x16) you may need to increase " .
 531             "the vga memory option. Since QEMU 2.9 the default VGA display type " .
 532             "is 'std' for all OS types besides some Windows versions (XP and " .
 533             "older) which use 'cirrus'. The 'qxl' option enables the SPICE " .
 534             "display server. For win* OS you can select how many independent " .
 535             "displays you want, Linux guests can add displays them self.\n".
 536             "You can also run without any graphic card, using a serial device as terminal.",
 537     },
 538     watchdog => {
 539         optional => 1,
 540         type => 'string', format => 'pve-qm-watchdog',
 541         description => "Create a virtual hardware watchdog device.",
 542         verbose_description => "Create a virtual hardware watchdog device. Once enabled" .
 543             " (by a guest action), the watchdog must be periodically polled " .
 544             "by an agent inside the guest or else the watchdog will reset " .
 545             "the guest (or execute the respective action specified)",
 546     },
 547     startdate => {
 548         optional => 1,
 549         type => 'string',
 550         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 551         description => "Set the initial date of the real time clock. Valid format for date are: 'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 552         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 553         default => 'now',
 554     },
 555     startup =>  get_standard_option('pve-startup-order'),
 556     template => {
 557         optional => 1,
 558         type => 'boolean',
 559         description => "Enable/disable Template.",
 560         default => 0,
 561     },
 562     args => {
 563         optional => 1,
 564         type => 'string',
 565         description => "Arbitrary arguments passed to kvm.",
 566         verbose_description => <<EODESCR,
 567 Arbitrary arguments passed to kvm, for example:
 568
 569 args: -no-reboot -no-hpet
 570
 571 NOTE: this option is for experts only.
 572 EODESCR
 573     },
 574     tablet => {
 575         optional => 1,
 576         type => 'boolean',
 577         default => 1,
 578         description => "Enable/disable the USB tablet device.",
 579         verbose_description => "Enable/disable the USB tablet device. This device is " .
 580             "usually needed to allow absolute mouse positioning with VNC. " .
 581             "Else the mouse runs out of sync with normal VNC clients. " .
 582             "If you're running lots of console-only guests on one host, " .
 583             "you may consider disabling this to save some context switches. " .
 584             "This is turned off by default if you use spice (-vga=qxl).",
 585     },
 586     migrate_speed => {
 587         optional => 1,
 588         type => 'integer',
 589         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 590         minimum => 0,
 591         default => 0,
 592     },
 593     migrate_downtime => {
 594         optional => 1,
 595         type => 'number',
 596         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 597         minimum => 0,
 598         default => 0.1,
 599     },
 600     cdrom => {
 601         optional => 1,
 602         type => 'string', format => 'pve-qm-ide',
 603         typetext => '<volume>',
 604         description => "This is an alias for option -ide2",
 605     },
 606     cpu => {
 607         optional => 1,
 608         description => "Emulated CPU type.",
 609         type => 'string',
 610         format => $cpu_fmt,
 611     },
 612     parent => get_standard_option('pve-snapshot-name', {
 613         optional => 1,
 614         description => "Parent snapshot name. This is used internally, and should not be modified.",
 615     }),
 616     snaptime => {
 617         optional => 1,
 618         description => "Timestamp for snapshots.",
 619         type => 'integer',
 620         minimum => 0,
 621     },
 622     vmstate => {
 623         optional => 1,
 624         type => 'string', format => 'pve-volume-id',
 625         description => "Reference to a volume which stores the VM state. This is used internally for snapshots.",
 626     },
 627     vmstatestorage => get_standard_option('pve-storage-id', {
 628         description => "Default storage for VM state volumes/files.",
 629         optional => 1,
 630     }),
 631     runningmachine => get_standard_option('pve-qemu-machine', {
 632         description => "Specifies the Qemu machine type of the running vm. This is used internally for snapshots.",
 633     }),
 634     machine => get_standard_option('pve-qemu-machine'),
 635     arch => {
 636         description => "Virtual processor architecture. Defaults to the host.",
 637         optional => 1,
 638         type => 'string',
 639         enum => [qw(x86_64 aarch64)],
 640     },
 641     smbios1 => {
 642         description => "Specify SMBIOS type 1 fields.",
 643         type => 'string', format => 'pve-qm-smbios1',
 644         maxLength => 512,
 645         optional => 1,
 646     },
 647     protection => {
 648         optional => 1,
 649         type => 'boolean',
 650         description => "Sets the protection flag of the VM. This will disable the remove VM and remove disk operations.",
 651         default => 0,
 652     },
 653     bios => {
 654         optional => 1,
 655         type => 'string',
 656         enum => [ qw(seabios ovmf) ],
 657         description => "Select BIOS implementation.",
 658         default => 'seabios',
 659     },
 660     vmgenid => {
 661         type => 'string',
 662         pattern => '(?:[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}|[01])',
 663         format_description => 'UUID',
 664         description => "Set VM Generation ID. Use '1' to autogenerate on create or update, pass '0' to disable explicitly.",
 665         verbose_description => "The VM generation ID (vmgenid) device exposes a".
 666             " 128-bit integer value identifier to the guest OS. This allows to".
 667             " notify the guest operating system when the virtual machine is".
 668             " executed with a different configuration (e.g. snapshot execution".
 669             " or creation from a template). The guest operating system notices".
 670             " the change, and is then able to react as appropriate by marking".
 671             " its copies of distributed databases as dirty, re-initializing its".
 672             " random number generator, etc.\n".
 673             "Note that auto-creation only works when done throug API/CLI create".
 674             " or update methods, but not when manually editing the config file.",
 675         default => "1 (autogenerated)",
 676         optional => 1,
 677     },
 678     hookscript => {
 679         type => 'string',
 680         format => 'pve-volume-id',
 681         optional => 1,
 682         description => "Script that will be executed during various steps in the vms lifetime.",
 683     },
 684     ivshmem => {
 685         type => 'string',
 686         format => $ivshmem_fmt,
 687         description => "Inter-VM shared memory. Useful for direct communication between VMs, or to the host.",
 688         optional => 1,
 689     },
 690     audio0 => {
 691         type => 'string',
 692         format => $audio_fmt,
 693         description => "Configure a audio device, useful in combination with QXL/Spice.",
 694         optional => 1
 695     },
 696     spice_enhancements => {
 697         type => 'string',
 698         format => $spice_enhancements_fmt,
 699         description => "Configure additional enhancements for SPICE.",
 700         optional => 1
 701     },
 702     tags => {
 703         type => 'string', format => 'pve-tag-list',
 704         description => 'Tags of the VM. This is only meta information.',
 705         optional => 1,
 706     },
 707 };
 708
 709 my $cicustom_fmt = {
 710     meta => {
 711         type => 'string',
 712         optional => 1,
 713         description => 'Specify a custom file containing all meta data passed to the VM via cloud-init. This is provider specific meaning configdrive2 and nocloud differ.',
 714         format => 'pve-volume-id',
 715         format_description => 'volume',
 716     },
 717     network => {
 718         type => 'string',
 719         optional => 1,
 720         description => 'Specify a custom file containing all network data passed to the VM via cloud-init.',
 721         format => 'pve-volume-id',
 722         format_description => 'volume',
 723     },
 724     user => {
 725         type => 'string',
 726         optional => 1,
 727         description => 'Specify a custom file containing all user data passed to the VM via cloud-init.',
 728         format => 'pve-volume-id',
 729         format_description => 'volume',
 730     },
 731 };
 732 PVE::JSONSchema::register_format('pve-qm-cicustom', $cicustom_fmt);
 733
 734 my $confdesc_cloudinit = {
 735     citype => {
 736         optional => 1,
 737         type => 'string',
 738         description => 'Specifies the cloud-init configuration format. The default depends on the configured operating system type (`ostype`. We use the `nocloud` format for Linux, and `configdrive2` for windows.',
 739         enum => ['configdrive2', 'nocloud'],
 740     },
 741     ciuser => {
 742         optional => 1,
 743         type => 'string',
 744         description => "cloud-init: User name to change ssh keys and password for instead of the image's configured default user.",
 745     },
 746     cipassword => {
 747         optional => 1,
 748         type => 'string',
 749         description => 'cloud-init: Password to assign the user. Using this is generally not recommended. Use ssh keys instead. Also note that older cloud-init versions do not support hashed passwords.',
 750     },
 751     cicustom => {
 752         optional => 1,
 753         type => 'string',
 754         description => 'cloud-init: Specify custom files to replace the automatically generated ones at start.',
 755         format => 'pve-qm-cicustom',
 756     },
 757     searchdomain => {
 758         optional => 1,
 759         type => 'string',
 760         description => "cloud-init: Sets DNS search domains for a container. Create will automatically use the setting from the host if neither searchdomain nor nameserver are set.",
 761     },
 762     nameserver => {
 763         optional => 1,
 764         type => 'string', format => 'address-list',
 765         description => "cloud-init: Sets DNS server IP address for a container. Create will automatically use the setting from the host if neither searchdomain nor nameserver are set.",
 766     },
 767     sshkeys => {
 768         optional => 1,
 769         type => 'string',
 770         format => 'urlencoded',
 771         description => "cloud-init: Setup public SSH keys (one key per line, OpenSSH format).",
 772     },
 773 };
 774
 775 # what about other qemu settings ?
 776 #cpu => 'string',
 777 #machine => 'string',
 778 #fda => 'file',
 779 #fdb => 'file',
 780 #mtdblock => 'file',
 781 #sd => 'file',
 782 #pflash => 'file',
 783 #snapshot => 'bool',
 784 #bootp => 'file',
 785 ##tftp => 'dir',
 786 ##smb => 'dir',
 787 #kernel => 'file',
 788 #append => 'string',
 789 #initrd => 'file',
 790 ##soundhw => 'string',
 791
 792 while (my ($k, $v) = each %$confdesc) {
 793     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 794 }
 795
 796 my $MAX_IDE_DISKS = 4;
 797 my $MAX_SCSI_DISKS = 14;
 798 my $MAX_VIRTIO_DISKS = 16;
 799 my $MAX_SATA_DISKS = 6;
 800 my $MAX_USB_DEVICES = 5;
 801 my $MAX_NETS = 32;
 802 my $MAX_UNUSED_DISKS = 256;
 803 my $MAX_HOSTPCI_DEVICES = 16;
 804 my $MAX_SERIAL_PORTS = 4;
 805 my $MAX_PARALLEL_PORTS = 3;
 806 my $MAX_NUMA = 8;
 807
 808 my $numa_fmt = {
 809     cpus => {
 810         type => "string",
 811         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 812         description => "CPUs accessing this NUMA node.",
 813         format_description => "id[-id];...",
 814     },
 815     memory => {
 816         type => "number",
 817         description => "Amount of memory this NUMA node provides.",
 818         optional => 1,
 819     },
 820     hostnodes => {
 821         type => "string",
 822         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 823         description => "Host NUMA nodes to use.",
 824         format_description => "id[-id];...",
 825         optional => 1,
 826     },
 827     policy => {
 828         type => 'string',
 829         enum => [qw(preferred bind interleave)],
 830         description => "NUMA allocation policy.",
 831         optional => 1,
 832     },
 833 };
 834 PVE::JSONSchema::register_format('pve-qm-numanode', $numa_fmt);
 835 my $numadesc = {
 836     optional => 1,
 837     type => 'string', format => $numa_fmt,
 838     description => "NUMA topology.",
 839 };
 840 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 841
 842 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 843     $confdesc->{"numa$i"} = $numadesc;
 844 }
 845
 846 my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000',  'pcnet',  'virtio',
 847                       'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3',
 848                       'e1000-82540em', 'e1000-82544gc', 'e1000-82545em'];
 849 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 850
 851 my $net_fmt_bridge_descr = <<__EOD__;
 852 Bridge to attach the network device to. The Proxmox VE standard bridge
 853 is called 'vmbr0'.
 854
 855 If you do not specify a bridge, we create a kvm user (NATed) network
 856 device, which provides DHCP and DNS services. The following addresses
 857 are used:
 858
 859  10.0.2.2   Gateway
 860  10.0.2.3   DNS Server
 861  10.0.2.4   SMB Server
 862
 863 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 864 __EOD__
 865
 866 my $net_fmt = {
 867     macaddr  => get_standard_option('mac-addr', {
 868         description => "MAC address. That address must be unique withing your network. This is automatically generated if not specified.",
 869     }),
 870     model => {
 871         type => 'string',
 872         description => "Network Card Model. The 'virtio' model provides the best performance with very low CPU overhead. If your guest does not support this driver, it is usually best to use 'e1000'.",
 873         enum => $nic_model_list,
 874         default_key => 1,
 875     },
 876     (map { $_ => { keyAlias => 'model', alias => 'macaddr' }} @$nic_model_list),
 877     bridge => {
 878         type => 'string',
 879         description => $net_fmt_bridge_descr,
 880         format_description => 'bridge',
 881         optional => 1,
 882     },
 883     queues => {
 884         type => 'integer',
 885         minimum => 0, maximum => 16,
 886         description => 'Number of packet queues to be used on the device.',
 887         optional => 1,
 888     },
 889     rate => {
 890         type => 'number',
 891         minimum => 0,
 892         description => "Rate limit in mbps (megabytes per second) as floating point number.",
 893         optional => 1,
 894     },
 895     tag => {
 896         type => 'integer',
 897         minimum => 1, maximum => 4094,
 898         description => 'VLAN tag to apply to packets on this interface.',
 899         optional => 1,
 900     },
 901     trunks => {
 902         type => 'string',
 903         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 904         description => 'VLAN trunks to pass through this interface.',
 905         format_description => 'vlanid[;vlanid...]',
 906         optional => 1,
 907     },
 908     firewall => {
 909         type => 'boolean',
 910         description => 'Whether this interface should be protected by the firewall.',
 911         optional => 1,
 912     },
 913     link_down => {
 914         type => 'boolean',
 915         description => 'Whether this interface should be disconnected (like pulling the plug).',
 916         optional => 1,
 917     },
 918 };
 919
 920 my $netdesc = {
 921     optional => 1,
 922     type => 'string', format => $net_fmt,
 923     description => "Specify network devices.",
 924 };
 925
 926 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 927
 928 my $ipconfig_fmt = {
 929     ip => {
 930         type => 'string',
 931         format => 'pve-ipv4-config',
 932         format_description => 'IPv4Format/CIDR',
 933         description => 'IPv4 address in CIDR format.',
 934         optional => 1,
 935         default => 'dhcp',
 936     },
 937     gw => {
 938         type => 'string',
 939         format => 'ipv4',
 940         format_description => 'GatewayIPv4',
 941         description => 'Default gateway for IPv4 traffic.',
 942         optional => 1,
 943         requires => 'ip',
 944     },
 945     ip6 => {
 946         type => 'string',
 947         format => 'pve-ipv6-config',
 948         format_description => 'IPv6Format/CIDR',
 949         description => 'IPv6 address in CIDR format.',
 950         optional => 1,
 951         default => 'dhcp',
 952     },
 953     gw6 => {
 954         type => 'string',
 955         format => 'ipv6',
 956         format_description => 'GatewayIPv6',
 957         description => 'Default gateway for IPv6 traffic.',
 958         optional => 1,
 959         requires => 'ip6',
 960     },
 961 };
 962 PVE::JSONSchema::register_format('pve-qm-ipconfig', $ipconfig_fmt);
 963 my $ipconfigdesc = {
 964     optional => 1,
 965     type => 'string', format => 'pve-qm-ipconfig',
 966     description => <<'EODESCR',
 967 cloud-init: Specify IP addresses and gateways for the corresponding interface.
 968
 969 IP addresses use CIDR notation, gateways are optional but need an IP of the same type specified.
 970
 971 The special string 'dhcp' can be used for IP addresses to use DHCP, in which case no explicit gateway should be provided.
 972 For IPv6 the special string 'auto' can be used to use stateless autoconfiguration.
 973
 974 If cloud-init is enabled and neither an IPv4 nor an IPv6 address is specified, it defaults to using dhcp on IPv4.
 975 EODESCR
 976 };
 977 PVE::JSONSchema::register_standard_option("pve-qm-ipconfig", $netdesc);
 978
 979 for (my $i = 0; $i < $MAX_NETS; $i++)  {
 980     $confdesc->{"net$i"} = $netdesc;
 981     $confdesc_cloudinit->{"ipconfig$i"} = $ipconfigdesc;
 982 }
 983
 984 foreach my $key (keys %$confdesc_cloudinit) {
 985     $confdesc->{$key} = $confdesc_cloudinit->{$key};
 986 }
 987
 988 PVE::JSONSchema::register_format('pve-volume-id-or-qm-path', \&verify_volume_id_or_qm_path);
 989 sub verify_volume_id_or_qm_path {
 990     my ($volid, $noerr) = @_;
 991
 992     if ($volid eq 'none' || $volid eq 'cdrom' || $volid =~ m|^/|) {
 993         return $volid;
 994     }
 995
 996     # if its neither 'none' nor 'cdrom' nor a path, check if its a volume-id
 997     $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') };
 998     if ($@) {
 999         return undef if $noerr;
1000         die $@;
1001     }
1002     return $volid;
1003 }
1004
1005 my $drivename_hash;
1006
1007 my %drivedesc_base = (
1008     volume => { alias => 'file' },
1009     file => {
1010         type => 'string',
1011         format => 'pve-volume-id-or-qm-path',
1012         default_key => 1,
1013         format_description => 'volume',
1014         description => "The drive's backing volume.",
1015     },
1016     media => {
1017         type => 'string',
1018         enum => [qw(cdrom disk)],
1019         description => "The drive's media type.",
1020         default => 'disk',
1021         optional => 1
1022     },
1023     cyls => {
1024         type => 'integer',
1025         description => "Force the drive's physical geometry to have a specific cylinder count.",
1026         optional => 1
1027     },
1028     heads => {
1029         type => 'integer',
1030         description => "Force the drive's physical geometry to have a specific head count.",
1031         optional => 1
1032     },
1033     secs => {
1034         type => 'integer',
1035         description => "Force the drive's physical geometry to have a specific sector count.",
1036         optional => 1
1037     },
1038     trans => {
1039         type => 'string',
1040         enum => [qw(none lba auto)],
1041         description => "Force disk geometry bios translation mode.",
1042         optional => 1,
1043     },
1044     snapshot => {
1045         type => 'boolean',
1046         description => "Controls qemu's snapshot mode feature."
1047             . " If activated, changes made to the disk are temporary and will"
1048             . " be discarded when the VM is shutdown.",
1049         optional => 1,
1050     },
1051     cache => {
1052         type => 'string',
1053         enum => [qw(none writethrough writeback unsafe directsync)],
1054         description => "The drive's cache mode",
1055         optional => 1,
1056     },
1057     format => get_standard_option('pve-qm-image-format'),
1058     size => {
1059         type => 'string',
1060         format => 'disk-size',
1061         format_description => 'DiskSize',
1062         description => "Disk size. This is purely informational and has no effect.",
1063         optional => 1,
1064     },
1065     backup => {
1066         type => 'boolean',
1067         description => "Whether the drive should be included when making backups.",
1068         optional => 1,
1069     },
1070     replicate => {
1071         type => 'boolean',
1072         description => 'Whether the drive should considered for replication jobs.',
1073         optional => 1,
1074         default => 1,
1075     },
1076     rerror => {
1077         type => 'string',
1078         enum => [qw(ignore report stop)],
1079         description => 'Read error action.',
1080         optional => 1,
1081     },
1082     werror => {
1083         type => 'string',
1084         enum => [qw(enospc ignore report stop)],
1085         description => 'Write error action.',
1086         optional => 1,
1087     },
1088     aio => {
1089         type => 'string',
1090         enum => [qw(native threads)],
1091         description => 'AIO type to use.',
1092         optional => 1,
1093     },
1094     discard => {
1095         type => 'string',
1096         enum => [qw(ignore on)],
1097         description => 'Controls whether to pass discard/trim requests to the underlying storage.',
1098         optional => 1,
1099     },
1100     detect_zeroes => {
1101         type => 'boolean',
1102         description => 'Controls whether to detect and try to optimize writes of zeroes.',
1103         optional => 1,
1104     },
1105     serial => {
1106         type => 'string',
1107         format => 'urlencoded',
1108         format_description => 'serial',
1109         maxLength => 20*3, # *3 since it's %xx url enoded
1110         description => "The drive's reported serial number, url-encoded, up to 20 bytes long.",
1111         optional => 1,
1112     },
1113     shared => {
1114         type => 'boolean',
1115         description => 'Mark this locally-managed volume as available on all nodes',
1116         verbose_description => "Mark this locally-managed volume as available on all nodes.\n\nWARNING: This option does not share the volume automatically, it assumes it is shared already!",
1117         optional => 1,
1118         default => 0,
1119     }
1120 );
1121
1122 my %iothread_fmt = ( iothread => {
1123         type => 'boolean',
1124         description => "Whether to use iothreads for this drive",
1125         optional => 1,
1126 });
1127
1128 my %model_fmt = (
1129     model => {
1130         type => 'string',
1131         format => 'urlencoded',
1132         format_description => 'model',
1133         maxLength => 40*3, # *3 since it's %xx url enoded
1134         description => "The drive's reported model name, url-encoded, up to 40 bytes long.",
1135         optional => 1,
1136     },
1137 );
1138
1139 my %queues_fmt = (
1140     queues => {
1141         type => 'integer',
1142         description => "Number of queues.",
1143         minimum => 2,
1144         optional => 1
1145     }
1146 );
1147
1148 my %scsiblock_fmt = (
1149     scsiblock => {
1150         type => 'boolean',
1151         description => "whether to use scsi-block for full passthrough of host block device\n\nWARNING: can lead to I/O errors in combination with low memory or high memory fragmentation on host",
1152         optional => 1,
1153         default => 0,
1154     },
1155 );
1156
1157 my %ssd_fmt = (
1158     ssd => {
1159         type => 'boolean',
1160         description => "Whether to expose this drive as an SSD, rather than a rotational hard disk.",
1161         optional => 1,
1162     },
1163 );
1164
1165 my %wwn_fmt = (
1166     wwn => {
1167         type => 'string',
1168         pattern => qr/^(0x)[0-9a-fA-F]{16}/,
1169         format_description => 'wwn',
1170         description => "The drive's worldwide name, encoded as 16 bytes hex string, prefixed by '0x'.",
1171         optional => 1,
1172     },
1173 );
1174
1175 my $add_throttle_desc = sub {
1176     my ($key, $type, $what, $unit, $longunit, $minimum) = @_;
1177     my $d = {
1178         type => $type,
1179         format_description => $unit,
1180         description => "Maximum $what in $longunit.",
1181         optional => 1,
1182     };
1183     $d->{minimum} = $minimum if defined($minimum);
1184     $drivedesc_base{$key} = $d;
1185 };
1186 # throughput: (leaky bucket)
1187 $add_throttle_desc->('bps',     'integer', 'r/w speed',   'bps',  'bytes per second');
1188 $add_throttle_desc->('bps_rd',  'integer', 'read speed',  'bps',  'bytes per second');
1189 $add_throttle_desc->('bps_wr',  'integer', 'write speed', 'bps',  'bytes per second');
1190 $add_throttle_desc->('mbps',    'number',  'r/w speed',   'mbps', 'megabytes per second');
1191 $add_throttle_desc->('mbps_rd', 'number',  'read speed',  'mbps', 'megabytes per second');
1192 $add_throttle_desc->('mbps_wr', 'number',  'write speed', 'mbps', 'megabytes per second');
1193 $add_throttle_desc->('iops',    'integer', 'r/w I/O',     'iops', 'operations per second');
1194 $add_throttle_desc->('iops_rd', 'integer', 'read I/O',    'iops', 'operations per second');
1195 $add_throttle_desc->('iops_wr', 'integer', 'write I/O',   'iops', 'operations per second');
1196
1197 # pools: (pool of IO before throttling starts taking effect)
1198 $add_throttle_desc->('mbps_max',    'number',  'unthrottled r/w pool',       'mbps', 'megabytes per second');
1199 $add_throttle_desc->('mbps_rd_max', 'number',  'unthrottled read pool',      'mbps', 'megabytes per second');
1200 $add_throttle_desc->('mbps_wr_max', 'number',  'unthrottled write pool',     'mbps', 'megabytes per second');
1201 $add_throttle_desc->('iops_max',    'integer', 'unthrottled r/w I/O pool',   'iops', 'operations per second');
1202 $add_throttle_desc->('iops_rd_max', 'integer', 'unthrottled read I/O pool',  'iops', 'operations per second');
1203 $add_throttle_desc->('iops_wr_max', 'integer', 'unthrottled write I/O pool', 'iops', 'operations per second');
1204
1205 # burst lengths
1206 $add_throttle_desc->('bps_max_length',     'integer', 'length of I/O bursts',       'seconds', 'seconds', 1);
1207 $add_throttle_desc->('bps_rd_max_length',  'integer', 'length of read I/O bursts',  'seconds', 'seconds', 1);
1208 $add_throttle_desc->('bps_wr_max_length',  'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
1209 $add_throttle_desc->('iops_max_length',    'integer', 'length of I/O bursts',       'seconds', 'seconds', 1);
1210 $add_throttle_desc->('iops_rd_max_length', 'integer', 'length of read I/O bursts',  'seconds', 'seconds', 1);
1211 $add_throttle_desc->('iops_wr_max_length', 'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
1212
1213 # legacy support
1214 $drivedesc_base{'bps_rd_length'} = { alias => 'bps_rd_max_length' };
1215 $drivedesc_base{'bps_wr_length'} = { alias => 'bps_wr_max_length' };
1216 $drivedesc_base{'iops_rd_length'} = { alias => 'iops_rd_max_length' };
1217 $drivedesc_base{'iops_wr_length'} = { alias => 'iops_wr_max_length' };
1218
1219 my $ide_fmt = {
1220     %drivedesc_base,
1221     %model_fmt,
1222     %ssd_fmt,
1223     %wwn_fmt,
1224 };
1225 PVE::JSONSchema::register_format("pve-qm-ide", $ide_fmt);
1226
1227 my $idedesc = {
1228     optional => 1,
1229     type => 'string', format => $ide_fmt,
1230     description => "Use volume as IDE hard disk or CD-ROM (n is 0 to " .($MAX_IDE_DISKS -1) . ").",
1231 };
1232 PVE::JSONSchema::register_standard_option("pve-qm-ide", $idedesc);
1233
1234 my $scsi_fmt = {
1235     %drivedesc_base,
1236     %iothread_fmt,
1237     %queues_fmt,
1238     %scsiblock_fmt,
1239     %ssd_fmt,
1240     %wwn_fmt,
1241 };
1242 my $scsidesc = {
1243     optional => 1,
1244     type => 'string', format => $scsi_fmt,
1245     description => "Use volume as SCSI hard disk or CD-ROM (n is 0 to " . ($MAX_SCSI_DISKS - 1) . ").",
1246 };
1247 PVE::JSONSchema::register_standard_option("pve-qm-scsi", $scsidesc);
1248
1249 my $sata_fmt = {
1250     %drivedesc_base,
1251     %ssd_fmt,
1252     %wwn_fmt,
1253 };
1254 my $satadesc = {
1255     optional => 1,
1256     type => 'string', format => $sata_fmt,
1257     description => "Use volume as SATA hard disk or CD-ROM (n is 0 to " . ($MAX_SATA_DISKS - 1). ").",
1258 };
1259 PVE::JSONSchema::register_standard_option("pve-qm-sata", $satadesc);
1260
1261 my $virtio_fmt = {
1262     %drivedesc_base,
1263     %iothread_fmt,
1264 };
1265 my $virtiodesc = {
1266     optional => 1,
1267     type => 'string', format => $virtio_fmt,
1268     description => "Use volume as VIRTIO hard disk (n is 0 to " . ($MAX_VIRTIO_DISKS - 1) . ").",
1269 };
1270 PVE::JSONSchema::register_standard_option("pve-qm-virtio", $virtiodesc);
1271
1272 my $alldrive_fmt = {
1273     %drivedesc_base,
1274     %iothread_fmt,
1275     %model_fmt,
1276     %queues_fmt,
1277     %scsiblock_fmt,
1278     %ssd_fmt,
1279     %wwn_fmt,
1280 };
1281
1282 my $efidisk_fmt = {
1283     volume => { alias => 'file' },
1284     file => {
1285         type => 'string',
1286         format => 'pve-volume-id-or-qm-path',
1287         default_key => 1,
1288         format_description => 'volume',
1289         description => "The drive's backing volume.",
1290     },
1291     format => get_standard_option('pve-qm-image-format'),
1292     size => {
1293         type => 'string',
1294         format => 'disk-size',
1295         format_description => 'DiskSize',
1296         description => "Disk size. This is purely informational and has no effect.",
1297         optional => 1,
1298     },
1299 };
1300
1301 my $efidisk_desc = {
1302     optional => 1,
1303     type => 'string', format => $efidisk_fmt,
1304     description => "Configure a Disk for storing EFI vars",
1305 };
1306
1307 PVE::JSONSchema::register_standard_option("pve-qm-efidisk", $efidisk_desc);
1308
1309 my $usb_fmt = {
1310     host => {
1311         default_key => 1,
1312         type => 'string', format => 'pve-qm-usb-device',
1313         format_description => 'HOSTUSBDEVICE|spice',
1314         description => <<EODESCR,
1315 The Host USB device or port or the value 'spice'. HOSTUSBDEVICE syntax is:
1316
1317  'bus-port(.port)*' (decimal numbers) or
1318  'vendor_id:product_id' (hexadeciaml numbers) or
1319  'spice'
1320
1321 You can use the 'lsusb -t' command to list existing usb devices.
1322
1323 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1324
1325 The value 'spice' can be used to add a usb redirection devices for spice.
1326 EODESCR
1327     },
1328     usb3 => {
1329         optional => 1,
1330         type => 'boolean',
1331         description => "Specifies whether if given host option is a USB3 device or port.",
1332         default => 0,
1333     },
1334 };
1335
1336 my $usbdesc = {
1337     optional => 1,
1338     type => 'string', format => $usb_fmt,
1339     description => "Configure an USB device (n is 0 to 4).",
1340 };
1341 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
1342
1343 my $PCIRE = qr/([a-f0-9]{4}:)?[a-f0-9]{2}:[a-f0-9]{2}(?:\.[a-f0-9])?/;
1344 my $hostpci_fmt = {
1345     host => {
1346         default_key => 1,
1347         type => 'string',
1348         pattern => qr/$PCIRE(;$PCIRE)*/,
1349         format_description => 'HOSTPCIID[;HOSTPCIID2...]',
1350         description => <<EODESCR,
1351 Host PCI device pass through. The PCI ID of a host's PCI device or a list
1352 of PCI virtual functions of the host. HOSTPCIID syntax is:
1353
1354 'bus:dev.func' (hexadecimal numbers)
1355
1356 You can us the 'lspci' command to list existing PCI devices.
1357 EODESCR
1358     },
1359     rombar => {
1360         type => 'boolean',
1361         description =>  "Specify whether or not the device's ROM will be visible in the guest's memory map.",
1362         optional => 1,
1363         default => 1,
1364     },
1365     romfile => {
1366         type => 'string',
1367         pattern => '[^,;]+',
1368         format_description => 'string',
1369         description => "Custom pci device rom filename (must be located in /usr/share/kvm/).",
1370         optional => 1,
1371     },
1372     pcie => {
1373         type => 'boolean',
1374         description =>  "Choose the PCI-express bus (needs the 'q35' machine model).",
1375         optional => 1,
1376         default => 0,
1377     },
1378     'x-vga' => {
1379         type => 'boolean',
1380         description =>  "Enable vfio-vga device support.",
1381         optional => 1,
1382         default => 0,
1383     },
1384     'mdev' => {
1385         type => 'string',
1386         format_description => 'string',
1387         pattern => '[^/\.:]+',
1388         optional => 1,
1389         description => <<EODESCR
1390 The type of mediated device to use.
1391 An instance of this type will be created on startup of the VM and
1392 will be cleaned up when the VM stops.
1393 EODESCR
1394     }
1395 };
1396 PVE::JSONSchema::register_format('pve-qm-hostpci', $hostpci_fmt);
1397
1398 my $hostpcidesc = {
1399         optional => 1,
1400         type => 'string', format => 'pve-qm-hostpci',
1401         description => "Map host PCI devices into guest.",
1402         verbose_description =>  <<EODESCR,
1403 Map host PCI devices into guest.
1404
1405 NOTE: This option allows direct access to host hardware. So it is no longer
1406 possible to migrate such machines - use with special care.
1407
1408 CAUTION: Experimental! User reported problems with this option.
1409 EODESCR
1410 };
1411 PVE::JSONSchema::register_standard_option("pve-qm-hostpci", $hostpcidesc);
1412
1413 my $serialdesc = {
1414         optional => 1,
1415         type => 'string',
1416         pattern => '(/dev/.+|socket)',
1417         description =>  "Create a serial device inside the VM (n is 0 to 3)",
1418         verbose_description =>  <<EODESCR,
1419 Create a serial device inside the VM (n is 0 to 3), and pass through a
1420 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1421 host side (use 'qm terminal' to open a terminal connection).
1422
1423 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines - use with special care.
1424
1425 CAUTION: Experimental! User reported problems with this option.
1426 EODESCR
1427 };
1428
1429 my $paralleldesc= {
1430         optional => 1,
1431         type => 'string',
1432         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
1433         description =>  "Map host parallel devices (n is 0 to 2).",
1434         verbose_description =>  <<EODESCR,
1435 Map host parallel devices (n is 0 to 2).
1436
1437 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1438
1439 CAUTION: Experimental! User reported problems with this option.
1440 EODESCR
1441 };
1442
1443 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
1444     $confdesc->{"parallel$i"} = $paralleldesc;
1445 }
1446
1447 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
1448     $confdesc->{"serial$i"} = $serialdesc;
1449 }
1450
1451 for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
1452     $confdesc->{"hostpci$i"} = $hostpcidesc;
1453 }
1454
1455 for (my $i = 0; $i < $MAX_IDE_DISKS; $i++)  {
1456     $drivename_hash->{"ide$i"} = 1;
1457     $confdesc->{"ide$i"} = $idedesc;
1458 }
1459
1460 for (my $i = 0; $i < $MAX_SATA_DISKS; $i++)  {
1461     $drivename_hash->{"sata$i"} = 1;
1462     $confdesc->{"sata$i"} = $satadesc;
1463 }
1464
1465 for (my $i = 0; $i < $MAX_SCSI_DISKS; $i++)  {
1466     $drivename_hash->{"scsi$i"} = 1;
1467     $confdesc->{"scsi$i"} = $scsidesc ;
1468 }
1469
1470 for (my $i = 0; $i < $MAX_VIRTIO_DISKS; $i++)  {
1471     $drivename_hash->{"virtio$i"} = 1;
1472     $confdesc->{"virtio$i"} = $virtiodesc;
1473 }
1474
1475 $drivename_hash->{efidisk0} = 1;
1476 $confdesc->{efidisk0} = $efidisk_desc;
1477
1478 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
1479     $confdesc->{"usb$i"} = $usbdesc;
1480 }
1481
1482 my $unuseddesc = {
1483     optional => 1,
1484     type => 'string', format => 'pve-volume-id',
1485     description => "Reference to unused volumes. This is used internally, and should not be modified manually.",
1486 };
1487
1488 for (my $i = 0; $i < $MAX_UNUSED_DISKS; $i++)  {
1489     $confdesc->{"unused$i"} = $unuseddesc;
1490 }
1491
1492 my $kvm_api_version = 0;
1493
1494 sub kvm_version {
1495     return $kvm_api_version if $kvm_api_version;
1496
1497     open my $fh, '<', '/dev/kvm'
1498         or return undef;
1499
1500     # 0xae00 => KVM_GET_API_VERSION
1501     $kvm_api_version = ioctl($fh, 0xae00, 0);
1502
1503     return $kvm_api_version;
1504 }
1505
1506 my $kvm_user_version = {};
1507 my $kvm_mtime = {};
1508
1509 sub kvm_user_version {
1510     my ($binary) = @_;
1511
1512     $binary //= get_command_for_arch(get_host_arch()); # get the native arch by default
1513     my $st = stat($binary);
1514
1515     my $cachedmtime = $kvm_mtime->{$binary} // -1;
1516     return $kvm_user_version->{$binary} if $kvm_user_version->{$binary} &&
1517         $cachedmtime == $st->mtime;
1518
1519     $kvm_user_version->{$binary} = 'unknown';
1520     $kvm_mtime->{$binary} = $st->mtime;
1521
1522     my $code = sub {
1523         my $line = shift;
1524         if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1525             $kvm_user_version->{$binary} = $2;
1526         }
1527     };
1528
1529     eval { run_command([$binary, '--version'], outfunc => $code); };
1530     warn $@ if $@;
1531
1532     return $kvm_user_version->{$binary};
1533
1534 }
1535
1536 sub kernel_has_vhost_net {
1537     return -c '/dev/vhost-net';
1538 }
1539
1540 sub valid_drive_names {
1541     # order is important - used to autoselect boot disk
1542     return ((map { "ide$_" } (0 .. ($MAX_IDE_DISKS - 1))),
1543             (map { "scsi$_" } (0 .. ($MAX_SCSI_DISKS - 1))),
1544             (map { "virtio$_" } (0 .. ($MAX_VIRTIO_DISKS - 1))),
1545             (map { "sata$_" } (0 .. ($MAX_SATA_DISKS - 1))),
1546             'efidisk0');
1547 }
1548
1549 sub is_valid_drivename {
1550     my $dev = shift;
1551
1552     return defined($drivename_hash->{$dev});
1553 }
1554
1555 sub option_exists {
1556     my $key = shift;
1557     return defined($confdesc->{$key});
1558 }
1559
1560 my $cdrom_path;
1561 sub get_cdrom_path {
1562
1563     return  $cdrom_path if $cdrom_path;
1564
1565     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
1566     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
1567     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
1568 }
1569
1570 sub get_iso_path {
1571     my ($storecfg, $vmid, $cdrom) = @_;
1572
1573     if ($cdrom eq 'cdrom') {
1574         return get_cdrom_path();
1575     } elsif ($cdrom eq 'none') {
1576         return '';
1577     } elsif ($cdrom =~ m|^/|) {
1578         return $cdrom;
1579     } else {
1580         return PVE::Storage::path($storecfg, $cdrom);
1581     }
1582 }
1583
1584 # try to convert old style file names to volume IDs
1585 sub filename_to_volume_id {
1586     my ($vmid, $file, $media) = @_;
1587
1588      if (!($file eq 'none' || $file eq 'cdrom' ||
1589           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1590
1591         return undef if $file =~ m|/|;
1592
1593         if ($media && $media eq 'cdrom') {
1594             $file = "local:iso/$file";
1595         } else {
1596             $file = "local:$vmid/$file";
1597         }
1598     }
1599
1600     return $file;
1601 }
1602
1603 sub verify_media_type {
1604     my ($opt, $vtype, $media) = @_;
1605
1606     return if !$media;
1607
1608     my $etype;
1609     if ($media eq 'disk') {
1610         $etype = 'images';
1611     } elsif ($media eq 'cdrom') {
1612         $etype = 'iso';
1613     } else {
1614         die "internal error";
1615     }
1616
1617     return if ($vtype eq $etype);
1618
1619     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1620 }
1621
1622 sub cleanup_drive_path {
1623     my ($opt, $storecfg, $drive) = @_;
1624
1625     # try to convert filesystem paths to volume IDs
1626
1627     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1628         ($drive->{file} !~ m|^/dev/.+|) &&
1629         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1630         ($drive->{file} !~ m/^\d+$/)) {
1631         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1632         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"}) if !$vtype;
1633         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1634         verify_media_type($opt, $vtype, $drive->{media});
1635         $drive->{file} = $volid;
1636     }
1637
1638     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1639 }
1640
1641 sub parse_hotplug_features {
1642     my ($data) = @_;
1643
1644     my $res = {};
1645
1646     return $res if $data eq '0';
1647
1648     $data = $confdesc->{hotplug}->{default} if $data eq '1';
1649
1650     foreach my $feature (PVE::Tools::split_list($data)) {
1651         if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
1652             $res->{$1} = 1;
1653         } else {
1654             die "invalid hotplug feature '$feature'\n";
1655         }
1656     }
1657     return $res;
1658 }
1659
1660 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1661 sub pve_verify_hotplug_features {
1662     my ($value, $noerr) = @_;
1663
1664     return $value if parse_hotplug_features($value);
1665
1666     return undef if $noerr;
1667
1668     die "unable to parse hotplug option\n";
1669 }
1670
1671 # ideX = [volume=]volume-id[,media=d][,cyls=c,heads=h,secs=s[,trans=t]]
1672 #        [,snapshot=on|off][,cache=on|off][,format=f][,backup=yes|no]
1673 #        [,rerror=ignore|report|stop][,werror=enospc|ignore|report|stop]
1674 #        [,aio=native|threads][,discard=ignore|on][,detect_zeroes=on|off]
1675 #        [,iothread=on][,serial=serial][,model=model]
1676
1677 sub parse_drive {
1678     my ($key, $data) = @_;
1679
1680     my ($interface, $index);
1681
1682     if ($key =~ m/^([^\d]+)(\d+)$/) {
1683         $interface = $1;
1684         $index = $2;
1685     } else {
1686         return undef;
1687     }
1688
1689     my $desc = $key =~ /^unused\d+$/ ? $alldrive_fmt
1690                                      : $confdesc->{$key}->{format};
1691     if (!$desc) {
1692         warn "invalid drive key: $key\n";
1693         return undef;
1694     }
1695     my $res = eval { PVE::JSONSchema::parse_property_string($desc, $data) };
1696     return undef if !$res;
1697     $res->{interface} = $interface;
1698     $res->{index} = $index;
1699
1700     my $error = 0;
1701     foreach my $opt (qw(bps bps_rd bps_wr)) {
1702         if (my $bps = defined(delete $res->{$opt})) {
1703             if (defined($res->{"m$opt"})) {
1704                 warn "both $opt and m$opt specified\n";
1705                 ++$error;
1706                 next;
1707             }
1708             $res->{"m$opt"} = sprintf("%.3f", $bps / (1024*1024.0));
1709         }
1710     }
1711
1712     # can't use the schema's 'requires' because of the mbps* => bps* "transforming aliases"
1713     for my $requirement (
1714         [mbps_max => 'mbps'],
1715         [mbps_rd_max => 'mbps_rd'],
1716         [mbps_wr_max => 'mbps_wr'],
1717         [miops_max => 'miops'],
1718         [miops_rd_max => 'miops_rd'],
1719         [miops_wr_max => 'miops_wr'],
1720         [bps_max_length => 'mbps_max'],
1721         [bps_rd_max_length => 'mbps_rd_max'],
1722         [bps_wr_max_length => 'mbps_wr_max'],
1723         [iops_max_length => 'iops_max'],
1724         [iops_rd_max_length => 'iops_rd_max'],
1725         [iops_wr_max_length => 'iops_wr_max']) {
1726         my ($option, $requires) = @$requirement;
1727         if ($res->{$option} && !$res->{$requires}) {
1728             warn "$option requires $requires\n";
1729             ++$error;
1730         }
1731     }
1732
1733     return undef if $error;
1734
1735     return undef if $res->{mbps_rd} && $res->{mbps};
1736     return undef if $res->{mbps_wr} && $res->{mbps};
1737     return undef if $res->{iops_rd} && $res->{iops};
1738     return undef if $res->{iops_wr} && $res->{iops};
1739
1740     if ($res->{media} && ($res->{media} eq 'cdrom')) {
1741         return undef if $res->{snapshot} || $res->{trans} || $res->{format};
1742         return undef if $res->{heads} || $res->{secs} || $res->{cyls};
1743         return undef if $res->{interface} eq 'virtio';
1744     }
1745
1746     if (my $size = $res->{size}) {
1747         return undef if !defined($res->{size} = PVE::JSONSchema::parse_size($size));
1748     }
1749
1750     return $res;
1751 }
1752
1753 sub print_drive {
1754     my ($drive) = @_;
1755     my $data = { %$drive };
1756     delete $data->{$_} for qw(index interface);
1757     return PVE::JSONSchema::print_property_string($data, $alldrive_fmt);
1758 }
1759
1760 sub scsi_inquiry {
1761     my($fh, $noerr) = @_;
1762
1763     my $SG_IO = 0x2285;
1764     my $SG_GET_VERSION_NUM = 0x2282;
1765
1766     my $versionbuf = "\x00" x 8;
1767     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1768     if (!$ret) {
1769         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1770         return undef;
1771     }
1772     my $version = unpack("I", $versionbuf);
1773     if ($version < 30000) {
1774         die "scsi generic interface too old\n"  if !$noerr;
1775         return undef;
1776     }
1777
1778     my $buf = "\x00" x 36;
1779     my $sensebuf = "\x00" x 8;
1780     my $cmd = pack("C x3 C x1", 0x12, 36);
1781
1782     # see /usr/include/scsi/sg.h
1783     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1784
1785     my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1786                       length($sensebuf), 0, length($buf), $buf,
1787                       $cmd, $sensebuf, 6000);
1788
1789     $ret = ioctl($fh, $SG_IO, $packet);
1790     if (!$ret) {
1791         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1792         return undef;
1793     }
1794
1795     my @res = unpack($sg_io_hdr_t, $packet);
1796     if ($res[17] || $res[18]) {
1797         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1798         return undef;
1799     }
1800
1801     my $res = {};
1802     (my $byte0, my $byte1, $res->{vendor},
1803      $res->{product}, $res->{revision}) = unpack("C C x6 A8 A16 A4", $buf);
1804
1805     $res->{removable} = $byte1 & 128 ? 1 : 0;
1806     $res->{type} = $byte0 & 31;
1807
1808     return $res;
1809 }
1810
1811 sub path_is_scsi {
1812     my ($path) = @_;
1813
1814     my $fh = IO::File->new("+<$path") || return undef;
1815     my $res = scsi_inquiry($fh, 1);
1816     close($fh);
1817
1818     return $res;
1819 }
1820
1821 sub print_tabletdevice_full {
1822     my ($conf, $arch) = @_;
1823
1824     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
1825
1826     # we use uhci for old VMs because tablet driver was buggy in older qemu
1827     my $usbbus;
1828     if (PVE::QemuServer::Machine::machine_type_is_q35($conf) || $arch eq 'aarch64') {
1829         $usbbus = 'ehci';
1830     } else {
1831         $usbbus = 'uhci';
1832     }
1833
1834     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1835 }
1836
1837 sub print_keyboarddevice_full {
1838     my ($conf, $arch, $machine) = @_;
1839
1840     return undef if $arch ne 'aarch64';
1841
1842     return "usb-kbd,id=keyboard,bus=ehci.0,port=2";
1843 }
1844
1845 sub print_drivedevice_full {
1846     my ($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type) = @_;
1847
1848     my $device = '';
1849     my $maxdev = 0;
1850
1851     if ($drive->{interface} eq 'virtio') {
1852         my $pciaddr = print_pci_addr("$drive->{interface}$drive->{index}", $bridges, $arch, $machine_type);
1853         $device = "virtio-blk-pci,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}$pciaddr";
1854         $device .= ",iothread=iothread-$drive->{interface}$drive->{index}" if $drive->{iothread};
1855     } elsif ($drive->{interface} eq 'scsi') {
1856
1857         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1858         my $unit = $drive->{index} % $maxdev;
1859         my $devicetype = 'hd';
1860         my $path = '';
1861         if (drive_is_cdrom($drive)) {
1862             $devicetype = 'cd';
1863         } else {
1864             if ($drive->{file} =~ m|^/|) {
1865                 $path = $drive->{file};
1866                 if (my $info = path_is_scsi($path)) {
1867                     if ($info->{type} == 0 && $drive->{scsiblock}) {
1868                         $devicetype = 'block';
1869                     } elsif ($info->{type} == 1) { # tape
1870                         $devicetype = 'generic';
1871                     }
1872                 }
1873             } else {
1874                  $path = PVE::Storage::path($storecfg, $drive->{file});
1875             }
1876
1877             # for compatibility only, we prefer scsi-hd (#2408, #2355, #2380)
1878             my $version = PVE::QemuServer::Machine::extract_version($machine_type, kvm_user_version());
1879             if ($path =~ m/^iscsi\:\/\// &&
1880                !min_version($version, 4, 1)) {
1881                 $devicetype = 'generic';
1882             }
1883         }
1884
1885         if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)){
1886             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1887         } else {
1888             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0,lun=$drive->{index},drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1889         }
1890
1891         if ($drive->{ssd} && ($devicetype eq 'block' || $devicetype eq 'hd')) {
1892             $device .= ",rotation_rate=1";
1893         }
1894         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1895
1896     } elsif ($drive->{interface} eq 'ide' || $drive->{interface} eq 'sata') {
1897         my $maxdev = ($drive->{interface} eq 'sata') ? $MAX_SATA_DISKS : 2;
1898         my $controller = int($drive->{index} / $maxdev);
1899         my $unit = $drive->{index} % $maxdev;
1900         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1901
1902         $device = "ide-$devicetype";
1903         if ($drive->{interface} eq 'ide') {
1904             $device .= ",bus=ide.$controller,unit=$unit";
1905         } else {
1906             $device .= ",bus=ahci$controller.$unit";
1907         }
1908         $device .= ",drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1909
1910         if ($devicetype eq 'hd') {
1911             if (my $model = $drive->{model}) {
1912                 $model = URI::Escape::uri_unescape($model);
1913                 $device .= ",model=$model";
1914             }
1915             if ($drive->{ssd}) {
1916                 $device .= ",rotation_rate=1";
1917             }
1918         }
1919         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1920     } elsif ($drive->{interface} eq 'usb') {
1921         die "implement me";
1922         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1923     } else {
1924         die "unsupported interface type";
1925     }
1926
1927     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1928
1929     if (my $serial = $drive->{serial}) {
1930         $serial = URI::Escape::uri_unescape($serial);
1931         $device .= ",serial=$serial";
1932     }
1933
1934
1935     return $device;
1936 }
1937
1938 sub get_initiator_name {
1939     my $initiator;
1940
1941     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return undef;
1942     while (defined(my $line = <$fh>)) {
1943         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1944         $initiator = $1;
1945         last;
1946     }
1947     $fh->close();
1948
1949     return $initiator;
1950 }
1951
1952 sub print_drive_full {
1953     my ($storecfg, $vmid, $drive) = @_;
1954
1955     my $path;
1956     my $volid = $drive->{file};
1957     my $format;
1958
1959     if (drive_is_cdrom($drive)) {
1960         $path = get_iso_path($storecfg, $vmid, $volid);
1961     } else {
1962         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1963         if ($storeid) {
1964             $path = PVE::Storage::path($storecfg, $volid);
1965             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
1966             $format = qemu_img_format($scfg, $volname);
1967         } else {
1968             $path = $volid;
1969             $format = "raw";
1970         }
1971    }
1972
1973     my $opts = '';
1974     my @qemu_drive_options = qw(heads secs cyls trans media format cache rerror werror aio discard);
1975     foreach my $o (@qemu_drive_options) {
1976         $opts .= ",$o=$drive->{$o}" if defined($drive->{$o});
1977     }
1978
1979     # snapshot only accepts on|off
1980     if (defined($drive->{snapshot})) {
1981         my $v = $drive->{snapshot} ? 'on' : 'off';
1982         $opts .= ",snapshot=$v";
1983     }
1984
1985     foreach my $type (['', '-total'], [_rd => '-read'], [_wr => '-write']) {
1986         my ($dir, $qmpname) = @$type;
1987         if (my $v = $drive->{"mbps$dir"}) {
1988             $opts .= ",throttling.bps$qmpname=".int($v*1024*1024);
1989         }
1990         if (my $v = $drive->{"mbps${dir}_max"}) {
1991             $opts .= ",throttling.bps$qmpname-max=".int($v*1024*1024);
1992         }
1993         if (my $v = $drive->{"bps${dir}_max_length"}) {
1994             $opts .= ",throttling.bps$qmpname-max-length=$v";
1995         }
1996         if (my $v = $drive->{"iops${dir}"}) {
1997             $opts .= ",throttling.iops$qmpname=$v";
1998         }
1999         if (my $v = $drive->{"iops${dir}_max"}) {
2000             $opts .= ",throttling.iops$qmpname-max=$v";
2001         }
2002         if (my $v = $drive->{"iops${dir}_max_length"}) {
2003             $opts .= ",throttling.iops$qmpname-max-length=$v";
2004         }
2005     }
2006
2007     $opts .= ",format=$format" if $format && !$drive->{format};
2008
2009     my $cache_direct = 0;
2010
2011     if (my $cache = $drive->{cache}) {
2012         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
2013     } elsif (!drive_is_cdrom($drive)) {
2014         $opts .= ",cache=none";
2015         $cache_direct = 1;
2016     }
2017
2018     # aio native works only with O_DIRECT
2019     if (!$drive->{aio}) {
2020         if($cache_direct) {
2021             $opts .= ",aio=native";
2022         } else {
2023             $opts .= ",aio=threads";
2024         }
2025     }
2026
2027     if (!drive_is_cdrom($drive)) {
2028         my $detectzeroes;
2029         if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
2030             $detectzeroes = 'off';
2031         } elsif ($drive->{discard}) {
2032             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
2033         } else {
2034             # This used to be our default with discard not being specified:
2035             $detectzeroes = 'on';
2036         }
2037         $opts .= ",detect-zeroes=$detectzeroes" if $detectzeroes;
2038     }
2039
2040     my $pathinfo = $path ? "file=$path," : '';
2041
2042     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
2043 }
2044
2045 sub print_netdevice_full {
2046     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files, $arch, $machine_type) = @_;
2047
2048     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
2049
2050     my $device = $net->{model};
2051     if ($net->{model} eq 'virtio') {
2052          $device = 'virtio-net-pci';
2053      };
2054
2055     my $pciaddr = print_pci_addr("$netid", $bridges, $arch, $machine_type);
2056     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
2057     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
2058         #Consider we have N queues, the number of vectors needed is 2*N + 2 (plus one config interrupt and control vq)
2059         my $vectors = $net->{queues} * 2 + 2;
2060         $tmpstr .= ",vectors=$vectors,mq=on";
2061     }
2062     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
2063
2064     if ($use_old_bios_files) {
2065         my $romfile;
2066         if ($device eq 'virtio-net-pci') {
2067             $romfile = 'pxe-virtio.rom';
2068         } elsif ($device eq 'e1000') {
2069             $romfile = 'pxe-e1000.rom';
2070         } elsif ($device eq 'ne2k') {
2071             $romfile = 'pxe-ne2k_pci.rom';
2072         } elsif ($device eq 'pcnet') {
2073             $romfile = 'pxe-pcnet.rom';
2074         } elsif ($device eq 'rtl8139') {
2075             $romfile = 'pxe-rtl8139.rom';
2076         }
2077         $tmpstr .= ",romfile=$romfile" if $romfile;
2078     }
2079
2080     return $tmpstr;
2081 }
2082
2083 sub print_netdev_full {
2084     my ($vmid, $conf, $arch, $net, $netid, $hotplug) = @_;
2085
2086     my $i = '';
2087     if ($netid =~ m/^net(\d+)$/) {
2088         $i = int($1);
2089     }
2090
2091     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
2092
2093     my $ifname = "tap${vmid}i$i";
2094
2095     # kvm uses TUNSETIFF ioctl, and that limits ifname length
2096     die "interface name '$ifname' is too long (max 15 character)\n"
2097         if length($ifname) >= 16;
2098
2099     my $vhostparam = '';
2100     if (is_native($arch)) {
2101         $vhostparam = ',vhost=on' if kernel_has_vhost_net() && $net->{model} eq 'virtio';
2102     }
2103
2104     my $vmname = $conf->{name} || "vm$vmid";
2105
2106     my $netdev = "";
2107     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
2108
2109     if ($net->{bridge}) {
2110         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script,downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
2111     } else {
2112         $netdev = "type=user,id=$netid,hostname=$vmname";
2113     }
2114
2115     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
2116
2117     return $netdev;
2118 }
2119
2120
2121 sub print_cpu_device {
2122     my ($conf, $id) = @_;
2123
2124     my $kvm = $conf->{kvm} // 1;
2125     my $cpu = $kvm ? "kvm64" : "qemu64";
2126     if (my $cputype = $conf->{cpu}) {
2127         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
2128             or die "Cannot parse cpu description: $cputype\n";
2129         $cpu = $cpuconf->{cputype};
2130     }
2131
2132     my $cores = $conf->{cores} || 1;
2133
2134     my $current_core = ($id - 1) % $cores;
2135     my $current_socket = int(($id - 1 - $current_core)/$cores);
2136
2137     return "$cpu-x86_64-cpu,id=cpu$id,socket-id=$current_socket,core-id=$current_core,thread-id=0";
2138 }
2139
2140 my $vga_map = {
2141     'cirrus' => 'cirrus-vga',
2142     'std' => 'VGA',
2143     'vmware' => 'vmware-svga',
2144     'virtio' => 'virtio-vga',
2145 };
2146
2147 sub print_vga_device {
2148     my ($conf, $vga, $arch, $machine_version, $machine, $id, $qxlnum, $bridges) = @_;
2149
2150     my $type = $vga_map->{$vga->{type}};
2151     if ($arch eq 'aarch64' && defined($type) && $type eq 'virtio-vga') {
2152         $type = 'virtio-gpu';
2153     }
2154     my $vgamem_mb = $vga->{memory};
2155
2156     my $max_outputs = '';
2157     if ($qxlnum) {
2158         $type = $id ? 'qxl' : 'qxl-vga';
2159
2160         if (!$conf->{ostype} || $conf->{ostype} =~ m/^(?:l\d\d)|(?:other)$/) {
2161             # set max outputs so linux can have up to 4 qxl displays with one device
2162             if (min_version($machine_version, 4, 1)) {
2163                 $max_outputs = ",max_outputs=4";
2164             }
2165         }
2166     }
2167
2168     die "no devicetype for $vga->{type}\n" if !$type;
2169
2170     my $memory = "";
2171     if ($vgamem_mb) {
2172         if ($vga->{type} eq 'virtio') {
2173             my $bytes = PVE::Tools::convert_size($vgamem_mb, "mb" => "b");
2174             $memory = ",max_hostmem=$bytes";
2175         } elsif ($qxlnum) {
2176             # from https://www.spice-space.org/multiple-monitors.html
2177             $memory = ",vgamem_mb=$vga->{memory}";
2178             my $ram = $vgamem_mb * 4;
2179             my $vram = $vgamem_mb * 2;
2180             $memory .= ",ram_size_mb=$ram,vram_size_mb=$vram";
2181         } else {
2182             $memory = ",vgamem_mb=$vga->{memory}";
2183         }
2184     } elsif ($qxlnum && $id) {
2185         $memory = ",ram_size=67108864,vram_size=33554432";
2186     }
2187
2188     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
2189     my $vgaid = "vga" . ($id // '');
2190     my $pciaddr;
2191
2192     if ($q35 && $vgaid eq 'vga') {
2193         # the first display uses pcie.0 bus on q35 machines
2194         $pciaddr = print_pcie_addr($vgaid, $bridges, $arch, $machine);
2195     } else {
2196         $pciaddr = print_pci_addr($vgaid, $bridges, $arch, $machine);
2197     }
2198
2199     return "$type,id=${vgaid}${memory}${max_outputs}${pciaddr}";
2200 }
2201
2202 sub drive_is_cloudinit {
2203     my ($drive) = @_;
2204     return $drive->{file} =~ m@[:/]vm-\d+-cloudinit(?:\.$QEMU_FORMAT_RE)?$@;
2205 }
2206
2207 sub drive_is_cdrom {
2208     my ($drive, $exclude_cloudinit) = @_;
2209
2210     return 0 if $exclude_cloudinit && drive_is_cloudinit($drive);
2211
2212     return $drive && $drive->{media} && ($drive->{media} eq 'cdrom');
2213
2214 }
2215
2216 sub parse_number_sets {
2217     my ($set) = @_;
2218     my $res = [];
2219     foreach my $part (split(/;/, $set)) {
2220         if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
2221             die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
2222             push @$res, [ $1, $2 ];
2223         } else {
2224             die "invalid range: $part\n";
2225         }
2226     }
2227     return $res;
2228 }
2229
2230 sub parse_numa {
2231     my ($data) = @_;
2232
2233     my $res = PVE::JSONSchema::parse_property_string($numa_fmt, $data);
2234     $res->{cpus} = parse_number_sets($res->{cpus}) if defined($res->{cpus});
2235     $res->{hostnodes} = parse_number_sets($res->{hostnodes}) if defined($res->{hostnodes});
2236     return $res;
2237 }
2238
2239 sub parse_hostpci {
2240     my ($value) = @_;
2241
2242     return undef if !$value;
2243
2244     my $res = PVE::JSONSchema::parse_property_string($hostpci_fmt, $value);
2245
2246     my @idlist = split(/;/, $res->{host});
2247     delete $res->{host};
2248     foreach my $id (@idlist) {
2249         my $devs = PVE::SysFSTools::lspci($id);
2250         die "no PCI device found for '$id'\n" if !scalar(@$devs);
2251         push @{$res->{pciid}}, @$devs;
2252     }
2253     return $res;
2254 }
2255
2256 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
2257 sub parse_net {
2258     my ($data) = @_;
2259
2260     my $res = eval { PVE::JSONSchema::parse_property_string($net_fmt, $data) };
2261     if ($@) {
2262         warn $@;
2263         return undef;
2264     }
2265     if (!defined($res->{macaddr})) {
2266         my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
2267         $res->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix});
2268     }
2269     return $res;
2270 }
2271
2272 # ipconfigX ip=cidr,gw=ip,ip6=cidr,gw6=ip
2273 sub parse_ipconfig {
2274     my ($data) = @_;
2275
2276     my $res = eval { PVE::JSONSchema::parse_property_string($ipconfig_fmt, $data) };
2277     if ($@) {
2278         warn $@;
2279         return undef;
2280     }
2281
2282     if ($res->{gw} && !$res->{ip}) {
2283         warn 'gateway specified without specifying an IP address';
2284         return undef;
2285     }
2286     if ($res->{gw6} && !$res->{ip6}) {
2287         warn 'IPv6 gateway specified without specifying an IPv6 address';
2288         return undef;
2289     }
2290     if ($res->{gw} && $res->{ip} eq 'dhcp') {
2291         warn 'gateway specified together with DHCP';
2292         return undef;
2293     }
2294     if ($res->{gw6} && $res->{ip6} !~ /^$IPV6RE/) {
2295         # gw6 + auto/dhcp
2296         warn "IPv6 gateway specified together with $res->{ip6} address";
2297         return undef;
2298     }
2299
2300     if (!$res->{ip} && !$res->{ip6}) {
2301         return { ip => 'dhcp', ip6 => 'dhcp' };
2302     }
2303
2304     return $res;
2305 }
2306
2307 sub print_net {
2308     my $net = shift;
2309
2310     return PVE::JSONSchema::print_property_string($net, $net_fmt);
2311 }
2312
2313 sub add_random_macs {
2314     my ($settings) = @_;
2315
2316     foreach my $opt (keys %$settings) {
2317         next if $opt !~ m/^net(\d+)$/;
2318         my $net = parse_net($settings->{$opt});
2319         next if !$net;
2320         $settings->{$opt} = print_net($net);
2321     }
2322 }
2323
2324 sub vm_is_volid_owner {
2325     my ($storecfg, $vmid, $volid) = @_;
2326
2327     if ($volid !~  m|^/|) {
2328         my ($path, $owner);
2329         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
2330         if ($owner && ($owner == $vmid)) {
2331             return 1;
2332         }
2333     }
2334
2335     return undef;
2336 }
2337
2338 sub vmconfig_register_unused_drive {
2339     my ($storecfg, $vmid, $conf, $drive) = @_;
2340
2341     if (drive_is_cloudinit($drive)) {
2342         eval { PVE::Storage::vdisk_free($storecfg, $drive->{file}) };
2343         warn $@ if $@;
2344     } elsif (!drive_is_cdrom($drive)) {
2345         my $volid = $drive->{file};
2346         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
2347             PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
2348         }
2349     }
2350 }
2351
2352 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str][,base64=bool]
2353 my $smbios1_fmt = {
2354     uuid => {
2355         type => 'string',
2356         pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
2357         format_description => 'UUID',
2358         description => "Set SMBIOS1 UUID.",
2359         optional => 1,
2360     },
2361     version => {
2362         type => 'string',
2363         pattern => '[A-Za-z0-9+\/]+={0,2}',
2364         format_description => 'Base64 encoded string',
2365         description => "Set SMBIOS1 version.",
2366         optional => 1,
2367     },
2368     serial => {
2369         type => 'string',
2370         pattern => '[A-Za-z0-9+\/]+={0,2}',
2371         format_description => 'Base64 encoded string',
2372         description => "Set SMBIOS1 serial number.",
2373         optional => 1,
2374     },
2375     manufacturer => {
2376         type => 'string',
2377         pattern => '[A-Za-z0-9+\/]+={0,2}',
2378         format_description => 'Base64 encoded string',
2379         description => "Set SMBIOS1 manufacturer.",
2380         optional => 1,
2381     },
2382     product => {
2383         type => 'string',
2384         pattern => '[A-Za-z0-9+\/]+={0,2}',
2385         format_description => 'Base64 encoded string',
2386         description => "Set SMBIOS1 product ID.",
2387         optional => 1,
2388     },
2389     sku => {
2390         type => 'string',
2391         pattern => '[A-Za-z0-9+\/]+={0,2}',
2392         format_description => 'Base64 encoded string',
2393         description => "Set SMBIOS1 SKU string.",
2394         optional => 1,
2395     },
2396     family => {
2397         type => 'string',
2398         pattern => '[A-Za-z0-9+\/]+={0,2}',
2399         format_description => 'Base64 encoded string',
2400         description => "Set SMBIOS1 family string.",
2401         optional => 1,
2402     },
2403     base64 => {
2404         type => 'boolean',
2405         description => 'Flag to indicate that the SMBIOS values are base64 encoded',
2406         optional => 1,
2407     },
2408 };
2409
2410 sub parse_smbios1 {
2411     my ($data) = @_;
2412
2413     my $res = eval { PVE::JSONSchema::parse_property_string($smbios1_fmt, $data) };
2414     warn $@ if $@;
2415     return $res;
2416 }
2417
2418 sub print_smbios1 {
2419     my ($smbios1) = @_;
2420     return PVE::JSONSchema::print_property_string($smbios1, $smbios1_fmt);
2421 }
2422
2423 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_fmt);
2424
2425 PVE::JSONSchema::register_format('pve-qm-bootdisk', \&verify_bootdisk);
2426 sub verify_bootdisk {
2427     my ($value, $noerr) = @_;
2428
2429     return $value if is_valid_drivename($value);
2430
2431     return undef if $noerr;
2432
2433     die "invalid boot disk '$value'\n";
2434 }
2435
2436 sub parse_watchdog {
2437     my ($value) = @_;
2438
2439     return undef if !$value;
2440
2441     my $res = eval { PVE::JSONSchema::parse_property_string($watchdog_fmt, $value) };
2442     warn $@ if $@;
2443     return $res;
2444 }
2445
2446 sub parse_guest_agent {
2447     my ($value) = @_;
2448
2449     return {} if !defined($value->{agent});
2450
2451     my $res = eval { PVE::JSONSchema::parse_property_string($agent_fmt, $value->{agent}) };
2452     warn $@ if $@;
2453
2454     # if the agent is disabled ignore the other potentially set properties
2455     return {} if !$res->{enabled};
2456     return $res;
2457 }
2458
2459 sub parse_vga {
2460     my ($value) = @_;
2461
2462     return {} if !$value;
2463     my $res = eval { PVE::JSONSchema::parse_property_string($vga_fmt, $value) };
2464     warn $@ if $@;
2465     return $res;
2466 }
2467
2468 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
2469 sub verify_usb_device {
2470     my ($value, $noerr) = @_;
2471
2472     return $value if parse_usb_device($value);
2473
2474     return undef if $noerr;
2475
2476     die "unable to parse usb device\n";
2477 }
2478
2479 # add JSON properties for create and set function
2480 sub json_config_properties {
2481     my $prop = shift;
2482
2483     foreach my $opt (keys %$confdesc) {
2484         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate' || $opt eq 'runningmachine';
2485         $prop->{$opt} = $confdesc->{$opt};
2486     }
2487
2488     return $prop;
2489 }
2490
2491 # return copy of $confdesc_cloudinit to generate documentation
2492 sub cloudinit_config_properties {
2493
2494     return dclone($confdesc_cloudinit);
2495 }
2496
2497 sub check_type {
2498     my ($key, $value) = @_;
2499
2500     die "unknown setting '$key'\n" if !$confdesc->{$key};
2501
2502     my $type = $confdesc->{$key}->{type};
2503
2504     if (!defined($value)) {
2505         die "got undefined value\n";
2506     }
2507
2508     if ($value =~ m/[\n\r]/) {
2509         die "property contains a line feed\n";
2510     }
2511
2512     if ($type eq 'boolean') {
2513         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
2514         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
2515         die "type check ('boolean') failed - got '$value'\n";
2516     } elsif ($type eq 'integer') {
2517         return int($1) if $value =~ m/^(\d+)$/;
2518         die "type check ('integer') failed - got '$value'\n";
2519     } elsif ($type eq 'number') {
2520         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
2521         die "type check ('number') failed - got '$value'\n";
2522     } elsif ($type eq 'string') {
2523         if (my $fmt = $confdesc->{$key}->{format}) {
2524             PVE::JSONSchema::check_format($fmt, $value);
2525             return $value;
2526         }
2527         $value =~ s/^\"(.*)\"$/$1/;
2528         return $value;
2529     } else {
2530         die "internal error"
2531     }
2532 }
2533
2534 sub destroy_vm {
2535     my ($storecfg, $vmid, $skiplock, $replacement_conf) = @_;
2536
2537     my $conf = PVE::QemuConfig->load_config($vmid);
2538
2539     PVE::QemuConfig->check_lock($conf) if !$skiplock;
2540
2541     if ($conf->{template}) {
2542         # check if any base image is still used by a linked clone
2543         foreach_drive($conf, sub {
2544                 my ($ds, $drive) = @_;
2545                 return if drive_is_cdrom($drive);
2546
2547                 my $volid = $drive->{file};
2548                 return if !$volid || $volid =~ m|^/|;
2549
2550                 die "base volume '$volid' is still in use by linked cloned\n"
2551                     if PVE::Storage::volume_is_base_and_used($storecfg, $volid);
2552
2553         });
2554     }
2555
2556     # only remove disks owned by this VM
2557     foreach_drive($conf, sub {
2558         my ($ds, $drive) = @_;
2559         return if drive_is_cdrom($drive, 1);
2560
2561         my $volid = $drive->{file};
2562         return if !$volid || $volid =~ m|^/|;
2563
2564         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
2565         return if !$path || !$owner || ($owner != $vmid);
2566
2567         eval { PVE::Storage::vdisk_free($storecfg, $volid) };
2568         warn "Could not remove disk '$volid', check manually: $@" if $@;
2569     });
2570
2571     # also remove unused disk
2572     my $vmdisks = PVE::Storage::vdisk_list($storecfg, undef, $vmid);
2573     PVE::Storage::foreach_volid($vmdisks, sub {
2574         my ($volid, $sid, $volname, $d) = @_;
2575         eval { PVE::Storage::vdisk_free($storecfg, $volid) };
2576         warn $@ if $@;
2577     });
2578
2579     if (defined $replacement_conf) {
2580         PVE::QemuConfig->write_config($vmid, $replacement_conf);
2581     } else {
2582         PVE::QemuConfig->destroy_config($vmid);
2583     }
2584 }
2585
2586 sub parse_vm_config {
2587     my ($filename, $raw) = @_;
2588
2589     return undef if !defined($raw);
2590
2591     my $res = {
2592         digest => Digest::SHA::sha1_hex($raw),
2593         snapshots => {},
2594         pending => {},
2595     };
2596
2597     $filename =~ m|/qemu-server/(\d+)\.conf$|
2598         || die "got strange filename '$filename'";
2599
2600     my $vmid = $1;
2601
2602     my $conf = $res;
2603     my $descr;
2604     my $section = '';
2605
2606     my @lines = split(/\n/, $raw);
2607     foreach my $line (@lines) {
2608         next if $line =~ m/^\s*$/;
2609
2610         if ($line =~ m/^\[PENDING\]\s*$/i) {
2611             $section = 'pending';
2612             if (defined($descr)) {
2613                 $descr =~ s/\s+$//;
2614                 $conf->{description} = $descr;
2615             }
2616             $descr = undef;
2617             $conf = $res->{$section} = {};
2618             next;
2619
2620         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2621             $section = $1;
2622             if (defined($descr)) {
2623                 $descr =~ s/\s+$//;
2624                 $conf->{description} = $descr;
2625             }
2626             $descr = undef;
2627             $conf = $res->{snapshots}->{$section} = {};
2628             next;
2629         }
2630
2631         if ($line =~ m/^\#(.*)\s*$/) {
2632             $descr = '' if !defined($descr);
2633             $descr .= PVE::Tools::decode_text($1) . "\n";
2634             next;
2635         }
2636
2637         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2638             $descr = '' if !defined($descr);
2639             $descr .= PVE::Tools::decode_text($2);
2640         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2641             $conf->{snapstate} = $1;
2642         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2643             my $key = $1;
2644             my $value = $2;
2645             $conf->{$key} = $value;
2646         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2647             my $value = $1;
2648             if ($section eq 'pending') {
2649                 $conf->{delete} = $value; # we parse this later
2650             } else {
2651                 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
2652             }
2653         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(.+?)\s*$/) {
2654             my $key = $1;
2655             my $value = $2;
2656             eval { $value = check_type($key, $value); };
2657             if ($@) {
2658                 warn "vm $vmid - unable to parse value of '$key' - $@";
2659             } else {
2660                 $key = 'ide2' if $key eq 'cdrom';
2661                 my $fmt = $confdesc->{$key}->{format};
2662                 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2663                     my $v = parse_drive($key, $value);
2664                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2665                         $v->{file} = $volid;
2666                         $value = print_drive($v);
2667                     } else {
2668                         warn "vm $vmid - unable to parse value of '$key'\n";
2669                         next;
2670                     }
2671                 }
2672
2673                 $conf->{$key} = $value;
2674             }
2675         }
2676     }
2677
2678     if (defined($descr)) {
2679         $descr =~ s/\s+$//;
2680         $conf->{description} = $descr;
2681     }
2682     delete $res->{snapstate}; # just to be sure
2683
2684     return $res;
2685 }
2686
2687 sub write_vm_config {
2688     my ($filename, $conf) = @_;
2689
2690     delete $conf->{snapstate}; # just to be sure
2691
2692     if ($conf->{cdrom}) {
2693         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2694         $conf->{ide2} = $conf->{cdrom};
2695         delete $conf->{cdrom};
2696     }
2697
2698     # we do not use 'smp' any longer
2699     if ($conf->{sockets}) {
2700         delete $conf->{smp};
2701     } elsif ($conf->{smp}) {
2702         $conf->{sockets} = $conf->{smp};
2703         delete $conf->{cores};
2704         delete $conf->{smp};
2705     }
2706
2707     my $used_volids = {};
2708
2709     my $cleanup_config = sub {
2710         my ($cref, $pending, $snapname) = @_;
2711
2712         foreach my $key (keys %$cref) {
2713             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2714                 $key eq 'snapstate' || $key eq 'pending';
2715             my $value = $cref->{$key};
2716             if ($key eq 'delete') {
2717                 die "propertry 'delete' is only allowed in [PENDING]\n"
2718                     if !$pending;
2719                 # fixme: check syntax?
2720                 next;
2721             }
2722             eval { $value = check_type($key, $value); };
2723             die "unable to parse value of '$key' - $@" if $@;
2724
2725             $cref->{$key} = $value;
2726
2727             if (!$snapname && is_valid_drivename($key)) {
2728                 my $drive = parse_drive($key, $value);
2729                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2730             }
2731         }
2732     };
2733
2734     &$cleanup_config($conf);
2735
2736     &$cleanup_config($conf->{pending}, 1);
2737
2738     foreach my $snapname (keys %{$conf->{snapshots}}) {
2739         die "internal error: snapshot name '$snapname' is forbidden" if lc($snapname) eq 'pending';
2740         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2741     }
2742
2743     # remove 'unusedX' settings if we re-add a volume
2744     foreach my $key (keys %$conf) {
2745         my $value = $conf->{$key};
2746         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2747             delete $conf->{$key};
2748         }
2749     }
2750
2751     my $generate_raw_config = sub {
2752         my ($conf, $pending) = @_;
2753
2754         my $raw = '';
2755
2756         # add description as comment to top of file
2757         if (defined(my $descr = $conf->{description})) {
2758             if ($descr) {
2759                 foreach my $cl (split(/\n/, $descr)) {
2760                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2761                 }
2762             } else {
2763                 $raw .= "#\n" if $pending;
2764             }
2765         }
2766
2767         foreach my $key (sort keys %$conf) {
2768             next if $key eq 'digest' || $key eq 'description' || $key eq 'pending' || $key eq 'snapshots';
2769             $raw .= "$key: $conf->{$key}\n";
2770         }
2771         return $raw;
2772     };
2773
2774     my $raw = &$generate_raw_config($conf);
2775
2776     if (scalar(keys %{$conf->{pending}})){
2777         $raw .= "\n[PENDING]\n";
2778         $raw .= &$generate_raw_config($conf->{pending}, 1);
2779     }
2780
2781     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2782         $raw .= "\n[$snapname]\n";
2783         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2784     }
2785
2786     return $raw;
2787 }
2788
2789 sub load_defaults {
2790
2791     my $res = {};
2792
2793     # we use static defaults from our JSON schema configuration
2794     foreach my $key (keys %$confdesc) {
2795         if (defined(my $default = $confdesc->{$key}->{default})) {
2796             $res->{$key} = $default;
2797         }
2798     }
2799
2800     return $res;
2801 }
2802
2803 sub config_list {
2804     my $vmlist = PVE::Cluster::get_vmlist();
2805     my $res = {};
2806     return $res if !$vmlist || !$vmlist->{ids};
2807     my $ids = $vmlist->{ids};
2808
2809     foreach my $vmid (keys %$ids) {
2810         my $d = $ids->{$vmid};
2811         next if !$d->{node} || $d->{node} ne $nodename;
2812         next if !$d->{type} || $d->{type} ne 'qemu';
2813         $res->{$vmid}->{exists} = 1;
2814     }
2815     return $res;
2816 }
2817
2818 # test if VM uses local resources (to prevent migration)
2819 sub check_local_resources {
2820     my ($conf, $noerr) = @_;
2821
2822     my @loc_res = ();
2823
2824     push @loc_res, "hostusb" if $conf->{hostusb}; # old syntax
2825     push @loc_res, "hostpci" if $conf->{hostpci}; # old syntax
2826
2827     push @loc_res, "ivshmem" if $conf->{ivshmem};
2828
2829     foreach my $k (keys %$conf) {
2830         next if $k =~ m/^usb/ && ($conf->{$k} =~ m/^spice(?![^,])/);
2831         # sockets are safe: they will recreated be on the target side post-migrate
2832         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2833         push @loc_res, $k if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2834     }
2835
2836     die "VM uses local resources\n" if scalar @loc_res && !$noerr;
2837
2838     return \@loc_res;
2839 }
2840
2841 # check if used storages are available on all nodes (use by migrate)
2842 sub check_storage_availability {
2843     my ($storecfg, $conf, $node) = @_;
2844
2845     foreach_drive($conf, sub {
2846         my ($ds, $drive) = @_;
2847
2848         my $volid = $drive->{file};
2849         return if !$volid;
2850
2851         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2852         return if !$sid;
2853
2854         # check if storage is available on both nodes
2855         my $scfg = PVE::Storage::storage_check_node($storecfg, $sid);
2856         PVE::Storage::storage_check_node($storecfg, $sid, $node);
2857    });
2858 }
2859
2860 # list nodes where all VM images are available (used by has_feature API)
2861 sub shared_nodes {
2862     my ($conf, $storecfg) = @_;
2863
2864     my $nodelist = PVE::Cluster::get_nodelist();
2865     my $nodehash = { map { $_ => 1 } @$nodelist };
2866     my $nodename = PVE::INotify::nodename();
2867
2868     foreach_drive($conf, sub {
2869         my ($ds, $drive) = @_;
2870
2871         my $volid = $drive->{file};
2872         return if !$volid;
2873
2874         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2875         if ($storeid) {
2876             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2877             if ($scfg->{disable}) {
2878                 $nodehash = {};
2879             } elsif (my $avail = $scfg->{nodes}) {
2880                 foreach my $node (keys %$nodehash) {
2881                     delete $nodehash->{$node} if !$avail->{$node};
2882                 }
2883             } elsif (!$scfg->{shared}) {
2884                 foreach my $node (keys %$nodehash) {
2885                     delete $nodehash->{$node} if $node ne $nodename
2886                 }
2887             }
2888         }
2889     });
2890
2891     return $nodehash
2892 }
2893
2894 sub check_local_storage_availability {
2895     my ($conf, $storecfg) = @_;
2896
2897     my $nodelist = PVE::Cluster::get_nodelist();
2898     my $nodehash = { map { $_ => {} } @$nodelist };
2899
2900     foreach_drive($conf, sub {
2901         my ($ds, $drive) = @_;
2902
2903         my $volid = $drive->{file};
2904         return if !$volid;
2905
2906         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2907         if ($storeid) {
2908             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2909
2910             if ($scfg->{disable}) {
2911                 foreach my $node (keys %$nodehash) {
2912                     $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2913                 }
2914             } elsif (my $avail = $scfg->{nodes}) {
2915                 foreach my $node (keys %$nodehash) {
2916                     if (!$avail->{$node}) {
2917                         $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2918                     }
2919                 }
2920             }
2921         }
2922     });
2923
2924     foreach my $node (values %$nodehash) {
2925         if (my $unavail = $node->{unavailable_storages}) {
2926             $node->{unavailable_storages} = [ sort keys %$unavail ];
2927         }
2928     }
2929
2930     return $nodehash
2931 }
2932
2933 # Compat only, use assert_config_exists_on_node and vm_running_locally where possible
2934 sub check_running {
2935     my ($vmid, $nocheck, $node) = @_;
2936
2937     PVE::QemuConfig::assert_config_exists_on_node($vmid, $node) if !$nocheck;
2938     return PVE::QemuServer::Helpers::vm_running_locally($vmid);
2939 }
2940
2941 sub vzlist {
2942
2943     my $vzlist = config_list();
2944
2945     my $fd = IO::Dir->new($PVE::QemuServer::Helpers::var_run_tmpdir) || return $vzlist;
2946
2947     while (defined(my $de = $fd->read)) {
2948         next if $de !~ m/^(\d+)\.pid$/;
2949         my $vmid = $1;
2950         next if !defined($vzlist->{$vmid});
2951         if (my $pid = check_running($vmid)) {
2952             $vzlist->{$vmid}->{pid} = $pid;
2953         }
2954     }
2955
2956     return $vzlist;
2957 }
2958
2959 sub disksize {
2960     my ($storecfg, $conf) = @_;
2961
2962     my $bootdisk = $conf->{bootdisk};
2963     return undef if !$bootdisk;
2964     return undef if !is_valid_drivename($bootdisk);
2965
2966     return undef if !$conf->{$bootdisk};
2967
2968     my $drive = parse_drive($bootdisk, $conf->{$bootdisk});
2969     return undef if !defined($drive);
2970
2971     return undef if drive_is_cdrom($drive);
2972
2973     my $volid = $drive->{file};
2974     return undef if !$volid;
2975
2976     return $drive->{size};
2977 }
2978
2979 our $vmstatus_return_properties = {
2980     vmid => get_standard_option('pve-vmid'),
2981     status => {
2982         description => "Qemu process status.",
2983         type => 'string',
2984         enum => ['stopped', 'running'],
2985     },
2986     maxmem => {
2987         description => "Maximum memory in bytes.",
2988         type => 'integer',
2989         optional => 1,
2990         renderer => 'bytes',
2991     },
2992     maxdisk => {
2993         description => "Root disk size in bytes.",
2994         type => 'integer',
2995         optional => 1,
2996         renderer => 'bytes',
2997     },
2998     name => {
2999         description => "VM name.",
3000         type => 'string',
3001         optional => 1,
3002     },
3003     qmpstatus => {
3004         description => "Qemu QMP agent status.",
3005         type => 'string',
3006         optional => 1,
3007     },
3008     pid => {
3009         description => "PID of running qemu process.",
3010         type => 'integer',
3011         optional => 1,
3012     },
3013     uptime => {
3014         description => "Uptime.",
3015         type => 'integer',
3016         optional => 1,
3017         renderer => 'duration',
3018     },
3019     cpus => {
3020         description => "Maximum usable CPUs.",
3021         type => 'number',
3022         optional => 1,
3023     },
3024     lock => {
3025         description => "The current config lock, if any.",
3026         type => 'string',
3027         optional => 1,
3028     },
3029     tags => {
3030         description  => "The current configured tags, if any",
3031         type => 'string',
3032         optional => 1,
3033     },
3034 };
3035
3036 my $last_proc_pid_stat;
3037
3038 # get VM status information
3039 # This must be fast and should not block ($full == false)
3040 # We only query KVM using QMP if $full == true (this can be slow)
3041 sub vmstatus {
3042     my ($opt_vmid, $full) = @_;
3043
3044     my $res = {};
3045
3046     my $storecfg = PVE::Storage::config();
3047
3048     my $list = vzlist();
3049     my $defaults = load_defaults();
3050
3051     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
3052
3053     my $cpucount = $cpuinfo->{cpus} || 1;
3054
3055     foreach my $vmid (keys %$list) {
3056         next if $opt_vmid && ($vmid ne $opt_vmid);
3057
3058         my $conf = PVE::QemuConfig->load_config($vmid);
3059
3060         my $d = { vmid => $vmid };
3061         $d->{pid} = $list->{$vmid}->{pid};
3062
3063         # fixme: better status?
3064         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
3065
3066         my $size = disksize($storecfg, $conf);
3067         if (defined($size)) {
3068             $d->{disk} = 0; # no info available
3069             $d->{maxdisk} = $size;
3070         } else {
3071             $d->{disk} = 0;
3072             $d->{maxdisk} = 0;
3073         }
3074
3075         $d->{cpus} = ($conf->{sockets} || $defaults->{sockets})
3076             * ($conf->{cores} || $defaults->{cores});
3077         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
3078         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
3079
3080         $d->{name} = $conf->{name} || "VM $vmid";
3081         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024)
3082             : $defaults->{memory}*(1024*1024);
3083
3084         if ($conf->{balloon}) {
3085             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
3086             $d->{shares} = defined($conf->{shares}) ? $conf->{shares}
3087                 : $defaults->{shares};
3088         }
3089
3090         $d->{uptime} = 0;
3091         $d->{cpu} = 0;
3092         $d->{mem} = 0;
3093
3094         $d->{netout} = 0;
3095         $d->{netin} = 0;
3096
3097         $d->{diskread} = 0;
3098         $d->{diskwrite} = 0;
3099
3100         $d->{template} = PVE::QemuConfig->is_template($conf);
3101
3102         $d->{serial} = 1 if conf_has_serial($conf);
3103         $d->{lock} = $conf->{lock} if $conf->{lock};
3104         $d->{tags} = $conf->{tags} if defined($conf->{tags});
3105
3106         $res->{$vmid} = $d;
3107     }
3108
3109     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
3110     foreach my $dev (keys %$netdev) {
3111         next if $dev !~ m/^tap([1-9]\d*)i/;
3112         my $vmid = $1;
3113         my $d = $res->{$vmid};
3114         next if !$d;
3115
3116         $d->{netout} += $netdev->{$dev}->{receive};
3117         $d->{netin} += $netdev->{$dev}->{transmit};
3118
3119         if ($full) {
3120             $d->{nics}->{$dev}->{netout} = $netdev->{$dev}->{receive};
3121             $d->{nics}->{$dev}->{netin} = $netdev->{$dev}->{transmit};
3122         }
3123
3124     }
3125
3126     my $ctime = gettimeofday;
3127
3128     foreach my $vmid (keys %$list) {
3129
3130         my $d = $res->{$vmid};
3131         my $pid = $d->{pid};
3132         next if !$pid;
3133
3134         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
3135         next if !$pstat; # not running
3136
3137         my $used = $pstat->{utime} + $pstat->{stime};
3138
3139         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
3140
3141         if ($pstat->{vsize}) {
3142             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
3143         }
3144
3145         my $old = $last_proc_pid_stat->{$pid};
3146         if (!$old) {
3147             $last_proc_pid_stat->{$pid} = {
3148                 time => $ctime,
3149                 used => $used,
3150                 cpu => 0,
3151             };
3152             next;
3153         }
3154
3155         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
3156
3157         if ($dtime > 1000) {
3158             my $dutime = $used -  $old->{used};
3159
3160             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
3161             $last_proc_pid_stat->{$pid} = {
3162                 time => $ctime,
3163                 used => $used,
3164                 cpu => $d->{cpu},
3165             };
3166         } else {
3167             $d->{cpu} = $old->{cpu};
3168         }
3169     }
3170
3171     return $res if !$full;
3172
3173     my $qmpclient = PVE::QMPClient->new();
3174
3175     my $ballooncb = sub {
3176         my ($vmid, $resp) = @_;
3177
3178         my $info = $resp->{'return'};
3179         return if !$info->{max_mem};
3180
3181         my $d = $res->{$vmid};
3182
3183         # use memory assigned to VM
3184         $d->{maxmem} = $info->{max_mem};
3185         $d->{balloon} = $info->{actual};
3186
3187         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
3188             $d->{mem} = $info->{total_mem} - $info->{free_mem};
3189             $d->{freemem} = $info->{free_mem};
3190         }
3191
3192         $d->{ballooninfo} = $info;
3193     };
3194
3195     my $blockstatscb = sub {
3196         my ($vmid, $resp) = @_;
3197         my $data = $resp->{'return'} || [];
3198         my $totalrdbytes = 0;
3199         my $totalwrbytes = 0;
3200
3201         for my $blockstat (@$data) {
3202             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
3203             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
3204
3205             $blockstat->{device} =~ s/drive-//;
3206             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
3207         }
3208         $res->{$vmid}->{diskread} = $totalrdbytes;
3209         $res->{$vmid}->{diskwrite} = $totalwrbytes;
3210     };
3211
3212     my $statuscb = sub {
3213         my ($vmid, $resp) = @_;
3214
3215         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
3216         # this fails if ballon driver is not loaded, so this must be
3217         # the last commnand (following command are aborted if this fails).
3218         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
3219
3220         my $status = 'unknown';
3221         if (!defined($status = $resp->{'return'}->{status})) {
3222             warn "unable to get VM status\n";
3223             return;
3224         }
3225
3226         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
3227     };
3228
3229     foreach my $vmid (keys %$list) {
3230         next if $opt_vmid && ($vmid ne $opt_vmid);
3231         next if !$res->{$vmid}->{pid}; # not running
3232         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
3233     }
3234
3235     $qmpclient->queue_execute(undef, 2);
3236
3237     foreach my $vmid (keys %$list) {
3238         next if $opt_vmid && ($vmid ne $opt_vmid);
3239         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
3240     }
3241
3242     return $res;
3243 }
3244
3245 sub foreach_drive {
3246     my ($conf, $func, @param) = @_;
3247
3248     foreach my $ds (valid_drive_names()) {
3249         next if !defined($conf->{$ds});
3250
3251         my $drive = parse_drive($ds, $conf->{$ds});
3252         next if !$drive;
3253
3254         &$func($ds, $drive, @param);
3255     }
3256 }
3257
3258 sub foreach_volid {
3259     my ($conf, $func, @param) = @_;
3260
3261     my $volhash = {};
3262
3263     my $test_volid = sub {
3264         my ($volid, $is_cdrom, $replicate, $shared, $snapname, $size) = @_;
3265
3266         return if !$volid;
3267
3268         $volhash->{$volid}->{cdrom} //= 1;
3269         $volhash->{$volid}->{cdrom} = 0 if !$is_cdrom;
3270
3271         $volhash->{$volid}->{replicate} //= 0;
3272         $volhash->{$volid}->{replicate} = 1 if $replicate;
3273
3274         $volhash->{$volid}->{shared} //= 0;
3275         $volhash->{$volid}->{shared} = 1 if $shared;
3276
3277         $volhash->{$volid}->{referenced_in_config} //= 0;
3278         $volhash->{$volid}->{referenced_in_config} = 1 if !defined($snapname);
3279
3280         $volhash->{$volid}->{referenced_in_snapshot}->{$snapname} = 1
3281             if defined($snapname);
3282         $volhash->{$volid}->{size} = $size if $size;
3283     };
3284
3285     foreach_drive($conf, sub {
3286         my ($ds, $drive) = @_;
3287         $test_volid->($drive->{file}, drive_is_cdrom($drive), $drive->{replicate} // 1, $drive->{shared}, undef, $drive->{size});
3288     });
3289
3290     foreach my $snapname (keys %{$conf->{snapshots}}) {
3291         my $snap = $conf->{snapshots}->{$snapname};
3292         $test_volid->($snap->{vmstate}, 0, 1, $snapname);
3293         foreach_drive($snap, sub {
3294             my ($ds, $drive) = @_;
3295             $test_volid->($drive->{file}, drive_is_cdrom($drive), $drive->{replicate} // 1, $drive->{shared}, $snapname);
3296         });
3297     }
3298
3299     foreach my $volid (keys %$volhash) {
3300         &$func($volid, $volhash->{$volid}, @param);
3301     }
3302 }
3303
3304 sub conf_has_serial {
3305     my ($conf) = @_;
3306
3307     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3308         if ($conf->{"serial$i"}) {
3309             return 1;
3310         }
3311     }
3312
3313     return 0;
3314 }
3315
3316 sub conf_has_audio {
3317     my ($conf, $id) = @_;
3318
3319     $id //= 0;
3320     my $audio = $conf->{"audio$id"};
3321     return undef if !defined($audio);
3322
3323     my $audioproperties = PVE::JSONSchema::parse_property_string($audio_fmt, $audio);
3324     my $audiodriver = $audioproperties->{driver} // 'spice';
3325
3326     return {
3327         dev => $audioproperties->{device},
3328         dev_id => "audiodev$id",
3329         backend => $audiodriver,
3330         backend_id => "$audiodriver-backend${id}",
3331     };
3332 }
3333
3334 sub vga_conf_has_spice {
3335     my ($vga) = @_;
3336
3337     my $vgaconf = parse_vga($vga);
3338     my $vgatype = $vgaconf->{type};
3339     return 0 if !$vgatype || $vgatype !~ m/^qxl([234])?$/;
3340
3341     return $1 || 1;
3342 }
3343
3344 sub is_native($) {
3345     my ($arch) = @_;
3346     return get_host_arch() eq $arch;
3347 }
3348
3349 sub get_vm_arch {
3350     my ($conf) = @_;
3351     return $conf->{arch} // get_host_arch();
3352 }
3353
3354 my $default_machines = {
3355     x86_64 => 'pc',
3356     aarch64 => 'virt',
3357 };
3358
3359 sub get_vm_machine {
3360     my ($conf, $forcemachine, $arch, $add_pve_version) = @_;
3361
3362     my $machine = $forcemachine || $conf->{machine};
3363
3364     if (!$machine || $machine =~ m/^(?:pc|q35|virt)$/) {
3365         $arch //= 'x86_64';
3366         $machine ||= $default_machines->{$arch};
3367         $machine .= "+pve$PVE::QemuServer::Machine::PVE_MACHINE_VERSION" if $add_pve_version;
3368     }
3369
3370     return $machine;
3371 }
3372
3373 sub get_ovmf_files($) {
3374     my ($arch) = @_;
3375
3376     my $ovmf = $OVMF->{$arch}
3377         or die "no OVMF images known for architecture '$arch'\n";
3378
3379     return @$ovmf;
3380 }
3381
3382 my $Arch2Qemu = {
3383     aarch64 => '/usr/bin/qemu-system-aarch64',
3384     x86_64 => '/usr/bin/qemu-system-x86_64',
3385 };
3386 sub get_command_for_arch($) {
3387     my ($arch) = @_;
3388     return '/usr/bin/kvm' if is_native($arch);
3389
3390     my $cmd = $Arch2Qemu->{$arch}
3391         or die "don't know how to emulate architecture '$arch'\n";
3392     return $cmd;
3393 }
3394
3395 sub get_cpu_options {
3396     my ($conf, $arch, $kvm, $kvm_off, $machine_version, $winversion, $gpu_passthrough) = @_;
3397
3398     my $cpuFlags = [];
3399     my $ostype = $conf->{ostype};
3400
3401     my $cpu = $kvm ? "kvm64" : "qemu64";
3402     if ($arch eq 'aarch64') {
3403         $cpu = 'cortex-a57';
3404     }
3405     my $hv_vendor_id;
3406     if (my $cputype = $conf->{cpu}) {
3407         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
3408             or die "Cannot parse cpu description: $cputype\n";
3409         $cpu = $cpuconf->{cputype};
3410         $kvm_off = 1 if $cpuconf->{hidden};
3411         $hv_vendor_id = $cpuconf->{'hv-vendor-id'};
3412
3413         if (defined(my $flags = $cpuconf->{flags})) {
3414             push @$cpuFlags, split(";", $flags);
3415         }
3416     }
3417
3418     push @$cpuFlags , '+lahf_lm' if $cpu eq 'kvm64' && $arch eq 'x86_64';
3419
3420     push @$cpuFlags , '-x2apic' if $ostype && $ostype eq 'solaris';
3421
3422     push @$cpuFlags, '+sep' if $cpu eq 'kvm64' || $cpu eq 'kvm32';
3423
3424     push @$cpuFlags, '-rdtscp' if $cpu =~ m/^Opteron/;
3425
3426     if (min_version($machine_version, 2, 3) && $arch eq 'x86_64') {
3427
3428         push @$cpuFlags , '+kvm_pv_unhalt' if $kvm;
3429         push @$cpuFlags , '+kvm_pv_eoi' if $kvm;
3430     }
3431
3432     add_hyperv_enlightenments($cpuFlags, $winversion, $machine_version, $conf->{bios}, $gpu_passthrough, $hv_vendor_id) if $kvm;
3433
3434     push @$cpuFlags, 'enforce' if $cpu ne 'host' && $kvm && $arch eq 'x86_64';
3435
3436     push @$cpuFlags, 'kvm=off' if $kvm_off;
3437
3438     if (my $cpu_vendor = $cpu_vendor_list->{$cpu}) {
3439         push @$cpuFlags, "vendor=${cpu_vendor}"
3440             if $cpu_vendor ne 'default';
3441     } elsif ($arch ne 'aarch64') {
3442         die "internal error"; # should not happen
3443     }
3444
3445     $cpu .= "," . join(',', @$cpuFlags) if scalar(@$cpuFlags);
3446
3447     return ('-cpu', $cpu);
3448 }
3449
3450 sub config_to_command {
3451     my ($storecfg, $vmid, $conf, $defaults, $forcemachine) = @_;
3452
3453     my $cmd = [];
3454     my $globalFlags = [];
3455     my $machineFlags = [];
3456     my $rtcFlags = [];
3457     my $devices = [];
3458     my $pciaddr = '';
3459     my $bridges = {};
3460     my $vernum = 0; # unknown
3461     my $ostype = $conf->{ostype};
3462     my $winversion = windows_version($ostype);
3463     my $kvm = $conf->{kvm};
3464
3465     my $arch = get_vm_arch($conf);
3466     my $kvm_binary = get_command_for_arch($arch);
3467     my $kvmver = kvm_user_version($kvm_binary);
3468
3469     my $add_pve_version = min_version($kvmver, 4, 1);
3470
3471     my $machine_type = get_vm_machine($conf, $forcemachine, $arch, $add_pve_version);
3472     my $machine_version = PVE::QemuServer::Machine::extract_version($machine_type, $kvmver);
3473     $kvm //= 1 if is_native($arch);
3474
3475     if ($kvm) {
3476         die "KVM virtualisation configured, but not available. Either disable in VM configuration or enable in BIOS.\n"
3477             if !defined kvm_version();
3478     }
3479
3480     if ($kvmver =~ m/^(\d+)\.(\d+)$/) {
3481         $vernum = $1*1000000+$2*1000;
3482     } elsif ($kvmver =~ m/^(\d+)\.(\d+)\.(\d+)$/) {
3483         $vernum = $1*1000000+$2*1000+$3;
3484     }
3485
3486     die "detected old qemu-kvm binary ($kvmver)\n" if $vernum < 15000;
3487
3488     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
3489     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
3490     my $use_old_bios_files = undef;
3491     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3492
3493     my $cpuunits = defined($conf->{cpuunits}) ?
3494             $conf->{cpuunits} : $defaults->{cpuunits};
3495
3496     push @$cmd, $kvm_binary;
3497
3498     push @$cmd, '-id', $vmid;
3499
3500     my $vmname = $conf->{name} || "vm$vmid";
3501
3502     push @$cmd, '-name', $vmname;
3503
3504     my $use_virtio = 0;
3505
3506     my $qmpsocket = PVE::QemuServer::Helpers::qmp_socket($vmid);
3507     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server,nowait";
3508     push @$cmd, '-mon', "chardev=qmp,mode=control";
3509
3510     if (min_version($machine_version, 2, 12)) {
3511         push @$cmd, '-chardev', "socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5";
3512         push @$cmd, '-mon', "chardev=qmp-event,mode=control";
3513     }
3514
3515     push @$cmd, '-pidfile' , PVE::QemuServer::Helpers::pidfile_name($vmid);
3516
3517     push @$cmd, '-daemonize';
3518
3519     if ($conf->{smbios1}) {
3520         my $smbios_conf = parse_smbios1($conf->{smbios1});
3521         if ($smbios_conf->{base64}) {
3522             # Do not pass base64 flag to qemu
3523             delete $smbios_conf->{base64};
3524             my $smbios_string = "";
3525             foreach my $key (keys %$smbios_conf) {
3526                 my $value;
3527                 if ($key eq "uuid") {
3528                     $value = $smbios_conf->{uuid}
3529                 } else {
3530                     $value = decode_base64($smbios_conf->{$key});
3531                 }
3532                 # qemu accepts any binary data, only commas need escaping by double comma
3533                 $value =~ s/,/,,/g;
3534                 $smbios_string .= "," . $key . "=" . $value if $value;
3535             }
3536             push @$cmd, '-smbios', "type=1" . $smbios_string;
3537         } else {
3538             push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
3539         }
3540     }
3541
3542     if ($conf->{vmgenid}) {
3543         push @$devices, '-device', 'vmgenid,guid='.$conf->{vmgenid};
3544     }
3545
3546     my ($ovmf_code, $ovmf_vars) = get_ovmf_files($arch);
3547     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3548         die "uefi base image not found\n" if ! -f $ovmf_code;
3549
3550         my $path;
3551         my $format;
3552         if (my $efidisk = $conf->{efidisk0}) {
3553             my $d = PVE::JSONSchema::parse_property_string($efidisk_fmt, $efidisk);
3554             my ($storeid, $volname) = PVE::Storage::parse_volume_id($d->{file}, 1);
3555             $format = $d->{format};
3556             if ($storeid) {
3557                 $path = PVE::Storage::path($storecfg, $d->{file});
3558                 if (!defined($format)) {
3559                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
3560                     $format = qemu_img_format($scfg, $volname);
3561                 }
3562             } else {
3563                 $path = $d->{file};
3564                 die "efidisk format must be specified\n"
3565                     if !defined($format);
3566             }
3567         } else {
3568             warn "no efidisk configured! Using temporary efivars disk.\n";
3569             $path = "/tmp/$vmid-ovmf.fd";
3570             PVE::Tools::file_copy($ovmf_vars, $path, -s $ovmf_vars);
3571             $format = 'raw';
3572         }
3573
3574         push @$cmd, '-drive', "if=pflash,unit=0,format=raw,readonly,file=$ovmf_code";
3575         push @$cmd, '-drive', "if=pflash,unit=1,format=$format,id=drive-efidisk0,file=$path";
3576     }
3577
3578     # load q35 config
3579     if ($q35) {
3580         # we use different pcie-port hardware for qemu >= 4.0 for passthrough
3581         if (min_version($machine_version, 4, 0)) {
3582             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35-4.0.cfg';
3583         } else {
3584             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35.cfg';
3585         }
3586     }
3587
3588     # add usb controllers
3589     my @usbcontrollers = PVE::QemuServer::USB::get_usb_controllers($conf, $bridges, $arch, $machine_type, $usbdesc->{format}, $MAX_USB_DEVICES);
3590     push @$devices, @usbcontrollers if @usbcontrollers;
3591     my $vga = parse_vga($conf->{vga});
3592
3593     my $qxlnum = vga_conf_has_spice($conf->{vga});
3594     $vga->{type} = 'qxl' if $qxlnum;
3595
3596     if (!$vga->{type}) {
3597         if ($arch eq 'aarch64') {
3598             $vga->{type} = 'virtio';
3599         } elsif (min_version($machine_version, 2, 9)) {
3600             $vga->{type} = (!$winversion || $winversion >= 6) ? 'std' : 'cirrus';
3601         } else {
3602             $vga->{type} = ($winversion >= 6) ? 'std' : 'cirrus';
3603         }
3604     }
3605
3606     # enable absolute mouse coordinates (needed by vnc)
3607     my $tablet;
3608     if (defined($conf->{tablet})) {
3609         $tablet = $conf->{tablet};
3610     } else {
3611         $tablet = $defaults->{tablet};
3612         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
3613         $tablet = 0 if $vga->{type} =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
3614     }
3615
3616     if ($tablet) {
3617         push @$devices, '-device', print_tabletdevice_full($conf, $arch) if $tablet;
3618         my $kbd = print_keyboarddevice_full($conf, $arch);
3619         push @$devices, '-device', $kbd if defined($kbd);
3620     }
3621
3622     my $kvm_off = 0;
3623     my $gpu_passthrough;
3624
3625     # host pci devices
3626     for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
3627         my $id = "hostpci$i";
3628         my $d = parse_hostpci($conf->{$id});
3629         next if !$d;
3630
3631         if (my $pcie = $d->{pcie}) {
3632             die "q35 machine model is not enabled" if !$q35;
3633             # win7 wants to have the pcie devices directly on the pcie bus
3634             # instead of in the root port
3635             if ($winversion == 7) {
3636                 $pciaddr = print_pcie_addr("${id}bus0");
3637             } else {
3638                 # add more root ports if needed, 4 are present by default
3639                 # by pve-q35 cfgs, rest added here on demand.
3640                 if ($i > 3) {
3641                     push @$devices, '-device', print_pcie_root_port($i);
3642                 }
3643                 $pciaddr = print_pcie_addr($id);
3644             }
3645         } else {
3646             $pciaddr = print_pci_addr($id, $bridges, $arch, $machine_type);
3647         }
3648
3649         my $xvga = '';
3650         if ($d->{'x-vga'}) {
3651             $xvga = ',x-vga=on' if !($conf->{bios} && $conf->{bios} eq 'ovmf');
3652             $kvm_off = 1;
3653             $vga->{type} = 'none' if !defined($conf->{vga});
3654             $gpu_passthrough = 1;
3655         }
3656
3657         my $pcidevices = $d->{pciid};
3658         my $multifunction = 1 if @$pcidevices > 1;
3659
3660         my $sysfspath;
3661         if ($d->{mdev} && scalar(@$pcidevices) == 1) {
3662             my $pci_id = $pcidevices->[0]->{id};
3663             my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $i);
3664             $sysfspath = "/sys/bus/pci/devices/$pci_id/$uuid";
3665         } elsif ($d->{mdev}) {
3666             warn "ignoring mediated device '$id' with multifunction device\n";
3667         }
3668
3669         my $j=0;
3670         foreach my $pcidevice (@$pcidevices) {
3671             my $devicestr = "vfio-pci";
3672
3673             if ($sysfspath) {
3674                 $devicestr .= ",sysfsdev=$sysfspath";
3675             } else {
3676                 $devicestr .= ",host=$pcidevice->{id}";
3677             }
3678
3679             my $mf_addr = $multifunction ? ".$j" : '';
3680             $devicestr .= ",id=${id}${mf_addr}${pciaddr}${mf_addr}";
3681
3682             if ($j == 0) {
3683                 $devicestr .= ',rombar=0' if defined($d->{rombar}) && !$d->{rombar};
3684                 $devicestr .= "$xvga";
3685                 $devicestr .= ",multifunction=on" if $multifunction;
3686                 $devicestr .= ",romfile=/usr/share/kvm/$d->{romfile}" if $d->{romfile};
3687             }
3688
3689             push @$devices, '-device', $devicestr;
3690             $j++;
3691         }
3692     }
3693
3694     # usb devices
3695     my $usb_dev_features = {};
3696     $usb_dev_features->{spice_usb3} = 1 if min_version($machine_version, 4, 0);
3697
3698     my @usbdevices = PVE::QemuServer::USB::get_usb_devices($conf, $usbdesc->{format}, $MAX_USB_DEVICES, $usb_dev_features);
3699     push @$devices, @usbdevices if @usbdevices;
3700     # serial devices
3701     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3702         if (my $path = $conf->{"serial$i"}) {
3703             if ($path eq 'socket') {
3704                 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
3705                 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server,nowait";
3706                 # On aarch64, serial0 is the UART device. Qemu only allows
3707                 # connecting UART devices via the '-serial' command line, as
3708                 # the device has a fixed slot on the hardware...
3709                 if ($arch eq 'aarch64' && $i == 0) {
3710                     push @$devices, '-serial', "chardev:serial$i";
3711                 } else {
3712                     push @$devices, '-device', "isa-serial,chardev=serial$i";
3713                 }
3714             } else {
3715                 die "no such serial device\n" if ! -c $path;
3716                 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
3717                 push @$devices, '-device', "isa-serial,chardev=serial$i";
3718             }
3719         }
3720     }
3721
3722     # parallel devices
3723     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
3724         if (my $path = $conf->{"parallel$i"}) {
3725             die "no such parallel device\n" if ! -c $path;
3726             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
3727             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
3728             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
3729         }
3730     }
3731
3732     if (my $audio = conf_has_audio($conf)) {
3733
3734         my $audiopciaddr = print_pci_addr("audio0", $bridges, $arch, $machine_type);
3735
3736         my $id = $audio->{dev_id};
3737         if ($audio->{dev} eq 'AC97') {
3738             push @$devices, '-device', "AC97,id=${id}${audiopciaddr}";
3739         } elsif ($audio->{dev} =~ /intel\-hda$/) {
3740             push @$devices, '-device', "$audio->{dev},id=${id}${audiopciaddr}";
3741             push @$devices, '-device', "hda-micro,id=${id}-codec0,bus=${id}.0,cad=0";
3742             push @$devices, '-device', "hda-duplex,id=${id}-codec1,bus=${id}.0,cad=1";
3743         } else {
3744             die "unkown audio device '$audio->{dev}', implement me!";
3745         }
3746
3747         push @$devices, '-audiodev', "$audio->{backend},id=$audio->{backend_id}";
3748     }
3749
3750     my $sockets = 1;
3751     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3752     $sockets = $conf->{sockets} if  $conf->{sockets};
3753
3754     my $cores = $conf->{cores} || 1;
3755
3756     my $maxcpus = $sockets * $cores;
3757
3758     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
3759
3760     my $allowed_vcpus = $cpuinfo->{cpus};
3761
3762     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n"
3763         if ($allowed_vcpus < $maxcpus);
3764
3765     if($hotplug_features->{cpu} && min_version($machine_version, 2, 7)) {
3766
3767         push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3768         for (my $i = 2; $i <= $vcpus; $i++)  {
3769             my $cpustr = print_cpu_device($conf,$i);
3770             push @$cmd, '-device', $cpustr;
3771         }
3772
3773     } else {
3774
3775         push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3776     }
3777     push @$cmd, '-nodefaults';
3778
3779     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
3780
3781     my $bootindex_hash = {};
3782     my $i = 1;
3783     foreach my $o (split(//, $bootorder)) {
3784         $bootindex_hash->{$o} = $i*100;
3785         $i++;
3786     }
3787
3788     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
3789
3790     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
3791
3792     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
3793
3794     if ($vga->{type} && $vga->{type} !~ m/^serial\d+$/ && $vga->{type} ne 'none'){
3795         push @$devices, '-device', print_vga_device($conf, $vga, $arch, $machine_version, $machine_type, undef, $qxlnum, $bridges);
3796         my $socket = PVE::QemuServer::Helpers::vnc_socket($vmid);
3797         push @$cmd,  '-vnc', "unix:$socket,password";
3798     } else {
3799         push @$cmd, '-vga', 'none' if $vga->{type} eq 'none';
3800         push @$cmd, '-nographic';
3801     }
3802
3803     # time drift fix
3804     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
3805
3806     my $useLocaltime = $conf->{localtime};
3807
3808     if ($winversion >= 5) { # windows
3809         $useLocaltime = 1 if !defined($conf->{localtime});
3810
3811         # use time drift fix when acpi is enabled
3812         if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
3813             $tdf = 1 if !defined($conf->{tdf});
3814         }
3815     }
3816
3817     if ($winversion >= 6) {
3818         push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3819         push @$cmd, '-no-hpet';
3820     }
3821
3822     push @$rtcFlags, 'driftfix=slew' if $tdf;
3823
3824     if (!$kvm) {
3825         push @$machineFlags, 'accel=tcg';
3826     }
3827
3828     if ($machine_type) {
3829         push @$machineFlags, "type=${machine_type}";
3830     }
3831
3832     if (($conf->{startdate}) && ($conf->{startdate} ne 'now')) {
3833         push @$rtcFlags, "base=$conf->{startdate}";
3834     } elsif ($useLocaltime) {
3835         push @$rtcFlags, 'base=localtime';
3836     }
3837
3838     push @$cmd, get_cpu_options($conf, $arch, $kvm, $kvm_off, $machine_version, $winversion, $gpu_passthrough);
3839
3840     PVE::QemuServer::Memory::config($conf, $vmid, $sockets, $cores, $defaults, $hotplug_features, $cmd);
3841
3842     push @$cmd, '-S' if $conf->{freeze};
3843
3844     push @$cmd, '-k', $conf->{keyboard} if defined($conf->{keyboard});
3845
3846     my $guest_agent = parse_guest_agent($conf);
3847
3848     if ($guest_agent->{enabled}) {
3849         my $qgasocket = PVE::QemuServer::Helpers::qmp_socket($vmid, 1);
3850         push @$devices, '-chardev', "socket,path=$qgasocket,server,nowait,id=qga0";
3851
3852         if (!$guest_agent->{type} || $guest_agent->{type} eq 'virtio') {
3853             my $pciaddr = print_pci_addr("qga0", $bridges, $arch, $machine_type);
3854             push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3855             push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3856         } elsif ($guest_agent->{type} eq 'isa') {
3857             push @$devices, '-device', "isa-serial,chardev=qga0";
3858         }
3859     }
3860
3861     my $spice_port;
3862
3863     if ($qxlnum) {
3864         if ($qxlnum > 1) {
3865             if ($winversion){
3866                 for(my $i = 1; $i < $qxlnum; $i++){
3867                     push @$devices, '-device', print_vga_device($conf, $vga, $arch, $machine_version, $machine_type, $i, $qxlnum, $bridges);
3868                 }
3869             } else {
3870                 # assume other OS works like Linux
3871                 my ($ram, $vram) = ("134217728", "67108864");
3872                 if ($vga->{memory}) {
3873                     $ram = PVE::Tools::convert_size($qxlnum*4*$vga->{memory}, 'mb' => 'b');
3874                     $vram = PVE::Tools::convert_size($qxlnum*2*$vga->{memory}, 'mb' => 'b');
3875                 }
3876                 push @$cmd, '-global', "qxl-vga.ram_size=$ram";
3877                 push @$cmd, '-global', "qxl-vga.vram_size=$vram";
3878             }
3879         }
3880
3881         my $pciaddr = print_pci_addr("spice", $bridges, $arch, $machine_type);
3882
3883         my $nodename = PVE::INotify::nodename();
3884         my $pfamily = PVE::Tools::get_host_address_family($nodename);
3885         my @nodeaddrs = PVE::Tools::getaddrinfo_all('localhost', family => $pfamily);
3886         die "failed to get an ip address of type $pfamily for 'localhost'\n" if !@nodeaddrs;
3887
3888         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3889         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3890         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3891
3892         my $localhost = PVE::Network::addr_to_ip($nodeaddrs[0]->{addr});
3893         $spice_port = PVE::Tools::next_spice_port($pfamily, $localhost);
3894
3895         my $spice_enhancement = PVE::JSONSchema::parse_property_string($spice_enhancements_fmt, $conf->{spice_enhancements} // '');
3896         if ($spice_enhancement->{foldersharing}) {
3897             push @$devices, '-chardev', "spiceport,id=foldershare,name=org.spice-space.webdav.0";
3898             push @$devices, '-device', "virtserialport,chardev=foldershare,name=org.spice-space.webdav.0";
3899         }
3900
3901         my $spice_opts = "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";
3902         $spice_opts .= ",streaming-video=$spice_enhancement->{videostreaming}" if $spice_enhancement->{videostreaming};
3903         push @$devices, '-spice', "$spice_opts";
3904     }
3905
3906     # enable balloon by default, unless explicitly disabled
3907     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3908         $pciaddr = print_pci_addr("balloon0", $bridges, $arch, $machine_type);
3909         push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
3910     }
3911
3912     if ($conf->{watchdog}) {
3913         my $wdopts = parse_watchdog($conf->{watchdog});
3914         $pciaddr = print_pci_addr("watchdog", $bridges, $arch, $machine_type);
3915         my $watchdog = $wdopts->{model} || 'i6300esb';
3916         push @$devices, '-device', "$watchdog$pciaddr";
3917         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3918     }
3919
3920     my $vollist = [];
3921     my $scsicontroller = {};
3922     my $ahcicontroller = {};
3923     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3924
3925     # Add iscsi initiator name if available
3926     if (my $initiator = get_initiator_name()) {
3927         push @$devices, '-iscsi', "initiator-name=$initiator";
3928     }
3929
3930     foreach_drive($conf, sub {
3931         my ($ds, $drive) = @_;
3932
3933         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3934             push @$vollist, $drive->{file};
3935         }
3936
3937         # ignore efidisk here, already added in bios/fw handling code above
3938         return if $drive->{interface} eq 'efidisk';
3939
3940         $use_virtio = 1 if $ds =~ m/^virtio/;
3941
3942         if (drive_is_cdrom ($drive)) {
3943             if ($bootindex_hash->{d}) {
3944                 $drive->{bootindex} = $bootindex_hash->{d};
3945                 $bootindex_hash->{d} += 1;
3946             }
3947         } else {
3948             if ($bootindex_hash->{c}) {
3949                 $drive->{bootindex} = $bootindex_hash->{c} if $conf->{bootdisk} && ($conf->{bootdisk} eq $ds);
3950                 $bootindex_hash->{c} += 1;
3951             }
3952         }
3953
3954         if($drive->{interface} eq 'virtio'){
3955            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3956         }
3957
3958         if ($drive->{interface} eq 'scsi') {
3959
3960             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
3961
3962             $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges, $arch, $machine_type);
3963             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
3964
3965             my $iothread = '';
3966             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
3967                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3968                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3969             } elsif ($drive->{iothread}) {
3970                 warn "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n";
3971             }
3972
3973             my $queues = '';
3974             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
3975                 $queues = ",num_queues=$drive->{queues}";
3976             }
3977
3978             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues" if !$scsicontroller->{$controller};
3979             $scsicontroller->{$controller}=1;
3980         }
3981
3982         if ($drive->{interface} eq 'sata') {
3983            my $controller = int($drive->{index} / $MAX_SATA_DISKS);
3984            $pciaddr = print_pci_addr("ahci$controller", $bridges, $arch, $machine_type);
3985            push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr" if !$ahcicontroller->{$controller};
3986            $ahcicontroller->{$controller}=1;
3987         }
3988
3989         my $drive_cmd = print_drive_full($storecfg, $vmid, $drive);
3990         push @$devices, '-drive',$drive_cmd;
3991         push @$devices, '-device', print_drivedevice_full($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type);
3992     });
3993
3994     for (my $i = 0; $i < $MAX_NETS; $i++) {
3995          next if !$conf->{"net$i"};
3996          my $d = parse_net($conf->{"net$i"});
3997          next if !$d;
3998
3999          $use_virtio = 1 if $d->{model} eq 'virtio';
4000
4001          if ($bootindex_hash->{n}) {
4002             $d->{bootindex} = $bootindex_hash->{n};
4003             $bootindex_hash->{n} += 1;
4004          }
4005
4006          my $netdevfull = print_netdev_full($vmid, $conf, $arch, $d, "net$i");
4007          push @$devices, '-netdev', $netdevfull;
4008
4009          my $netdevicefull = print_netdevice_full($vmid, $conf, $d, "net$i", $bridges, $use_old_bios_files, $arch, $machine_type);
4010          push @$devices, '-device', $netdevicefull;
4011     }
4012
4013     if ($conf->{ivshmem}) {
4014         my $ivshmem = PVE::JSONSchema::parse_property_string($ivshmem_fmt, $conf->{ivshmem});
4015
4016         my $bus;
4017         if ($q35) {
4018             $bus = print_pcie_addr("ivshmem");
4019         } else {
4020             $bus = print_pci_addr("ivshmem", $bridges, $arch, $machine_type);
4021         }
4022
4023         my $ivshmem_name = $ivshmem->{name} // $vmid;
4024         my $path = '/dev/shm/pve-shm-' . $ivshmem_name;
4025
4026         push @$devices, '-device', "ivshmem-plain,memdev=ivshmem$bus,";
4027         push @$devices, '-object', "memory-backend-file,id=ivshmem,share=on,mem-path=$path,size=$ivshmem->{size}M";
4028     }
4029
4030     if (!$q35) {
4031         # add pci bridges
4032         if (min_version($machine_version, 2, 3)) {
4033            $bridges->{1} = 1;
4034            $bridges->{2} = 1;
4035         }
4036
4037         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
4038
4039         for my $k (sort {$b cmp $a} keys %$bridges) {
4040             $pciaddr = print_pci_addr("pci.$k", undef, $arch, $machine_type);
4041             unshift @$devices, '-device', "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr" if $k > 0;
4042         }
4043     }
4044
4045     push @$cmd, @$devices;
4046     push @$cmd, '-rtc', join(',', @$rtcFlags)
4047         if scalar(@$rtcFlags);
4048     push @$cmd, '-machine', join(',', @$machineFlags)
4049         if scalar(@$machineFlags);
4050     push @$cmd, '-global', join(',', @$globalFlags)
4051         if scalar(@$globalFlags);
4052
4053     if (my $vmstate = $conf->{vmstate}) {
4054         my $statepath = PVE::Storage::path($storecfg, $vmstate);
4055         push @$vollist, $vmstate;
4056         push @$cmd, '-loadstate', $statepath;
4057         print "activating and using '$vmstate' as vmstate\n";
4058     }
4059
4060     # add custom args
4061     if ($conf->{args}) {
4062         my $aa = PVE::Tools::split_args($conf->{args});
4063         push @$cmd, @$aa;
4064     }
4065
4066     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
4067 }
4068
4069 sub spice_port {
4070     my ($vmid) = @_;
4071
4072     my $res = mon_cmd($vmid, 'query-spice');
4073
4074     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
4075 }
4076
4077 sub vm_devices_list {
4078     my ($vmid) = @_;
4079
4080     my $res = mon_cmd($vmid, 'query-pci');
4081     my $devices_to_check = [];
4082     my $devices = {};
4083     foreach my $pcibus (@$res) {
4084         push @$devices_to_check, @{$pcibus->{devices}},
4085     }
4086
4087     while (@$devices_to_check) {
4088         my $to_check = [];
4089         for my $d (@$devices_to_check) {
4090             $devices->{$d->{'qdev_id'}} = 1 if $d->{'qdev_id'};
4091             next if !$d->{'pci_bridge'};
4092
4093             $devices->{$d->{'qdev_id'}} += scalar(@{$d->{'pci_bridge'}->{devices}});
4094             push @$to_check, @{$d->{'pci_bridge'}->{devices}};
4095         }
4096         $devices_to_check = $to_check;
4097     }
4098
4099     my $resblock = mon_cmd($vmid, 'query-block');
4100     foreach my $block (@$resblock) {
4101         if($block->{device} =~ m/^drive-(\S+)/){
4102                 $devices->{$1} = 1;
4103         }
4104     }
4105
4106     my $resmice = mon_cmd($vmid, 'query-mice');
4107     foreach my $mice (@$resmice) {
4108         if ($mice->{name} eq 'QEMU HID Tablet') {
4109             $devices->{tablet} = 1;
4110             last;
4111         }
4112     }
4113
4114     # for usb devices there is no query-usb
4115     # but we can iterate over the entries in
4116     # qom-list path=/machine/peripheral
4117     my $resperipheral = mon_cmd($vmid, 'qom-list', path => '/machine/peripheral');
4118     foreach my $per (@$resperipheral) {
4119         if ($per->{name} =~ m/^usb\d+$/) {
4120             $devices->{$per->{name}} = 1;
4121         }
4122     }
4123
4124     return $devices;
4125 }
4126
4127 sub vm_deviceplug {
4128     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4129
4130     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
4131
4132     my $devices_list = vm_devices_list($vmid);
4133     return 1 if defined($devices_list->{$deviceid});
4134
4135     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid, $arch, $machine_type); # add PCI bridge if we need it for the device
4136
4137     if ($deviceid eq 'tablet') {
4138
4139         qemu_deviceadd($vmid, print_tabletdevice_full($conf, $arch));
4140
4141     } elsif ($deviceid eq 'keyboard') {
4142
4143         qemu_deviceadd($vmid, print_keyboarddevice_full($conf, $arch));
4144
4145     } elsif ($deviceid =~ m/^usb(\d+)$/) {
4146
4147         die "usb hotplug currently not reliable\n";
4148         # since we can't reliably hot unplug all added usb devices
4149         # and usb passthrough disables live migration
4150         # we disable usb hotplugging for now
4151         qemu_deviceadd($vmid, PVE::QemuServer::USB::print_usbdevice_full($conf, $deviceid, $device));
4152
4153     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4154
4155         qemu_iothread_add($vmid, $deviceid, $device);
4156
4157         qemu_driveadd($storecfg, $vmid, $device);
4158         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, $arch, $machine_type);
4159
4160         qemu_deviceadd($vmid, $devicefull);
4161         eval { qemu_deviceaddverify($vmid, $deviceid); };
4162         if (my $err = $@) {
4163             eval { qemu_drivedel($vmid, $deviceid); };
4164             warn $@ if $@;
4165             die $err;
4166         }
4167
4168     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4169
4170
4171         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
4172         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
4173         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
4174
4175         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
4176
4177         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
4178             qemu_iothread_add($vmid, $deviceid, $device);
4179             $devicefull .= ",iothread=iothread-$deviceid";
4180         }
4181
4182         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
4183             $devicefull .= ",num_queues=$device->{queues}";
4184         }
4185
4186         qemu_deviceadd($vmid, $devicefull);
4187         qemu_deviceaddverify($vmid, $deviceid);
4188
4189     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4190
4191         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device, $arch, $machine_type);
4192         qemu_driveadd($storecfg, $vmid, $device);
4193
4194         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, $arch, $machine_type);
4195         eval { qemu_deviceadd($vmid, $devicefull); };
4196         if (my $err = $@) {
4197             eval { qemu_drivedel($vmid, $deviceid); };
4198             warn $@ if $@;
4199             die $err;
4200         }
4201
4202     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4203
4204         return undef if !qemu_netdevadd($vmid, $conf, $arch, $device, $deviceid);
4205
4206         my $machine_type = PVE::QemuServer::Machine::qemu_machine_pxe($vmid, $conf);
4207         my $use_old_bios_files = undef;
4208         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
4209
4210         my $netdevicefull = print_netdevice_full($vmid, $conf, $device, $deviceid, undef, $use_old_bios_files, $arch, $machine_type);
4211         qemu_deviceadd($vmid, $netdevicefull);
4212         eval {
4213             qemu_deviceaddverify($vmid, $deviceid);
4214             qemu_set_link_status($vmid, $deviceid, !$device->{link_down});
4215         };
4216         if (my $err = $@) {
4217             eval { qemu_netdevdel($vmid, $deviceid); };
4218             warn $@ if $@;
4219             die $err;
4220         }
4221
4222     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
4223
4224         my $bridgeid = $2;
4225         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
4226         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
4227
4228         qemu_deviceadd($vmid, $devicefull);
4229         qemu_deviceaddverify($vmid, $deviceid);
4230
4231     } else {
4232         die "can't hotplug device '$deviceid'\n";
4233     }
4234
4235     return 1;
4236 }
4237
4238 # fixme: this should raise exceptions on error!
4239 sub vm_deviceunplug {
4240     my ($vmid, $conf, $deviceid) = @_;
4241
4242     my $devices_list = vm_devices_list($vmid);
4243     return 1 if !defined($devices_list->{$deviceid});
4244
4245     die "can't unplug bootdisk" if $conf->{bootdisk} && $conf->{bootdisk} eq $deviceid;
4246
4247     if ($deviceid eq 'tablet' || $deviceid eq 'keyboard') {
4248
4249         qemu_devicedel($vmid, $deviceid);
4250
4251     } elsif ($deviceid =~ m/^usb\d+$/) {
4252
4253         die "usb hotplug currently not reliable\n";
4254         # when unplugging usb devices this way,
4255         # there may be remaining usb controllers/hubs
4256         # so we disable it for now
4257         qemu_devicedel($vmid, $deviceid);
4258         qemu_devicedelverify($vmid, $deviceid);
4259
4260     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4261
4262         qemu_devicedel($vmid, $deviceid);
4263         qemu_devicedelverify($vmid, $deviceid);
4264         qemu_drivedel($vmid, $deviceid);
4265         qemu_iothread_del($conf, $vmid, $deviceid);
4266
4267     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4268
4269         qemu_devicedel($vmid, $deviceid);
4270         qemu_devicedelverify($vmid, $deviceid);
4271         qemu_iothread_del($conf, $vmid, $deviceid);
4272
4273     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4274
4275         qemu_devicedel($vmid, $deviceid);
4276         qemu_drivedel($vmid, $deviceid);
4277         qemu_deletescsihw($conf, $vmid, $deviceid);
4278
4279     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4280
4281         qemu_devicedel($vmid, $deviceid);
4282         qemu_devicedelverify($vmid, $deviceid);
4283         qemu_netdevdel($vmid, $deviceid);
4284
4285     } else {
4286         die "can't unplug device '$deviceid'\n";
4287     }
4288
4289     return 1;
4290 }
4291
4292 sub qemu_deviceadd {
4293     my ($vmid, $devicefull) = @_;
4294
4295     $devicefull = "driver=".$devicefull;
4296     my %options =  split(/[=,]/, $devicefull);
4297
4298     mon_cmd($vmid, "device_add" , %options);
4299 }
4300
4301 sub qemu_devicedel {
4302     my ($vmid, $deviceid) = @_;
4303
4304     my $ret = mon_cmd($vmid, "device_del", id => $deviceid);
4305 }
4306
4307 sub qemu_iothread_add {
4308     my($vmid, $deviceid, $device) = @_;
4309
4310     if ($device->{iothread}) {
4311         my $iothreads = vm_iothreads_list($vmid);
4312         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
4313     }
4314 }
4315
4316 sub qemu_iothread_del {
4317     my($conf, $vmid, $deviceid) = @_;
4318
4319     my $confid = $deviceid;
4320     if ($deviceid =~ m/^(?:virtioscsi|scsihw)(\d+)$/) {
4321         $confid = 'scsi' . $1;
4322     }
4323     my $device = parse_drive($confid, $conf->{$confid});
4324     if ($device->{iothread}) {
4325         my $iothreads = vm_iothreads_list($vmid);
4326         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
4327     }
4328 }
4329
4330 sub qemu_objectadd {
4331     my($vmid, $objectid, $qomtype) = @_;
4332
4333     mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
4334
4335     return 1;
4336 }
4337
4338 sub qemu_objectdel {
4339     my($vmid, $objectid) = @_;
4340
4341     mon_cmd($vmid, "object-del", id => $objectid);
4342
4343     return 1;
4344 }
4345
4346 sub qemu_driveadd {
4347     my ($storecfg, $vmid, $device) = @_;
4348
4349     my $drive = print_drive_full($storecfg, $vmid, $device);
4350     $drive =~ s/\\/\\\\/g;
4351     my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_add auto \"$drive\"");
4352
4353     # If the command succeeds qemu prints: "OK"
4354     return 1 if $ret =~ m/OK/s;
4355
4356     die "adding drive failed: $ret\n";
4357 }
4358
4359 sub qemu_drivedel {
4360     my($vmid, $deviceid) = @_;
4361
4362     my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_del drive-$deviceid");
4363     $ret =~ s/^\s+//;
4364
4365     return 1 if $ret eq "";
4366
4367     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
4368     return 1 if $ret =~ m/Device \'.*?\' not found/s;
4369
4370     die "deleting drive $deviceid failed : $ret\n";
4371 }
4372
4373 sub qemu_deviceaddverify {
4374     my ($vmid, $deviceid) = @_;
4375
4376     for (my $i = 0; $i <= 5; $i++) {
4377          my $devices_list = vm_devices_list($vmid);
4378          return 1 if defined($devices_list->{$deviceid});
4379          sleep 1;
4380     }
4381
4382     die "error on hotplug device '$deviceid'\n";
4383 }
4384
4385
4386 sub qemu_devicedelverify {
4387     my ($vmid, $deviceid) = @_;
4388
4389     # need to verify that the device is correctly removed as device_del
4390     # is async and empty return is not reliable
4391
4392     for (my $i = 0; $i <= 5; $i++) {
4393          my $devices_list = vm_devices_list($vmid);
4394          return 1 if !defined($devices_list->{$deviceid});
4395          sleep 1;
4396     }
4397
4398     die "error on hot-unplugging device '$deviceid'\n";
4399 }
4400
4401 sub qemu_findorcreatescsihw {
4402     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4403
4404     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4405
4406     my $scsihwid="$controller_prefix$controller";
4407     my $devices_list = vm_devices_list($vmid);
4408
4409     if(!defined($devices_list->{$scsihwid})) {
4410         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device, $arch, $machine_type);
4411     }
4412
4413     return 1;
4414 }
4415
4416 sub qemu_deletescsihw {
4417     my ($conf, $vmid, $opt) = @_;
4418
4419     my $device = parse_drive($opt, $conf->{$opt});
4420
4421     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
4422         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
4423         return 1;
4424     }
4425
4426     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4427
4428     my $devices_list = vm_devices_list($vmid);
4429     foreach my $opt (keys %{$devices_list}) {
4430         if (PVE::QemuServer::is_valid_drivename($opt)) {
4431             my $drive = PVE::QemuServer::parse_drive($opt, $conf->{$opt});
4432             if($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
4433                 return 1;
4434             }
4435         }
4436     }
4437
4438     my $scsihwid="scsihw$controller";
4439
4440     vm_deviceunplug($vmid, $conf, $scsihwid);
4441
4442     return 1;
4443 }
4444
4445 sub qemu_add_pci_bridge {
4446     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4447
4448     my $bridges = {};
4449
4450     my $bridgeid;
4451
4452     print_pci_addr($device, $bridges, $arch, $machine_type);
4453
4454     while (my ($k, $v) = each %$bridges) {
4455         $bridgeid = $k;
4456     }
4457     return 1 if !defined($bridgeid) || $bridgeid < 1;
4458
4459     my $bridge = "pci.$bridgeid";
4460     my $devices_list = vm_devices_list($vmid);
4461
4462     if (!defined($devices_list->{$bridge})) {
4463         vm_deviceplug($storecfg, $conf, $vmid, $bridge, $arch, $machine_type);
4464     }
4465
4466     return 1;
4467 }
4468
4469 sub qemu_set_link_status {
4470     my ($vmid, $device, $up) = @_;
4471
4472     mon_cmd($vmid, "set_link", name => $device,
4473                up => $up ? JSON::true : JSON::false);
4474 }
4475
4476 sub qemu_netdevadd {
4477     my ($vmid, $conf, $arch, $device, $deviceid) = @_;
4478
4479     my $netdev = print_netdev_full($vmid, $conf, $arch, $device, $deviceid, 1);
4480     my %options =  split(/[=,]/, $netdev);
4481
4482     mon_cmd($vmid, "netdev_add",  %options);
4483     return 1;
4484 }
4485
4486 sub qemu_netdevdel {
4487     my ($vmid, $deviceid) = @_;
4488
4489     mon_cmd($vmid, "netdev_del", id => $deviceid);
4490 }
4491
4492 sub qemu_usb_hotplug {
4493     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4494
4495     return if !$device;
4496
4497     # remove the old one first
4498     vm_deviceunplug($vmid, $conf, $deviceid);
4499
4500     # check if xhci controller is necessary and available
4501     if ($device->{usb3}) {
4502
4503         my $devicelist = vm_devices_list($vmid);
4504
4505         if (!$devicelist->{xhci}) {
4506             my $pciaddr = print_pci_addr("xhci", undef, $arch, $machine_type);
4507             qemu_deviceadd($vmid, "nec-usb-xhci,id=xhci$pciaddr");
4508         }
4509     }
4510     my $d = parse_usb_device($device->{host});
4511     $d->{usb3} = $device->{usb3};
4512
4513     # add the new one
4514     vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $d, $arch, $machine_type);
4515 }
4516
4517 sub qemu_cpu_hotplug {
4518     my ($vmid, $conf, $vcpus) = @_;
4519
4520     my $machine_type = PVE::QemuServer::Machine::get_current_qemu_machine($vmid);
4521
4522     my $sockets = 1;
4523     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
4524     $sockets = $conf->{sockets} if  $conf->{sockets};
4525     my $cores = $conf->{cores} || 1;
4526     my $maxcpus = $sockets * $cores;
4527
4528     $vcpus = $maxcpus if !$vcpus;
4529
4530     die "you can't add more vcpus than maxcpus\n"
4531         if $vcpus > $maxcpus;
4532
4533     my $currentvcpus = $conf->{vcpus} || $maxcpus;
4534
4535     if ($vcpus < $currentvcpus) {
4536
4537         if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4538
4539             for (my $i = $currentvcpus; $i > $vcpus; $i--) {
4540                 qemu_devicedel($vmid, "cpu$i");
4541                 my $retry = 0;
4542                 my $currentrunningvcpus = undef;
4543                 while (1) {
4544                     $currentrunningvcpus = mon_cmd($vmid, "query-cpus");
4545                     last if scalar(@{$currentrunningvcpus}) == $i-1;
4546                     raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
4547                     $retry++;
4548                     sleep 1;
4549                 }
4550                 #update conf after each succesfull cpu unplug
4551                 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4552                 PVE::QemuConfig->write_config($vmid, $conf);
4553             }
4554         } else {
4555             die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
4556         }
4557
4558         return;
4559     }
4560
4561     my $currentrunningvcpus = mon_cmd($vmid, "query-cpus");
4562     die "vcpus in running vm does not match its configuration\n"
4563         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
4564
4565     if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4566
4567         for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
4568             my $cpustr = print_cpu_device($conf, $i);
4569             qemu_deviceadd($vmid, $cpustr);
4570
4571             my $retry = 0;
4572             my $currentrunningvcpus = undef;
4573             while (1) {
4574                 $currentrunningvcpus = mon_cmd($vmid, "query-cpus");
4575                 last if scalar(@{$currentrunningvcpus}) == $i;
4576                 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
4577                 sleep 1;
4578                 $retry++;
4579             }
4580             #update conf after each succesfull cpu hotplug
4581             $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4582             PVE::QemuConfig->write_config($vmid, $conf);
4583         }
4584     } else {
4585
4586         for (my $i = $currentvcpus; $i < $vcpus; $i++) {
4587             mon_cmd($vmid, "cpu-add", id => int($i));
4588         }
4589     }
4590 }
4591
4592 sub qemu_block_set_io_throttle {
4593     my ($vmid, $deviceid,
4594         $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
4595         $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max,
4596         $bps_max_length, $bps_rd_max_length, $bps_wr_max_length,
4597         $iops_max_length, $iops_rd_max_length, $iops_wr_max_length) = @_;
4598
4599     return if !check_running($vmid) ;
4600
4601     mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
4602         bps => int($bps),
4603         bps_rd => int($bps_rd),
4604         bps_wr => int($bps_wr),
4605         iops => int($iops),
4606         iops_rd => int($iops_rd),
4607         iops_wr => int($iops_wr),
4608         bps_max => int($bps_max),
4609         bps_rd_max => int($bps_rd_max),
4610         bps_wr_max => int($bps_wr_max),
4611         iops_max => int($iops_max),
4612         iops_rd_max => int($iops_rd_max),
4613         iops_wr_max => int($iops_wr_max),
4614         bps_max_length => int($bps_max_length),
4615         bps_rd_max_length => int($bps_rd_max_length),
4616         bps_wr_max_length => int($bps_wr_max_length),
4617         iops_max_length => int($iops_max_length),
4618         iops_rd_max_length => int($iops_rd_max_length),
4619         iops_wr_max_length => int($iops_wr_max_length),
4620     );
4621
4622 }
4623
4624 # old code, only used to shutdown old VM after update
4625 sub __read_avail {
4626     my ($fh, $timeout) = @_;
4627
4628     my $sel = new IO::Select;
4629     $sel->add($fh);
4630
4631     my $res = '';
4632     my $buf;
4633
4634     my @ready;
4635     while (scalar (@ready = $sel->can_read($timeout))) {
4636         my $count;
4637         if ($count = $fh->sysread($buf, 8192)) {
4638             if ($buf =~ /^(.*)\(qemu\) $/s) {
4639                 $res .= $1;
4640                 last;
4641             } else {
4642                 $res .= $buf;
4643             }
4644         } else {
4645             if (!defined($count)) {
4646                 die "$!\n";
4647             }
4648             last;
4649         }
4650     }
4651
4652     die "monitor read timeout\n" if !scalar(@ready);
4653
4654     return $res;
4655 }
4656
4657 sub qemu_block_resize {
4658     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4659
4660     my $running = check_running($vmid);
4661
4662     $size = 0 if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4663
4664     return if !$running;
4665
4666     mon_cmd($vmid, "block_resize", device => $deviceid, size => int($size));
4667
4668 }
4669
4670 sub qemu_volume_snapshot {
4671     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4672
4673     my $running = check_running($vmid);
4674
4675     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4676         mon_cmd($vmid, 'blockdev-snapshot-internal-sync', device => $deviceid, name => $snap);
4677     } else {
4678         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4679     }
4680 }
4681
4682 sub qemu_volume_snapshot_delete {
4683     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4684
4685     my $running = check_running($vmid);
4686
4687     if($running) {
4688
4689         $running = undef;
4690         my $conf = PVE::QemuConfig->load_config($vmid);
4691         foreach_drive($conf, sub {
4692             my ($ds, $drive) = @_;
4693             $running = 1 if $drive->{file} eq $volid;
4694         });
4695     }
4696
4697     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4698         mon_cmd($vmid, 'blockdev-snapshot-delete-internal-sync', device => $deviceid, name => $snap);
4699     } else {
4700         PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4701     }
4702 }
4703
4704 sub set_migration_caps {
4705     my ($vmid) = @_;
4706
4707     my $cap_ref = [];
4708
4709     my $enabled_cap = {
4710         "auto-converge" => 1,
4711         "xbzrle" => 1,
4712         "x-rdma-pin-all" => 0,
4713         "zero-blocks" => 0,
4714         "compress" => 0
4715     };
4716
4717     my $supported_capabilities = mon_cmd($vmid, "query-migrate-capabilities");
4718
4719     for my $supported_capability (@$supported_capabilities) {
4720         push @$cap_ref, {
4721             capability => $supported_capability->{capability},
4722             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4723         };
4724     }
4725
4726     mon_cmd($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4727 }
4728
4729 my $fast_plug_option = {
4730     'lock' => 1,
4731     'name' => 1,
4732     'onboot' => 1,
4733     'shares' => 1,
4734     'startup' => 1,
4735     'description' => 1,
4736     'protection' => 1,
4737     'vmstatestorage' => 1,
4738     'hookscript' => 1,
4739     'tags' => 1,
4740 };
4741
4742 # hotplug changes in [PENDING]
4743 # $selection hash can be used to only apply specified options, for
4744 # example: { cores => 1 } (only apply changed 'cores')
4745 # $errors ref is used to return error messages
4746 sub vmconfig_hotplug_pending {
4747     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4748
4749     my $defaults = load_defaults();
4750     my $arch = get_vm_arch($conf);
4751     my $machine_type = get_vm_machine($conf, undef, $arch);
4752
4753     # commit values which do not have any impact on running VM first
4754     # Note: those option cannot raise errors, we we do not care about
4755     # $selection and always apply them.
4756
4757     my $add_error = sub {
4758         my ($opt, $msg) = @_;
4759         $errors->{$opt} = "hotplug problem - $msg";
4760     };
4761
4762     my $changes = 0;
4763     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4764         if ($fast_plug_option->{$opt}) {
4765             $conf->{$opt} = $conf->{pending}->{$opt};
4766             delete $conf->{pending}->{$opt};
4767             $changes = 1;
4768         }
4769     }
4770
4771     if ($changes) {
4772         PVE::QemuConfig->write_config($vmid, $conf);
4773         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4774     }
4775
4776     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4777
4778     my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4779     foreach my $opt (sort keys %$pending_delete_hash) {
4780         next if $selection && !$selection->{$opt};
4781         my $force = $pending_delete_hash->{$opt}->{force};
4782         eval {
4783             if ($opt eq 'hotplug') {
4784                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
4785             } elsif ($opt eq 'tablet') {
4786                 die "skip\n" if !$hotplug_features->{usb};
4787                 if ($defaults->{tablet}) {
4788                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4789                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4790                         if $arch eq 'aarch64';
4791                 } else {
4792                     vm_deviceunplug($vmid, $conf, 'tablet');
4793                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4794                 }
4795             } elsif ($opt =~ m/^usb\d+/) {
4796                 die "skip\n";
4797                 # since we cannot reliably hot unplug usb devices
4798                 # we are disabling it
4799                 die "skip\n" if !$hotplug_features->{usb} || $conf->{$opt} =~ m/spice/i;
4800                 vm_deviceunplug($vmid, $conf, $opt);
4801             } elsif ($opt eq 'vcpus') {
4802                 die "skip\n" if !$hotplug_features->{cpu};
4803                 qemu_cpu_hotplug($vmid, $conf, undef);
4804             } elsif ($opt eq 'balloon') {
4805                 # enable balloon device is not hotpluggable
4806                 die "skip\n" if defined($conf->{balloon}) && $conf->{balloon} == 0;
4807                 # here we reset the ballooning value to memory
4808                 my $balloon = $conf->{memory} || $defaults->{memory};
4809                 mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4810             } elsif ($fast_plug_option->{$opt}) {
4811                 # do nothing
4812             } elsif ($opt =~ m/^net(\d+)$/) {
4813                 die "skip\n" if !$hotplug_features->{network};
4814                 vm_deviceunplug($vmid, $conf, $opt);
4815             } elsif (is_valid_drivename($opt)) {
4816                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4817                 vm_deviceunplug($vmid, $conf, $opt);
4818                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4819             } elsif ($opt =~ m/^memory$/) {
4820                 die "skip\n" if !$hotplug_features->{memory};
4821                 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
4822             } elsif ($opt eq 'cpuunits') {
4823                 cgroups_write("cpu", $vmid, "cpu.shares", $defaults->{cpuunits});
4824             } elsif ($opt eq 'cpulimit') {
4825                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", -1);
4826             } else {
4827                 die "skip\n";
4828             }
4829         };
4830         if (my $err = $@) {
4831             &$add_error($opt, $err) if $err ne "skip\n";
4832         } else {
4833             # save new config if hotplug was successful
4834             delete $conf->{$opt};
4835             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4836             PVE::QemuConfig->write_config($vmid, $conf);
4837             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4838         }
4839     }
4840
4841     my ($apply_pending_cloudinit, $apply_pending_cloudinit_done);
4842     $apply_pending_cloudinit = sub {
4843         return if $apply_pending_cloudinit_done; # once is enough
4844         $apply_pending_cloudinit_done = 1; # once is enough
4845
4846         my ($key, $value) = @_;
4847
4848         my @cloudinit_opts = keys %$confdesc_cloudinit;
4849         foreach my $opt (keys %{$conf->{pending}}) {
4850             next if !grep { $_ eq $opt } @cloudinit_opts;
4851             $conf->{$opt} = delete $conf->{pending}->{$opt};
4852         }
4853
4854         my $new_conf = { %$conf };
4855         $new_conf->{$key} = $value;
4856         PVE::QemuServer::Cloudinit::generate_cloudinitconfig($new_conf, $vmid);
4857     };
4858
4859     foreach my $opt (keys %{$conf->{pending}}) {
4860         next if $selection && !$selection->{$opt};
4861         my $value = $conf->{pending}->{$opt};
4862         eval {
4863             if ($opt eq 'hotplug') {
4864                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
4865             } elsif ($opt eq 'tablet') {
4866                 die "skip\n" if !$hotplug_features->{usb};
4867                 if ($value == 1) {
4868                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4869                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4870                         if $arch eq 'aarch64';
4871                 } elsif ($value == 0) {
4872                     vm_deviceunplug($vmid, $conf, 'tablet');
4873                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4874                 }
4875             } elsif ($opt =~ m/^usb\d+$/) {
4876                 die "skip\n";
4877                 # since we cannot reliably hot unplug usb devices
4878                 # we are disabling it
4879                 die "skip\n" if !$hotplug_features->{usb} || $value =~ m/spice/i;
4880                 my $d = eval { PVE::JSONSchema::parse_property_string($usbdesc->{format}, $value) };
4881                 die "skip\n" if !$d;
4882                 qemu_usb_hotplug($storecfg, $conf, $vmid, $opt, $d, $arch, $machine_type);
4883             } elsif ($opt eq 'vcpus') {
4884                 die "skip\n" if !$hotplug_features->{cpu};
4885                 qemu_cpu_hotplug($vmid, $conf, $value);
4886             } elsif ($opt eq 'balloon') {
4887                 # enable/disable balloning device is not hotpluggable
4888                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
4889                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
4890                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
4891
4892                 # allow manual ballooning if shares is set to zero
4893                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
4894                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
4895                     mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4896                 }
4897             } elsif ($opt =~ m/^net(\d+)$/) {
4898                 # some changes can be done without hotplug
4899                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
4900                                     $vmid, $opt, $value, $arch, $machine_type);
4901             } elsif (is_valid_drivename($opt)) {
4902                 die "skip\n" if $opt eq 'efidisk0';
4903                 # some changes can be done without hotplug
4904                 my $drive = parse_drive($opt, $value);
4905                 if (drive_is_cloudinit($drive)) {
4906                     &$apply_pending_cloudinit($opt, $value);
4907                 }
4908                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
4909                                      $vmid, $opt, $value, 1, $arch, $machine_type);
4910             } elsif ($opt =~ m/^memory$/) { #dimms
4911                 die "skip\n" if !$hotplug_features->{memory};
4912                 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
4913             } elsif ($opt eq 'cpuunits') {
4914                 cgroups_write("cpu", $vmid, "cpu.shares", $conf->{pending}->{$opt});
4915             } elsif ($opt eq 'cpulimit') {
4916                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
4917                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", $cpulimit);
4918             } else {
4919                 die "skip\n";  # skip non-hot-pluggable options
4920             }
4921         };
4922         if (my $err = $@) {
4923             &$add_error($opt, $err) if $err ne "skip\n";
4924         } else {
4925             # save new config if hotplug was successful
4926             $conf->{$opt} = $value;
4927             delete $conf->{pending}->{$opt};
4928             PVE::QemuConfig->write_config($vmid, $conf);
4929             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4930         }
4931     }
4932 }
4933
4934 sub try_deallocate_drive {
4935     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
4936
4937     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
4938         my $volid = $drive->{file};
4939         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
4940             my $sid = PVE::Storage::parse_volume_id($volid);
4941             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
4942
4943             # check if the disk is really unused
4944             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
4945                 if is_volume_in_use($storecfg, $conf, $key, $volid);
4946             PVE::Storage::vdisk_free($storecfg, $volid);
4947             return 1;
4948         } else {
4949             # If vm is not owner of this disk remove from config
4950             return 1;
4951         }
4952     }
4953
4954     return undef;
4955 }
4956
4957 sub vmconfig_delete_or_detach_drive {
4958     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
4959
4960     my $drive = parse_drive($opt, $conf->{$opt});
4961
4962     my $rpcenv = PVE::RPCEnvironment::get();
4963     my $authuser = $rpcenv->get_user();
4964
4965     if ($force) {
4966         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
4967         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
4968     } else {
4969         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
4970     }
4971 }
4972
4973
4974
4975 sub vmconfig_apply_pending {
4976     my ($vmid, $conf, $storecfg) = @_;
4977
4978     # cold plug
4979
4980     my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4981     foreach my $opt (sort keys %$pending_delete_hash) {
4982         die "internal error" if $opt =~ m/^unused/;
4983         my $force = $pending_delete_hash->{$opt}->{force};
4984         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4985         if (!defined($conf->{$opt})) {
4986             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4987             PVE::QemuConfig->write_config($vmid, $conf);
4988         } elsif (is_valid_drivename($opt)) {
4989             vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4990             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4991             delete $conf->{$opt};
4992             PVE::QemuConfig->write_config($vmid, $conf);
4993         } else {
4994             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4995             delete $conf->{$opt};
4996             PVE::QemuConfig->write_config($vmid, $conf);
4997         }
4998     }
4999
5000     $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5001
5002     foreach my $opt (keys %{$conf->{pending}}) { # add/change
5003         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5004
5005         if (defined($conf->{$opt}) && ($conf->{$opt} eq $conf->{pending}->{$opt})) {
5006             # skip if nothing changed
5007         } elsif (is_valid_drivename($opt)) {
5008             vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
5009                 if defined($conf->{$opt});
5010             $conf->{$opt} = $conf->{pending}->{$opt};
5011         } else {
5012             $conf->{$opt} = $conf->{pending}->{$opt};
5013         }
5014
5015         delete $conf->{pending}->{$opt};
5016         PVE::QemuConfig->write_config($vmid, $conf);
5017     }
5018 }
5019
5020 my $safe_num_ne = sub {
5021     my ($a, $b) = @_;
5022
5023     return 0 if !defined($a) && !defined($b);
5024     return 1 if !defined($a);
5025     return 1 if !defined($b);
5026
5027     return $a != $b;
5028 };
5029
5030 my $safe_string_ne = sub {
5031     my ($a, $b) = @_;
5032
5033     return 0 if !defined($a) && !defined($b);
5034     return 1 if !defined($a);
5035     return 1 if !defined($b);
5036
5037     return $a ne $b;
5038 };
5039
5040 sub vmconfig_update_net {
5041     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
5042
5043     my $newnet = parse_net($value);
5044
5045     if ($conf->{$opt}) {
5046         my $oldnet = parse_net($conf->{$opt});
5047
5048         if (&$safe_string_ne($oldnet->{model}, $newnet->{model}) ||
5049             &$safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
5050             &$safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
5051             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
5052
5053             # for non online change, we try to hot-unplug
5054             die "skip\n" if !$hotplug;
5055             vm_deviceunplug($vmid, $conf, $opt);
5056         } else {
5057
5058             die "internal error" if $opt !~ m/net(\d+)/;
5059             my $iface = "tap${vmid}i$1";
5060
5061             if (&$safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
5062                 &$safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
5063                 &$safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
5064                 &$safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
5065                 PVE::Network::tap_unplug($iface);
5066                 PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
5067             } elsif (&$safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
5068                 # Rate can be applied on its own but any change above needs to
5069                 # include the rate in tap_plug since OVS resets everything.
5070                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
5071             }
5072
5073             if (&$safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
5074                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
5075             }
5076
5077             return 1;
5078         }
5079     }
5080
5081     if ($hotplug) {
5082         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet, $arch, $machine_type);
5083     } else {
5084         die "skip\n";
5085     }
5086 }
5087
5088 sub vmconfig_update_disk {
5089     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $force, $arch, $machine_type) = @_;
5090
5091     # fixme: do we need force?
5092
5093     my $drive = parse_drive($opt, $value);
5094
5095     if ($conf->{$opt}) {
5096
5097         if (my $old_drive = parse_drive($opt, $conf->{$opt}))  {
5098
5099             my $media = $drive->{media} || 'disk';
5100             my $oldmedia = $old_drive->{media} || 'disk';
5101             die "unable to change media type\n" if $media ne $oldmedia;
5102
5103             if (!drive_is_cdrom($old_drive)) {
5104
5105                 if ($drive->{file} ne $old_drive->{file}) {
5106
5107                     die "skip\n" if !$hotplug;
5108
5109                     # unplug and register as unused
5110                     vm_deviceunplug($vmid, $conf, $opt);
5111                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
5112
5113                 } else {
5114                     # update existing disk
5115
5116                     # skip non hotpluggable value
5117                     if (&$safe_string_ne($drive->{discard}, $old_drive->{discard}) ||
5118                         &$safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
5119                         &$safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
5120                         &$safe_string_ne($drive->{cache}, $old_drive->{cache})) {
5121                         die "skip\n";
5122                     }
5123
5124                     # apply throttle
5125                     if (&$safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
5126                         &$safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
5127                         &$safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
5128                         &$safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
5129                         &$safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
5130                         &$safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
5131                         &$safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
5132                         &$safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
5133                         &$safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
5134                         &$safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
5135                         &$safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
5136                         &$safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max}) ||
5137                         &$safe_num_ne($drive->{bps_max_length}, $old_drive->{bps_max_length}) ||
5138                         &$safe_num_ne($drive->{bps_rd_max_length}, $old_drive->{bps_rd_max_length}) ||
5139                         &$safe_num_ne($drive->{bps_wr_max_length}, $old_drive->{bps_wr_max_length}) ||
5140                         &$safe_num_ne($drive->{iops_max_length}, $old_drive->{iops_max_length}) ||
5141                         &$safe_num_ne($drive->{iops_rd_max_length}, $old_drive->{iops_rd_max_length}) ||
5142                         &$safe_num_ne($drive->{iops_wr_max_length}, $old_drive->{iops_wr_max_length})) {
5143
5144                         qemu_block_set_io_throttle($vmid,"drive-$opt",
5145                                                    ($drive->{mbps} || 0)*1024*1024,
5146                                                    ($drive->{mbps_rd} || 0)*1024*1024,
5147                                                    ($drive->{mbps_wr} || 0)*1024*1024,
5148                                                    $drive->{iops} || 0,
5149                                                    $drive->{iops_rd} || 0,
5150                                                    $drive->{iops_wr} || 0,
5151                                                    ($drive->{mbps_max} || 0)*1024*1024,
5152                                                    ($drive->{mbps_rd_max} || 0)*1024*1024,
5153                                                    ($drive->{mbps_wr_max} || 0)*1024*1024,
5154                                                    $drive->{iops_max} || 0,
5155                                                    $drive->{iops_rd_max} || 0,
5156                                                    $drive->{iops_wr_max} || 0,
5157                                                    $drive->{bps_max_length} || 1,
5158                                                    $drive->{bps_rd_max_length} || 1,
5159                                                    $drive->{bps_wr_max_length} || 1,
5160                                                    $drive->{iops_max_length} || 1,
5161                                                    $drive->{iops_rd_max_length} || 1,
5162                                                    $drive->{iops_wr_max_length} || 1);
5163
5164                     }
5165
5166                     return 1;
5167                 }
5168
5169             } else { # cdrom
5170
5171                 if ($drive->{file} eq 'none') {
5172                     mon_cmd($vmid, "eject",force => JSON::true,device => "drive-$opt");
5173                     if (drive_is_cloudinit($old_drive)) {
5174                         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive);
5175                     }
5176                 } else {
5177                     my $path = get_iso_path($storecfg, $vmid, $drive->{file});
5178                     mon_cmd($vmid, "eject", force => JSON::true,device => "drive-$opt"); # force eject if locked
5179                     mon_cmd($vmid, "change", device => "drive-$opt",target => "$path") if $path;
5180                 }
5181
5182                 return 1;
5183             }
5184         }
5185     }
5186
5187     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
5188     # hotplug new disks
5189     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
5190     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive, $arch, $machine_type);
5191 }
5192
5193 sub vm_start {
5194     my ($storecfg, $vmid, $statefile, $skiplock, $migratedfrom, $paused,
5195         $forcemachine, $spice_ticket, $migration_network, $migration_type, $targetstorage) = @_;
5196
5197     PVE::QemuConfig->lock_config($vmid, sub {
5198         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
5199
5200         die "you can't start a vm if it's a template\n" if PVE::QemuConfig->is_template($conf);
5201
5202         my $is_suspended = PVE::QemuConfig->has_lock($conf, 'suspended');
5203
5204         PVE::QemuConfig->check_lock($conf)
5205             if !($skiplock || $is_suspended);
5206
5207         die "VM $vmid already running\n" if check_running($vmid, undef, $migratedfrom);
5208
5209         # clean up leftover reboot request files
5210         eval { clear_reboot_request($vmid); };
5211         warn $@ if $@;
5212
5213         if (!$statefile && scalar(keys %{$conf->{pending}})) {
5214             vmconfig_apply_pending($vmid, $conf, $storecfg);
5215             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5216         }
5217
5218         PVE::QemuServer::Cloudinit::generate_cloudinitconfig($conf, $vmid);
5219
5220         my $defaults = load_defaults();
5221
5222         # set environment variable useful inside network script
5223         $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
5224
5225         my $local_volumes = {};
5226
5227         if ($targetstorage) {
5228             foreach_drive($conf, sub {
5229                 my ($ds, $drive) = @_;
5230
5231                 return if drive_is_cdrom($drive);
5232
5233                 my $volid = $drive->{file};
5234
5235                 return if !$volid;
5236
5237                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
5238
5239                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5240                 return if $scfg->{shared};
5241                 $local_volumes->{$ds} = [$volid, $storeid, $volname];
5242             });
5243
5244             my $format = undef;
5245
5246             foreach my $opt (sort keys %$local_volumes) {
5247
5248                 my ($volid, $storeid, $volname) = @{$local_volumes->{$opt}};
5249                 my $drive = parse_drive($opt, $conf->{$opt});
5250
5251                 #if remote storage is specified, use default format
5252                 if ($targetstorage && $targetstorage ne "1") {
5253                     $storeid = $targetstorage;
5254                     my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
5255                     $format = $defFormat;
5256                 } else {
5257                     #else we use same format than original
5258                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5259                     $format = qemu_img_format($scfg, $volid);
5260                 }
5261
5262                 my $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $format, undef, ($drive->{size}/1024));
5263                 my $newdrive = $drive;
5264                 $newdrive->{format} = $format;
5265                 $newdrive->{file} = $newvolid;
5266                 my $drivestr = print_drive($newdrive);
5267                 $local_volumes->{$opt} = $drivestr;
5268                 #pass drive to conf for command line
5269                 $conf->{$opt} = $drivestr;
5270             }
5271         }
5272
5273         PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-start', 1);
5274
5275         if ($is_suspended) {
5276             # enforce machine type on suspended vm to ensure HW compatibility
5277             $forcemachine = $conf->{runningmachine};
5278             print "Resuming suspended VM\n";
5279         }
5280
5281         my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine);
5282
5283         my $migration_ip;
5284         my $get_migration_ip = sub {
5285             my ($cidr, $nodename) = @_;
5286
5287             return $migration_ip if defined($migration_ip);
5288
5289             if (!defined($cidr)) {
5290                 my $dc_conf = PVE::Cluster::cfs_read_file('datacenter.cfg');
5291                 $cidr = $dc_conf->{migration}->{network};
5292             }
5293
5294             if (defined($cidr)) {
5295                 my $ips = PVE::Network::get_local_ip_from_cidr($cidr);
5296
5297                 die "could not get IP: no address configured on local " .
5298                     "node for network '$cidr'\n" if scalar(@$ips) == 0;
5299
5300                 die "could not get IP: multiple addresses configured on local " .
5301                     "node for network '$cidr'\n" if scalar(@$ips) > 1;
5302
5303                 $migration_ip = @$ips[0];
5304             }
5305
5306             $migration_ip = PVE::Cluster::remote_node_ip($nodename, 1)
5307                 if !defined($migration_ip);
5308
5309             return $migration_ip;
5310         };
5311
5312         my $migrate_uri;
5313         if ($statefile) {
5314             if ($statefile eq 'tcp') {
5315                 my $localip = "localhost";
5316                 my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
5317                 my $nodename = PVE::INotify::nodename();
5318
5319                 if (!defined($migration_type)) {
5320                     if (defined($datacenterconf->{migration}->{type})) {
5321                         $migration_type = $datacenterconf->{migration}->{type};
5322                     } else {
5323                         $migration_type = 'secure';
5324                     }
5325                 }
5326
5327                 if ($migration_type eq 'insecure') {
5328                     $localip = $get_migration_ip->($migration_network, $nodename);
5329                     $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5330                 }
5331
5332                 my $pfamily = PVE::Tools::get_host_address_family($nodename);
5333                 my $migrate_port = PVE::Tools::next_migrate_port($pfamily);
5334                 $migrate_uri = "tcp:${localip}:${migrate_port}";
5335                 push @$cmd, '-incoming', $migrate_uri;
5336                 push @$cmd, '-S';
5337
5338             } elsif ($statefile eq 'unix') {
5339                 # should be default for secure migrations as a ssh TCP forward
5340                 # tunnel is not deterministic reliable ready and fails regurarly
5341                 # to set up in time, so use UNIX socket forwards
5342                 my $socket_addr = "/run/qemu-server/$vmid.migrate";
5343                 unlink $socket_addr;
5344
5345                 $migrate_uri = "unix:$socket_addr";
5346
5347                 push @$cmd, '-incoming', $migrate_uri;
5348                 push @$cmd, '-S';
5349
5350             } elsif (-e $statefile) {
5351                 push @$cmd, '-loadstate', $statefile;
5352             } else {
5353                 my $statepath = PVE::Storage::path($storecfg, $statefile);
5354                 push @$vollist, $statefile;
5355                 push @$cmd, '-loadstate', $statepath;
5356             }
5357         } elsif ($paused) {
5358             push @$cmd, '-S';
5359         }
5360
5361         # host pci devices
5362         for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
5363           my $d = parse_hostpci($conf->{"hostpci$i"});
5364           next if !$d;
5365           my $pcidevices = $d->{pciid};
5366           foreach my $pcidevice (@$pcidevices) {
5367                 my $pciid = $pcidevice->{id};
5368
5369                 my $info = PVE::SysFSTools::pci_device_info("$pciid");
5370                 die "IOMMU not present\n" if !PVE::SysFSTools::check_iommu_support();
5371                 die "no pci device info for device '$pciid'\n" if !$info;
5372
5373                 if ($d->{mdev}) {
5374                     my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $i);
5375                     PVE::SysFSTools::pci_create_mdev_device($pciid, $uuid, $d->{mdev});
5376                 } else {
5377                     die "can't unbind/bind pci group to vfio '$pciid'\n"
5378                         if !PVE::SysFSTools::pci_dev_group_bind_to_vfio($pciid);
5379                     die "can't reset pci device '$pciid'\n"
5380                         if $info->{has_fl_reset} and !PVE::SysFSTools::pci_dev_reset($info);
5381                 }
5382           }
5383         }
5384
5385         PVE::Storage::activate_volumes($storecfg, $vollist);
5386
5387         eval {
5388             run_command(['/bin/systemctl', 'stop', "$vmid.scope"],
5389                 outfunc => sub {}, errfunc => sub {});
5390         };
5391         # Issues with the above 'stop' not being fully completed are extremely rare, a very low
5392         # timeout should be more than enough here...
5393         PVE::Systemd::wait_for_unit_removed("$vmid.scope", 5);
5394
5395         my $cpuunits = defined($conf->{cpuunits}) ? $conf->{cpuunits}
5396                                                   : $defaults->{cpuunits};
5397
5398         my $start_timeout = ($conf->{hugepages} || $is_suspended) ? 300 : 30;
5399         my %run_params = (timeout => $statefile ? undef : $start_timeout, umask => 0077);
5400
5401         my %properties = (
5402             Slice => 'qemu.slice',
5403             KillMode => 'none',
5404             CPUShares => $cpuunits
5405         );
5406
5407         if (my $cpulimit = $conf->{cpulimit}) {
5408             $properties{CPUQuota} = int($cpulimit * 100);
5409         }
5410         $properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
5411
5412         my $run_qemu = sub {
5413             PVE::Tools::run_fork sub {
5414                 PVE::Systemd::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
5415                 run_command($cmd, %run_params);
5416             };
5417         };
5418
5419         if ($conf->{hugepages}) {
5420
5421             my $code = sub {
5422                 my $hugepages_topology = PVE::QemuServer::Memory::hugepages_topology($conf);
5423                 my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
5424
5425                 PVE::QemuServer::Memory::hugepages_mount();
5426                 PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
5427
5428                 eval { $run_qemu->() };
5429                 if (my $err = $@) {
5430                     PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology);
5431                     die $err;
5432                 }
5433
5434                 PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology);
5435             };
5436             eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
5437
5438         } else {
5439             eval { $run_qemu->() };
5440         }
5441
5442         if (my $err = $@) {
5443             # deactivate volumes if start fails
5444             eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
5445             die "start failed: $err";
5446         }
5447
5448         print "migration listens on $migrate_uri\n" if $migrate_uri;
5449
5450         if ($statefile && $statefile ne 'tcp' && $statefile ne 'unix')  {
5451             eval { mon_cmd($vmid, "cont"); };
5452             warn $@ if $@;
5453         }
5454
5455         #start nbd server for storage migration
5456         if ($targetstorage) {
5457             my $nodename = PVE::INotify::nodename();
5458             my $localip = $get_migration_ip->($migration_network, $nodename);
5459             my $pfamily = PVE::Tools::get_host_address_family($nodename);
5460             my $storage_migrate_port = PVE::Tools::next_migrate_port($pfamily);
5461
5462             mon_cmd($vmid, "nbd-server-start", addr => { type => 'inet', data => { host => "${localip}", port => "${storage_migrate_port}" } } );
5463
5464             $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5465
5466             foreach my $opt (sort keys %$local_volumes) {
5467                 my $volid = $local_volumes->{$opt};
5468                 mon_cmd($vmid, "nbd-server-add", device => "drive-$opt", writable => JSON::true );
5469                 my $migrate_storage_uri = "nbd:${localip}:${storage_migrate_port}:exportname=drive-$opt";
5470                 print "storage migration listens on $migrate_storage_uri volume:$volid\n";
5471             }
5472         }
5473
5474         if ($migratedfrom) {
5475             eval {
5476                 set_migration_caps($vmid);
5477             };
5478             warn $@ if $@;
5479
5480             if ($spice_port) {
5481                 print "spice listens on port $spice_port\n";
5482                 if ($spice_ticket) {
5483                     mon_cmd($vmid, "set_password", protocol => 'spice', password => $spice_ticket);
5484                     mon_cmd($vmid, "expire_password", protocol => 'spice', time => "+30");
5485                 }
5486             }
5487
5488         } else {
5489             mon_cmd($vmid, "balloon", value => $conf->{balloon}*1024*1024)
5490                 if !$statefile && $conf->{balloon};
5491
5492             foreach my $opt (keys %$conf) {
5493                 next if $opt !~  m/^net\d+$/;
5494                 my $nicconf = parse_net($conf->{$opt});
5495                 qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
5496             }
5497         }
5498
5499         mon_cmd($vmid, 'qom-set',
5500                     path => "machine/peripheral/balloon0",
5501                     property => "guest-stats-polling-interval",
5502                     value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
5503
5504         if ($is_suspended) {
5505             print "Resumed VM, removing state\n";
5506             if (my $vmstate = $conf->{vmstate}) {
5507                 PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
5508                 PVE::Storage::vdisk_free($storecfg, $vmstate);
5509             }
5510             delete $conf->@{qw(lock vmstate runningmachine)};
5511             PVE::QemuConfig->write_config($vmid, $conf);
5512         }
5513
5514         PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'post-start');
5515     });
5516 }
5517
5518 sub vm_commandline {
5519     my ($storecfg, $vmid, $snapname) = @_;
5520
5521     my $conf = PVE::QemuConfig->load_config($vmid);
5522     my $forcemachine;
5523
5524     if ($snapname) {
5525         my $snapshot = $conf->{snapshots}->{$snapname};
5526         die "snapshot '$snapname' does not exist\n" if !defined($snapshot);
5527
5528         # check for a 'runningmachine' in snapshot
5529         $forcemachine = $snapshot->{runningmachine} if $snapshot->{runningmachine};
5530
5531         $snapshot->{digest} = $conf->{digest}; # keep file digest for API
5532
5533         $conf = $snapshot;
5534     }
5535
5536     my $defaults = load_defaults();
5537
5538     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine);
5539
5540     return PVE::Tools::cmd2string($cmd);
5541 }
5542
5543 sub vm_reset {
5544     my ($vmid, $skiplock) = @_;
5545
5546     PVE::QemuConfig->lock_config($vmid, sub {
5547
5548         my $conf = PVE::QemuConfig->load_config($vmid);
5549
5550         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5551
5552         mon_cmd($vmid, "system_reset");
5553     });
5554 }
5555
5556 sub get_vm_volumes {
5557     my ($conf) = @_;
5558
5559     my $vollist = [];
5560     foreach_volid($conf, sub {
5561         my ($volid, $attr) = @_;
5562
5563         return if $volid =~ m|^/|;
5564
5565         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5566         return if !$sid;
5567
5568         push @$vollist, $volid;
5569     });
5570
5571     return $vollist;
5572 }
5573
5574 sub vm_stop_cleanup {
5575     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
5576
5577     eval {
5578
5579         if (!$keepActive) {
5580             my $vollist = get_vm_volumes($conf);
5581             PVE::Storage::deactivate_volumes($storecfg, $vollist);
5582         }
5583
5584         foreach my $ext (qw(mon qmp pid vnc qga)) {
5585             unlink "/var/run/qemu-server/${vmid}.$ext";
5586         }
5587
5588         if ($conf->{ivshmem}) {
5589             my $ivshmem = PVE::JSONSchema::parse_property_string($ivshmem_fmt, $conf->{ivshmem});
5590             # just delete it for now, VMs which have this already open do not
5591             # are affected, but new VMs will get a separated one. If this
5592             # becomes an issue we either add some sort of ref-counting or just
5593             # add a "don't delete on stop" flag to the ivshmem format.
5594             unlink '/dev/shm/pve-shm-' . ($ivshmem->{name} // $vmid);
5595         }
5596
5597         foreach my $key (keys %$conf) {
5598             next if $key !~ m/^hostpci(\d+)$/;
5599             my $hostpciindex = $1;
5600             my $d = parse_hostpci($conf->{$key});
5601             my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $hostpciindex);
5602
5603             foreach my $pci (@{$d->{pciid}}) {
5604                 my $pciid = $pci->{id};
5605                 PVE::SysFSTools::pci_cleanup_mdev_device($pciid, $uuid);
5606             }
5607         }
5608
5609         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
5610     };
5611     warn $@ if $@; # avoid errors - just warn
5612 }
5613
5614 # call only in locked context
5615 sub _do_vm_stop {
5616     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive) = @_;
5617
5618     my $pid = check_running($vmid, $nocheck);
5619     return if !$pid;
5620
5621     my $conf;
5622     if (!$nocheck) {
5623         $conf = PVE::QemuConfig->load_config($vmid);
5624         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5625         if (!defined($timeout) && $shutdown && $conf->{startup}) {
5626             my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
5627             $timeout = $opts->{down} if $opts->{down};
5628         }
5629         PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-stop');
5630     }
5631
5632     eval {
5633         if ($shutdown) {
5634             if (defined($conf) && parse_guest_agent($conf)->{enabled}) {
5635                 mon_cmd($vmid, "guest-shutdown", timeout => $timeout);
5636             } else {
5637                 mon_cmd($vmid, "system_powerdown");
5638             }
5639         } else {
5640             mon_cmd($vmid, "quit");
5641         }
5642     };
5643     my $err = $@;
5644
5645     if (!$err) {
5646         $timeout = 60 if !defined($timeout);
5647
5648         my $count = 0;
5649         while (($count < $timeout) && check_running($vmid, $nocheck)) {
5650             $count++;
5651             sleep 1;
5652         }
5653
5654         if ($count >= $timeout) {
5655             if ($force) {
5656                 warn "VM still running - terminating now with SIGTERM\n";
5657                 kill 15, $pid;
5658             } else {
5659                 die "VM quit/powerdown failed - got timeout\n";
5660             }
5661         } else {
5662             vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5663             return;
5664         }
5665     } else {
5666         if ($force) {
5667             warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
5668             kill 15, $pid;
5669         } else {
5670             die "VM quit/powerdown failed\n";
5671         }
5672     }
5673
5674     # wait again
5675     $timeout = 10;
5676
5677     my $count = 0;
5678     while (($count < $timeout) && check_running($vmid, $nocheck)) {
5679         $count++;
5680         sleep 1;
5681     }
5682
5683     if ($count >= $timeout) {
5684         warn "VM still running - terminating now with SIGKILL\n";
5685         kill 9, $pid;
5686         sleep 1;
5687     }
5688
5689     vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5690 }
5691
5692 # Note: use $nocheck to skip tests if VM configuration file exists.
5693 # We need that when migration VMs to other nodes (files already moved)
5694 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
5695 sub vm_stop {
5696     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
5697
5698     $force = 1 if !defined($force) && !$shutdown;
5699
5700     if ($migratedfrom){
5701         my $pid = check_running($vmid, $nocheck, $migratedfrom);
5702         kill 15, $pid if $pid;
5703         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
5704         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
5705         return;
5706     }
5707
5708     PVE::QemuConfig->lock_config($vmid, sub {
5709         _do_vm_stop($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive);
5710    });
5711 }
5712
5713 sub vm_reboot {
5714     my ($vmid, $timeout) = @_;
5715
5716     PVE::QemuConfig->lock_config($vmid, sub {
5717         eval {
5718
5719             # only reboot if running, as qmeventd starts it again on a stop event
5720             return if !check_running($vmid);
5721
5722             create_reboot_request($vmid);
5723
5724             my $storecfg = PVE::Storage::config();
5725             _do_vm_stop($storecfg, $vmid, undef, undef, $timeout, 1);
5726
5727         };
5728         if (my $err = $@) {
5729             # avoid that the next normal shutdown will be confused for a reboot
5730             clear_reboot_request($vmid);
5731             die $err;
5732         }
5733    });
5734 }
5735
5736 sub vm_suspend {
5737     my ($vmid, $skiplock, $includestate, $statestorage) = @_;
5738
5739     my $conf;
5740     my $path;
5741     my $storecfg;
5742     my $vmstate;
5743
5744     PVE::QemuConfig->lock_config($vmid, sub {
5745
5746         $conf = PVE::QemuConfig->load_config($vmid);
5747
5748         my $is_backing_up = PVE::QemuConfig->has_lock($conf, 'backup');
5749         PVE::QemuConfig->check_lock($conf)
5750             if !($skiplock || $is_backing_up);
5751
5752         die "cannot suspend to disk during backup\n"
5753             if $is_backing_up && $includestate;
5754
5755         if ($includestate) {
5756             $conf->{lock} = 'suspending';
5757             my $date = strftime("%Y-%m-%d", localtime(time()));
5758             $storecfg = PVE::Storage::config();
5759             $vmstate = PVE::QemuConfig->__snapshot_save_vmstate($vmid, $conf, "suspend-$date", $storecfg, $statestorage, 1);
5760             $path = PVE::Storage::path($storecfg, $vmstate);
5761             PVE::QemuConfig->write_config($vmid, $conf);
5762         } else {
5763             mon_cmd($vmid, "stop");
5764         }
5765     });
5766
5767     if ($includestate) {
5768         # save vm state
5769         PVE::Storage::activate_volumes($storecfg, [$vmstate]);
5770
5771         eval {
5772             mon_cmd($vmid, "savevm-start", statefile => $path);
5773             for(;;) {
5774                 my $state = mon_cmd($vmid, "query-savevm");
5775                 if (!$state->{status}) {
5776                     die "savevm not active\n";
5777                 } elsif ($state->{status} eq 'active') {
5778                     sleep(1);
5779                     next;
5780                 } elsif ($state->{status} eq 'completed') {
5781                     print "State saved, quitting\n";
5782                     last;
5783                 } elsif ($state->{status} eq 'failed' && $state->{error}) {
5784                     die "query-savevm failed with error '$state->{error}'\n"
5785                 } else {
5786                     die "query-savevm returned status '$state->{status}'\n";
5787                 }
5788             }
5789         };
5790         my $err = $@;
5791
5792         PVE::QemuConfig->lock_config($vmid, sub {
5793             $conf = PVE::QemuConfig->load_config($vmid);
5794             if ($err) {
5795                 # cleanup, but leave suspending lock, to indicate something went wrong
5796                 eval {
5797                     mon_cmd($vmid, "savevm-end");
5798                     PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
5799                     PVE::Storage::vdisk_free($storecfg, $vmstate);
5800                     delete $conf->@{qw(vmstate runningmachine)};
5801                     PVE::QemuConfig->write_config($vmid, $conf);
5802                 };
5803                 warn $@ if $@;
5804                 die $err;
5805             }
5806
5807             die "lock changed unexpectedly\n"
5808                 if !PVE::QemuConfig->has_lock($conf, 'suspending');
5809
5810             mon_cmd($vmid, "quit");
5811             $conf->{lock} = 'suspended';
5812             PVE::QemuConfig->write_config($vmid, $conf);
5813         });
5814     }
5815 }
5816
5817 sub vm_resume {
5818     my ($vmid, $skiplock, $nocheck) = @_;
5819
5820     PVE::QemuConfig->lock_config($vmid, sub {
5821         my $res = mon_cmd($vmid, 'query-status');
5822         my $resume_cmd = 'cont';
5823
5824         if ($res->{status} && $res->{status} eq 'suspended') {
5825             $resume_cmd = 'system_wakeup';
5826         }
5827
5828         if (!$nocheck) {
5829
5830             my $conf = PVE::QemuConfig->load_config($vmid);
5831
5832             PVE::QemuConfig->check_lock($conf)
5833                 if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
5834         }
5835
5836         mon_cmd($vmid, $resume_cmd);
5837     });
5838 }
5839
5840 sub vm_sendkey {
5841     my ($vmid, $skiplock, $key) = @_;
5842
5843     PVE::QemuConfig->lock_config($vmid, sub {
5844
5845         my $conf = PVE::QemuConfig->load_config($vmid);
5846
5847         # there is no qmp command, so we use the human monitor command
5848         my $res = PVE::QemuServer::Monitor::hmp_cmd($vmid, "sendkey $key");
5849         die $res if $res ne '';
5850     });
5851 }
5852
5853 # vzdump restore implementaion
5854
5855 sub tar_archive_read_firstfile {
5856     my $archive = shift;
5857
5858     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
5859
5860     # try to detect archive type first
5861     my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
5862         die "unable to open file '$archive'\n";
5863     my $firstfile = <$fh>;
5864     kill 15, $pid;
5865     close $fh;
5866
5867     die "ERROR: archive contaions no data\n" if !$firstfile;
5868     chomp $firstfile;
5869
5870     return $firstfile;
5871 }
5872
5873 sub tar_restore_cleanup {
5874     my ($storecfg, $statfile) = @_;
5875
5876     print STDERR "starting cleanup\n";
5877
5878     if (my $fd = IO::File->new($statfile, "r")) {
5879         while (defined(my $line = <$fd>)) {
5880             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5881                 my $volid = $2;
5882                 eval {
5883                     if ($volid =~ m|^/|) {
5884                         unlink $volid || die 'unlink failed\n';
5885                     } else {
5886                         PVE::Storage::vdisk_free($storecfg, $volid);
5887                     }
5888                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
5889                 };
5890                 print STDERR "unable to cleanup '$volid' - $@" if $@;
5891             } else {
5892                 print STDERR "unable to parse line in statfile - $line";
5893             }
5894         }
5895         $fd->close();
5896     }
5897 }
5898
5899 sub restore_archive {
5900     my ($archive, $vmid, $user, $opts) = @_;
5901
5902     my $format = $opts->{format};
5903     my $comp;
5904
5905     if ($archive =~ m/\.tgz$/ || $archive =~ m/\.tar\.gz$/) {
5906         $format = 'tar' if !$format;
5907         $comp = 'gzip';
5908     } elsif ($archive =~ m/\.tar$/) {
5909         $format = 'tar' if !$format;
5910     } elsif ($archive =~ m/.tar.lzo$/) {
5911         $format = 'tar' if !$format;
5912         $comp = 'lzop';
5913     } elsif ($archive =~ m/\.vma$/) {
5914         $format = 'vma' if !$format;
5915     } elsif ($archive =~ m/\.vma\.gz$/) {
5916         $format = 'vma' if !$format;
5917         $comp = 'gzip';
5918     } elsif ($archive =~ m/\.vma\.lzo$/) {
5919         $format = 'vma' if !$format;
5920         $comp = 'lzop';
5921     } else {
5922         $format = 'vma' if !$format; # default
5923     }
5924
5925     # try to detect archive format
5926     if ($format eq 'tar') {
5927         return restore_tar_archive($archive, $vmid, $user, $opts);
5928     } else {
5929         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
5930     }
5931 }
5932
5933 sub restore_update_config_line {
5934     my ($outfd, $cookie, $vmid, $map, $line, $unique) = @_;
5935
5936     return if $line =~ m/^\#qmdump\#/;
5937     return if $line =~ m/^\#vzdump\#/;
5938     return if $line =~ m/^lock:/;
5939     return if $line =~ m/^unused\d+:/;
5940     return if $line =~ m/^parent:/;
5941
5942     my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
5943     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
5944         # try to convert old 1.X settings
5945         my ($id, $ind, $ethcfg) = ($1, $2, $3);
5946         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
5947             my ($model, $macaddr) = split(/\=/, $devconfig);
5948             $macaddr = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if !$macaddr || $unique;
5949             my $net = {
5950                 model => $model,
5951                 bridge => "vmbr$ind",
5952                 macaddr => $macaddr,
5953             };
5954             my $netstr = print_net($net);
5955
5956             print $outfd "net$cookie->{netcount}: $netstr\n";
5957             $cookie->{netcount}++;
5958         }
5959     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
5960         my ($id, $netstr) = ($1, $2);
5961         my $net = parse_net($netstr);
5962         $net->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if $net->{macaddr};
5963         $netstr = print_net($net);
5964         print $outfd "$id: $netstr\n";
5965     } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk)\d+):\s*(\S+)\s*$/) {
5966         my $virtdev = $1;
5967         my $value = $3;
5968         my $di = parse_drive($virtdev, $value);
5969         if (defined($di->{backup}) && !$di->{backup}) {
5970             print $outfd "#$line";
5971         } elsif ($map->{$virtdev}) {
5972             delete $di->{format}; # format can change on restore
5973             $di->{file} = $map->{$virtdev};
5974             $value = print_drive($di);
5975             print $outfd "$virtdev: $value\n";
5976         } else {
5977             print $outfd $line;
5978         }
5979     } elsif (($line =~ m/^vmgenid: (.*)/)) {
5980         my $vmgenid = $1;
5981         if ($vmgenid ne '0') {
5982             # always generate a new vmgenid if there was a valid one setup
5983             $vmgenid = generate_uuid();
5984         }
5985         print $outfd "vmgenid: $vmgenid\n";
5986     } elsif (($line =~ m/^(smbios1: )(.*)/) && $unique) {
5987         my ($uuid, $uuid_str);
5988         UUID::generate($uuid);
5989         UUID::unparse($uuid, $uuid_str);
5990         my $smbios1 = parse_smbios1($2);
5991         $smbios1->{uuid} = $uuid_str;
5992         print $outfd $1.print_smbios1($smbios1)."\n";
5993     } else {
5994         print $outfd $line;
5995     }
5996 }
5997
5998 sub scan_volids {
5999     my ($cfg, $vmid) = @_;
6000
6001     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid);
6002
6003     my $volid_hash = {};
6004     foreach my $storeid (keys %$info) {
6005         foreach my $item (@{$info->{$storeid}}) {
6006             next if !($item->{volid} && $item->{size});
6007             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
6008             $volid_hash->{$item->{volid}} = $item;
6009         }
6010     }
6011
6012     return $volid_hash;
6013 }
6014
6015 sub is_volume_in_use {
6016     my ($storecfg, $conf, $skip_drive, $volid) = @_;
6017
6018     my $path = PVE::Storage::path($storecfg, $volid);
6019
6020     my $scan_config = sub {
6021         my ($cref, $snapname) = @_;
6022
6023         foreach my $key (keys %$cref) {
6024             my $value = $cref->{$key};
6025             if (is_valid_drivename($key)) {
6026                 next if $skip_drive && $key eq $skip_drive;
6027                 my $drive = parse_drive($key, $value);
6028                 next if !$drive || !$drive->{file} || drive_is_cdrom($drive);
6029                 return 1 if $volid eq $drive->{file};
6030                 if ($drive->{file} =~ m!^/!) {
6031                     return 1 if $drive->{file} eq $path;
6032                 } else {
6033                     my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file}, 1);
6034                     next if !$storeid;
6035                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid, 1);
6036                     next if !$scfg;
6037                     return 1 if $path eq PVE::Storage::path($storecfg, $drive->{file}, $snapname);
6038                 }
6039             }
6040         }
6041
6042         return 0;
6043     };
6044
6045     return 1 if &$scan_config($conf);
6046
6047     undef $skip_drive;
6048
6049     foreach my $snapname (keys %{$conf->{snapshots}}) {
6050         return 1 if &$scan_config($conf->{snapshots}->{$snapname}, $snapname);
6051     }
6052
6053     return 0;
6054 }
6055
6056 sub update_disksize {
6057     my ($vmid, $conf, $volid_hash) = @_;
6058
6059     my $changes;
6060     my $prefix = "VM $vmid:";
6061
6062     # used and unused disks
6063     my $referenced = {};
6064
6065     # Note: it is allowed to define multiple storages with same path (alias), so
6066     # we need to check both 'volid' and real 'path' (two different volid can point
6067     # to the same path).
6068
6069     my $referencedpath = {};
6070
6071     # update size info
6072     foreach my $opt (keys %$conf) {
6073         if (is_valid_drivename($opt)) {
6074             my $drive = parse_drive($opt, $conf->{$opt});
6075             my $volid = $drive->{file};
6076             next if !$volid;
6077
6078             $referenced->{$volid} = 1;
6079             if ($volid_hash->{$volid} &&
6080                 (my $path = $volid_hash->{$volid}->{path})) {
6081                 $referencedpath->{$path} = 1;
6082             }
6083
6084             next if drive_is_cdrom($drive);
6085             next if !$volid_hash->{$volid};
6086
6087             $drive->{size} = $volid_hash->{$volid}->{size};
6088             my $new = print_drive($drive);
6089             if ($new ne $conf->{$opt}) {
6090                 $changes = 1;
6091                 $conf->{$opt} = $new;
6092                 print "$prefix update disk '$opt' information.\n";
6093             }
6094         }
6095     }
6096
6097     # remove 'unusedX' entry if volume is used
6098     foreach my $opt (keys %$conf) {
6099         next if $opt !~ m/^unused\d+$/;
6100         my $volid = $conf->{$opt};
6101         my $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
6102         if ($referenced->{$volid} || ($path && $referencedpath->{$path})) {
6103             print "$prefix remove entry '$opt', its volume '$volid' is in use.\n";
6104             $changes = 1;
6105             delete $conf->{$opt};
6106         }
6107
6108         $referenced->{$volid} = 1;
6109         $referencedpath->{$path} = 1 if $path;
6110     }
6111
6112     foreach my $volid (sort keys %$volid_hash) {
6113         next if $volid =~ m/vm-$vmid-state-/;
6114         next if $referenced->{$volid};
6115         my $path = $volid_hash->{$volid}->{path};
6116         next if !$path; # just to be sure
6117         next if $referencedpath->{$path};
6118         $changes = 1;
6119         my $key = PVE::QemuConfig->add_unused_volume($conf, $volid);
6120         print "$prefix add unreferenced volume '$volid' as '$key' to config.\n";
6121         $referencedpath->{$path} = 1; # avoid to add more than once (aliases)
6122     }
6123
6124     return $changes;
6125 }
6126
6127 sub rescan {
6128     my ($vmid, $nolock, $dryrun) = @_;
6129
6130     my $cfg = PVE::Storage::config();
6131
6132     # FIXME: Remove once our RBD plugin can handle CT and VM on a single storage
6133     # see: https://pve.proxmox.com/pipermail/pve-devel/2018-July/032900.html
6134     foreach my $stor (keys %{$cfg->{ids}}) {
6135         delete($cfg->{ids}->{$stor}) if ! $cfg->{ids}->{$stor}->{content}->{images};
6136     }
6137
6138     print "rescan volumes...\n";
6139     my $volid_hash = scan_volids($cfg, $vmid);
6140
6141     my $updatefn =  sub {
6142         my ($vmid) = @_;
6143
6144         my $conf = PVE::QemuConfig->load_config($vmid);
6145
6146         PVE::QemuConfig->check_lock($conf);
6147
6148         my $vm_volids = {};
6149         foreach my $volid (keys %$volid_hash) {
6150             my $info = $volid_hash->{$volid};
6151             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
6152         }
6153
6154         my $changes = update_disksize($vmid, $conf, $vm_volids);
6155
6156         PVE::QemuConfig->write_config($vmid, $conf) if $changes && !$dryrun;
6157     };
6158
6159     if (defined($vmid)) {
6160         if ($nolock) {
6161             &$updatefn($vmid);
6162         } else {
6163             PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6164         }
6165     } else {
6166         my $vmlist = config_list();
6167         foreach my $vmid (keys %$vmlist) {
6168             if ($nolock) {
6169                 &$updatefn($vmid);
6170             } else {
6171                 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6172             }
6173         }
6174     }
6175 }
6176
6177 sub restore_vma_archive {
6178     my ($archive, $vmid, $user, $opts, $comp) = @_;
6179
6180     my $readfrom = $archive;
6181
6182     my $cfg = PVE::Storage::config();
6183     my $commands = [];
6184     my $bwlimit = $opts->{bwlimit};
6185
6186     my $dbg_cmdstring = '';
6187     my $add_pipe = sub {
6188         my ($cmd) = @_;
6189         push @$commands, $cmd;
6190         $dbg_cmdstring .= ' | ' if length($dbg_cmdstring);
6191         $dbg_cmdstring .= PVE::Tools::cmd2string($cmd);
6192         $readfrom = '-';
6193     };
6194
6195     my $input = undef;
6196     if ($archive eq '-') {
6197         $input = '<&STDIN';
6198     } else {
6199         # If we use a backup from a PVE defined storage we also consider that
6200         # storage's rate limit:
6201         my (undef, $volid) = PVE::Storage::path_to_volume_id($cfg, $archive);
6202         if (defined($volid)) {
6203             my ($sid, undef) = PVE::Storage::parse_volume_id($volid);
6204             my $readlimit = PVE::Storage::get_bandwidth_limit('restore', [$sid], $bwlimit);
6205             if ($readlimit) {
6206                 print STDERR "applying read rate limit: $readlimit\n";
6207                 my $cstream = ['cstream', '-t', $readlimit*1024, '--', $readfrom];
6208                 $add_pipe->($cstream);
6209             }
6210         }
6211     }
6212
6213     if ($comp) {
6214         my $cmd;
6215         if ($comp eq 'gzip') {
6216             $cmd = ['zcat', $readfrom];
6217         } elsif ($comp eq 'lzop') {
6218             $cmd = ['lzop', '-d', '-c', $readfrom];
6219         } else {
6220             die "unknown compression method '$comp'\n";
6221         }
6222         $add_pipe->($cmd);
6223     }
6224
6225     my $tmpdir = "/var/tmp/vzdumptmp$$";
6226     rmtree $tmpdir;
6227
6228     # disable interrupts (always do cleanups)
6229     local $SIG{INT} =
6230         local $SIG{TERM} =
6231         local $SIG{QUIT} =
6232         local $SIG{HUP} = sub { warn "got interrupt - ignored\n"; };
6233
6234     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
6235     POSIX::mkfifo($mapfifo, 0600);
6236     my $fifofh;
6237
6238     my $openfifo = sub {
6239         open($fifofh, '>', $mapfifo) || die $!;
6240     };
6241
6242     $add_pipe->(['vma', 'extract', '-v', '-r', $mapfifo, $readfrom, $tmpdir]);
6243
6244     my $oldtimeout;
6245     my $timeout = 5;
6246
6247     my $devinfo = {};
6248
6249     my $rpcenv = PVE::RPCEnvironment::get();
6250
6251     my $conffile = PVE::QemuConfig->config_file($vmid);
6252     my $tmpfn = "$conffile.$$.tmp";
6253
6254     # Note: $oldconf is undef if VM does not exists
6255     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
6256     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
6257
6258     my %storage_limits;
6259
6260     my $print_devmap = sub {
6261         my $virtdev_hash = {};
6262
6263         my $cfgfn = "$tmpdir/qemu-server.conf";
6264
6265         # we can read the config - that is already extracted
6266         my $fh = IO::File->new($cfgfn, "r") ||
6267             "unable to read qemu-server.conf - $!\n";
6268
6269         my $fwcfgfn = "$tmpdir/qemu-server.fw";
6270         if (-f $fwcfgfn) {
6271             my $pve_firewall_dir = '/etc/pve/firewall';
6272             mkdir $pve_firewall_dir; # make sure the dir exists
6273             PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
6274         }
6275
6276         while (defined(my $line = <$fh>)) {
6277             if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
6278                 my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
6279                 die "archive does not contain data for drive '$virtdev'\n"
6280                     if !$devinfo->{$devname};
6281                 if (defined($opts->{storage})) {
6282                     $storeid = $opts->{storage} || 'local';
6283                 } elsif (!$storeid) {
6284                     $storeid = 'local';
6285                 }
6286                 $format = 'raw' if !$format;
6287                 $devinfo->{$devname}->{devname} = $devname;
6288                 $devinfo->{$devname}->{virtdev} = $virtdev;
6289                 $devinfo->{$devname}->{format} = $format;
6290                 $devinfo->{$devname}->{storeid} = $storeid;
6291
6292                 # check permission on storage
6293                 my $pool = $opts->{pool}; # todo: do we need that?
6294                 if ($user ne 'root@pam') {
6295                     $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
6296                 }
6297
6298                 $storage_limits{$storeid} = $bwlimit;
6299
6300                 $virtdev_hash->{$virtdev} = $devinfo->{$devname};
6301             } elsif ($line =~ m/^((?:ide|sata|scsi)\d+):\s*(.*)\s*$/) {
6302                 my $virtdev = $1;
6303                 my $drive = parse_drive($virtdev, $2);
6304                 if (drive_is_cloudinit($drive)) {
6305                     my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6306                     my $scfg = PVE::Storage::storage_config($cfg, $storeid);
6307                     my $format = qemu_img_format($scfg, $volname); # has 'raw' fallback
6308
6309                     my $d = {
6310                         format => $format,
6311                         storeid => $opts->{storage} // $storeid,
6312                         size => PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE,
6313                         file => $drive->{file}, # to make drive_is_cloudinit check possible
6314                         name => "vm-$vmid-cloudinit",
6315                         is_cloudinit => 1,
6316                     };
6317                     $virtdev_hash->{$virtdev} = $d;
6318                 }
6319             }
6320         }
6321
6322         foreach my $key (keys %storage_limits) {
6323             my $limit = PVE::Storage::get_bandwidth_limit('restore', [$key], $bwlimit);
6324             next if !$limit;
6325             print STDERR "rate limit for storage $key: $limit KiB/s\n";
6326             $storage_limits{$key} = $limit * 1024;
6327         }
6328
6329         foreach my $devname (keys %$devinfo) {
6330             die "found no device mapping information for device '$devname'\n"
6331                 if !$devinfo->{$devname}->{virtdev};
6332         }
6333
6334         # create empty/temp config
6335         if ($oldconf) {
6336             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
6337             foreach_drive($oldconf, sub {
6338                 my ($ds, $drive) = @_;
6339
6340                 return if drive_is_cdrom($drive, 1);
6341
6342                 my $volid = $drive->{file};
6343                 return if !$volid || $volid =~ m|^/|;
6344
6345                 my ($path, $owner) = PVE::Storage::path($cfg, $volid);
6346                 return if !$path || !$owner || ($owner != $vmid);
6347
6348                 # Note: only delete disk we want to restore
6349                 # other volumes will become unused
6350                 if ($virtdev_hash->{$ds}) {
6351                     eval { PVE::Storage::vdisk_free($cfg, $volid); };
6352                     if (my $err = $@) {
6353                         warn $err;
6354                     }
6355                 }
6356             });
6357
6358             # delete vmstate files, after the restore we have no snapshots anymore
6359             foreach my $snapname (keys %{$oldconf->{snapshots}}) {
6360                 my $snap = $oldconf->{snapshots}->{$snapname};
6361                 if ($snap->{vmstate}) {
6362                     eval { PVE::Storage::vdisk_free($cfg, $snap->{vmstate}); };
6363                     if (my $err = $@) {
6364                         warn $err;
6365                     }
6366                 }
6367             }
6368         }
6369
6370         my $map = {};
6371         foreach my $virtdev (sort keys %$virtdev_hash) {
6372             my $d = $virtdev_hash->{$virtdev};
6373             my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
6374             my $storeid = $d->{storeid};
6375             my $scfg = PVE::Storage::storage_config($cfg, $storeid);
6376
6377             my $map_opts = '';
6378             if (my $limit = $storage_limits{$storeid}) {
6379                 $map_opts .= "throttling.bps=$limit:throttling.group=$storeid:";
6380             }
6381
6382             # test if requested format is supported
6383             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($cfg, $storeid);
6384             my $supported = grep { $_ eq $d->{format} } @$validFormats;
6385             $d->{format} = $defFormat if !$supported;
6386
6387             my $name;
6388             if ($d->{is_cloudinit}) {
6389                 $name = $d->{name};
6390                 $name .= ".$d->{format}" if $d->{format} ne 'raw';
6391             }
6392
6393             my $volid = PVE::Storage::vdisk_alloc($cfg, $storeid, $vmid, $d->{format}, $name, $alloc_size);
6394             print STDERR "new volume ID is '$volid'\n";
6395             $d->{volid} = $volid;
6396
6397             PVE::Storage::activate_volumes($cfg, [$volid]);
6398
6399             my $write_zeros = 1;
6400             if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
6401                 $write_zeros = 0;
6402             }
6403
6404             if (!$d->{is_cloudinit}) {
6405                 my $path = PVE::Storage::path($cfg, $volid);
6406
6407                 print $fifofh "${map_opts}format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
6408
6409                 print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
6410             }
6411             $map->{$virtdev} = $volid;
6412         }
6413
6414         $fh->seek(0, 0) || die "seek failed - $!\n";
6415
6416         my $outfd = new IO::File ($tmpfn, "w") ||
6417             die "unable to write config for VM $vmid\n";
6418
6419         my $cookie = { netcount => 0 };
6420         while (defined(my $line = <$fh>)) {
6421             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
6422         }
6423
6424         $fh->close();
6425         $outfd->close();
6426     };
6427
6428     eval {
6429         # enable interrupts
6430         local $SIG{INT} =
6431             local $SIG{TERM} =
6432             local $SIG{QUIT} =
6433             local $SIG{HUP} =
6434             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6435         local $SIG{ALRM} = sub { die "got timeout\n"; };
6436
6437         $oldtimeout = alarm($timeout);
6438
6439         my $parser = sub {
6440             my $line = shift;
6441
6442             print "$line\n";
6443
6444             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
6445                 my ($dev_id, $size, $devname) = ($1, $2, $3);
6446                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
6447             } elsif ($line =~ m/^CTIME: /) {
6448                 # we correctly received the vma config, so we can disable
6449                 # the timeout now for disk allocation (set to 10 minutes, so
6450                 # that we always timeout if something goes wrong)
6451                 alarm(600);
6452                 &$print_devmap();
6453                 print $fifofh "done\n";
6454                 my $tmp = $oldtimeout || 0;
6455                 $oldtimeout = undef;
6456                 alarm($tmp);
6457                 close($fifofh);
6458             }
6459         };
6460
6461         print "restore vma archive: $dbg_cmdstring\n";
6462         run_command($commands, input => $input, outfunc => $parser, afterfork => $openfifo);
6463     };
6464     my $err = $@;
6465
6466     alarm($oldtimeout) if $oldtimeout;
6467
6468     my $vollist = [];
6469     foreach my $devname (keys %$devinfo) {
6470         my $volid = $devinfo->{$devname}->{volid};
6471         push @$vollist, $volid if $volid;
6472     }
6473
6474     PVE::Storage::deactivate_volumes($cfg, $vollist);
6475
6476     unlink $mapfifo;
6477
6478     if ($err) {
6479         rmtree $tmpdir;
6480         unlink $tmpfn;
6481
6482         foreach my $devname (keys %$devinfo) {
6483             my $volid = $devinfo->{$devname}->{volid};
6484             next if !$volid;
6485             eval {
6486                 if ($volid =~ m|^/|) {
6487                     unlink $volid || die 'unlink failed\n';
6488                 } else {
6489                     PVE::Storage::vdisk_free($cfg, $volid);
6490                 }
6491                 print STDERR "temporary volume '$volid' sucessfuly removed\n";
6492             };
6493             print STDERR "unable to cleanup '$volid' - $@" if $@;
6494         }
6495         die $err;
6496     }
6497
6498     rmtree $tmpdir;
6499
6500     rename($tmpfn, $conffile) ||
6501         die "unable to commit configuration file '$conffile'\n";
6502
6503     PVE::Cluster::cfs_update(); # make sure we read new file
6504
6505     eval { rescan($vmid, 1); };
6506     warn $@ if $@;
6507 }
6508
6509 sub restore_tar_archive {
6510     my ($archive, $vmid, $user, $opts) = @_;
6511
6512     if ($archive ne '-') {
6513         my $firstfile = tar_archive_read_firstfile($archive);
6514         die "ERROR: file '$archive' dos not lock like a QemuServer vzdump backup\n"
6515             if $firstfile ne 'qemu-server.conf';
6516     }
6517
6518     my $storecfg = PVE::Storage::config();
6519
6520     # avoid zombie disks when restoring over an existing VM -> cleanup first
6521     # pass keep_empty_config=1 to keep the config (thus VMID) reserved for us
6522     # skiplock=1 because qmrestore has set the 'create' lock itself already
6523     my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
6524     destroy_vm($storecfg, $vmid, 1, { lock => 'restore' }) if -f $vmcfgfn;
6525
6526     my $tocmd = "/usr/lib/qemu-server/qmextract";
6527
6528     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
6529     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
6530     $tocmd .= ' --prealloc' if $opts->{prealloc};
6531     $tocmd .= ' --info' if $opts->{info};
6532
6533     # tar option "xf" does not autodetect compression when read from STDIN,
6534     # so we pipe to zcat
6535     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
6536         PVE::Tools::shellquote("--to-command=$tocmd");
6537
6538     my $tmpdir = "/var/tmp/vzdumptmp$$";
6539     mkpath $tmpdir;
6540
6541     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
6542     local $ENV{VZDUMP_VMID} = $vmid;
6543     local $ENV{VZDUMP_USER} = $user;
6544
6545     my $conffile = PVE::QemuConfig->config_file($vmid);
6546     my $tmpfn = "$conffile.$$.tmp";
6547
6548     # disable interrupts (always do cleanups)
6549     local $SIG{INT} =
6550         local $SIG{TERM} =
6551         local $SIG{QUIT} =
6552         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
6553
6554     eval {
6555         # enable interrupts
6556         local $SIG{INT} =
6557             local $SIG{TERM} =
6558             local $SIG{QUIT} =
6559             local $SIG{HUP} =
6560             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6561
6562         if ($archive eq '-') {
6563             print "extracting archive from STDIN\n";
6564             run_command($cmd, input => "<&STDIN");
6565         } else {
6566             print "extracting archive '$archive'\n";
6567             run_command($cmd);
6568         }
6569
6570         return if $opts->{info};
6571
6572         # read new mapping
6573         my $map = {};
6574         my $statfile = "$tmpdir/qmrestore.stat";
6575         if (my $fd = IO::File->new($statfile, "r")) {
6576             while (defined (my $line = <$fd>)) {
6577                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
6578                     $map->{$1} = $2 if $1;
6579                 } else {
6580                     print STDERR "unable to parse line in statfile - $line\n";
6581                 }
6582             }
6583             $fd->close();
6584         }
6585
6586         my $confsrc = "$tmpdir/qemu-server.conf";
6587
6588         my $srcfd = new IO::File($confsrc, "r") ||
6589             die "unable to open file '$confsrc'\n";
6590
6591         my $outfd = new IO::File ($tmpfn, "w") ||
6592             die "unable to write config for VM $vmid\n";
6593
6594         my $cookie = { netcount => 0 };
6595         while (defined (my $line = <$srcfd>)) {
6596             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
6597         }
6598
6599         $srcfd->close();
6600         $outfd->close();
6601     };
6602     if (my $err = $@) {
6603         unlink $tmpfn;
6604         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
6605         die $err;
6606     }
6607
6608     rmtree $tmpdir;
6609
6610     rename $tmpfn, $conffile ||
6611         die "unable to commit configuration file '$conffile'\n";
6612
6613     PVE::Cluster::cfs_update(); # make sure we read new file
6614
6615     eval { rescan($vmid, 1); };
6616     warn $@ if $@;
6617 };
6618
6619 sub foreach_storage_used_by_vm {
6620     my ($conf, $func) = @_;
6621
6622     my $sidhash = {};
6623
6624     foreach_drive($conf, sub {
6625         my ($ds, $drive) = @_;
6626         return if drive_is_cdrom($drive);
6627
6628         my $volid = $drive->{file};
6629
6630         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
6631         $sidhash->{$sid} = $sid if $sid;
6632     });
6633
6634     foreach my $sid (sort keys %$sidhash) {
6635         &$func($sid);
6636     }
6637 }
6638
6639 my $qemu_snap_storage = {
6640     rbd => 1,
6641 };
6642 sub do_snapshots_with_qemu {
6643     my ($storecfg, $volid) = @_;
6644
6645     my $storage_name = PVE::Storage::parse_volume_id($volid);
6646     my $scfg = $storecfg->{ids}->{$storage_name};
6647
6648     if ($qemu_snap_storage->{$scfg->{type}} && !$scfg->{krbd}){
6649         return 1;
6650     }
6651
6652     if ($volid =~ m/\.(qcow2|qed)$/){
6653         return 1;
6654     }
6655
6656     return undef;
6657 }
6658
6659 sub qga_check_running {
6660     my ($vmid, $nowarn) = @_;
6661
6662     eval { mon_cmd($vmid, "guest-ping", timeout => 3); };
6663     if ($@) {
6664         warn "Qemu Guest Agent is not running - $@" if !$nowarn;
6665         return 0;
6666     }
6667     return 1;
6668 }
6669
6670 sub template_create {
6671     my ($vmid, $conf, $disk) = @_;
6672
6673     my $storecfg = PVE::Storage::config();
6674
6675     foreach_drive($conf, sub {
6676         my ($ds, $drive) = @_;
6677
6678         return if drive_is_cdrom($drive);
6679         return if $disk && $ds ne $disk;
6680
6681         my $volid = $drive->{file};
6682         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
6683
6684         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
6685         $drive->{file} = $voliddst;
6686         $conf->{$ds} = print_drive($drive);
6687         PVE::QemuConfig->write_config($vmid, $conf);
6688     });
6689 }
6690
6691 sub convert_iscsi_path {
6692     my ($path) = @_;
6693
6694     if ($path =~ m|^iscsi://([^/]+)/([^/]+)/(.+)$|) {
6695         my $portal = $1;
6696         my $target = $2;
6697         my $lun = $3;
6698
6699         my $initiator_name = get_initiator_name();
6700
6701         return "file.driver=iscsi,file.transport=tcp,file.initiator-name=$initiator_name,".
6702                "file.portal=$portal,file.target=$target,file.lun=$lun,driver=raw";
6703     }
6704
6705     die "cannot convert iscsi path '$path', unkown format\n";
6706 }
6707
6708 sub qemu_img_convert {
6709     my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized) = @_;
6710
6711     my $storecfg = PVE::Storage::config();
6712     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
6713     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
6714
6715     die "destination '$dst_volid' is not a valid volid form qemu-img convert\n" if !$dst_storeid;
6716
6717     my $cachemode;
6718     my $src_path;
6719     my $src_is_iscsi = 0;
6720     my $src_format;
6721
6722     if ($src_storeid) {
6723         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
6724         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
6725         $src_format = qemu_img_format($src_scfg, $src_volname);
6726         $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
6727         $src_is_iscsi = ($src_path =~ m|^iscsi://|);
6728         $cachemode = 'none' if $src_scfg->{type} eq 'zfspool';
6729     } elsif (-f $src_volid) {
6730         $src_path = $src_volid;
6731         if ($src_path =~ m/\.($QEMU_FORMAT_RE)$/) {
6732             $src_format = $1;
6733         }
6734     }
6735
6736     die "source '$src_volid' is not a valid volid nor path for qemu-img convert\n" if !$src_path;
6737
6738     my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6739     my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
6740     my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6741     my $dst_is_iscsi = ($dst_path =~ m|^iscsi://|);
6742
6743     my $cmd = [];
6744     push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
6745     push @$cmd, '-l', "snapshot.name=$snapname"
6746         if $snapname && $src_format && $src_format eq "qcow2";
6747     push @$cmd, '-t', 'none' if $dst_scfg->{type} eq 'zfspool';
6748     push @$cmd, '-T', $cachemode if defined($cachemode);
6749
6750     if ($src_is_iscsi) {
6751         push @$cmd, '--image-opts';
6752         $src_path = convert_iscsi_path($src_path);
6753     } elsif ($src_format) {
6754         push @$cmd, '-f', $src_format;
6755     }
6756
6757     if ($dst_is_iscsi) {
6758         push @$cmd, '--target-image-opts';
6759         $dst_path = convert_iscsi_path($dst_path);
6760     } else {
6761         push @$cmd, '-O', $dst_format;
6762     }
6763
6764     push @$cmd, $src_path;
6765
6766     if (!$dst_is_iscsi && $is_zero_initialized) {
6767         push @$cmd, "zeroinit:$dst_path";
6768     } else {
6769         push @$cmd, $dst_path;
6770     }
6771
6772     my $parser = sub {
6773         my $line = shift;
6774         if($line =~ m/\((\S+)\/100\%\)/){
6775             my $percent = $1;
6776             my $transferred = int($size * $percent / 100);
6777             my $remaining = $size - $transferred;
6778
6779             print "transferred: $transferred bytes remaining: $remaining bytes total: $size bytes progression: $percent %\n";
6780         }
6781
6782     };
6783
6784     eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
6785     my $err = $@;
6786     die "copy failed: $err" if $err;
6787 }
6788
6789 sub qemu_img_format {
6790     my ($scfg, $volname) = @_;
6791
6792     if ($scfg->{path} && $volname =~ m/\.($QEMU_FORMAT_RE)$/) {
6793         return $1;
6794     } else {
6795         return "raw";
6796     }
6797 }
6798
6799 sub qemu_drive_mirror {
6800     my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $skipcomplete, $qga, $bwlimit) = @_;
6801
6802     $jobs = {} if !$jobs;
6803
6804     my $qemu_target;
6805     my $format;
6806     $jobs->{"drive-$drive"} = {};
6807
6808     if ($dst_volid =~ /^nbd:/) {
6809         $qemu_target = $dst_volid;
6810         $format = "nbd";
6811     } else {
6812         my $storecfg = PVE::Storage::config();
6813         my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
6814
6815         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6816
6817         $format = qemu_img_format($dst_scfg, $dst_volname);
6818
6819         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6820
6821         $qemu_target = $is_zero_initialized ? "zeroinit:$dst_path" : $dst_path;
6822     }
6823
6824     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $qemu_target };
6825     $opts->{format} = $format if $format;
6826
6827     if (defined($bwlimit)) {
6828         $opts->{speed} = $bwlimit * 1024;
6829         print "drive mirror is starting for drive-$drive with bandwidth limit: ${bwlimit} KB/s\n";
6830     } else {
6831         print "drive mirror is starting for drive-$drive\n";
6832     }
6833
6834     # if a job already runs for this device we get an error, catch it for cleanup
6835     eval { mon_cmd($vmid, "drive-mirror", %$opts); };
6836     if (my $err = $@) {
6837         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6838         warn "$@\n" if $@;
6839         die "mirroring error: $err\n";
6840     }
6841
6842     qemu_drive_mirror_monitor ($vmid, $vmiddst, $jobs, $skipcomplete, $qga);
6843 }
6844
6845 sub qemu_drive_mirror_monitor {
6846     my ($vmid, $vmiddst, $jobs, $skipcomplete, $qga) = @_;
6847
6848     eval {
6849         my $err_complete = 0;
6850
6851         while (1) {
6852             die "storage migration timed out\n" if $err_complete > 300;
6853
6854             my $stats = mon_cmd($vmid, "query-block-jobs");
6855
6856             my $running_mirror_jobs = {};
6857             foreach my $stat (@$stats) {
6858                 next if $stat->{type} ne 'mirror';
6859                 $running_mirror_jobs->{$stat->{device}} = $stat;
6860             }
6861
6862             my $readycounter = 0;
6863
6864             foreach my $job (keys %$jobs) {
6865
6866                 if(defined($jobs->{$job}->{complete}) && !defined($running_mirror_jobs->{$job})) {
6867                     print "$job : finished\n";
6868                     delete $jobs->{$job};
6869                     next;
6870                 }
6871
6872                 die "$job: mirroring has been cancelled\n" if !defined($running_mirror_jobs->{$job});
6873
6874                 my $busy = $running_mirror_jobs->{$job}->{busy};
6875                 my $ready = $running_mirror_jobs->{$job}->{ready};
6876                 if (my $total = $running_mirror_jobs->{$job}->{len}) {
6877                     my $transferred = $running_mirror_jobs->{$job}->{offset} || 0;
6878                     my $remaining = $total - $transferred;
6879                     my $percent = sprintf "%.2f", ($transferred * 100 / $total);
6880
6881                     print "$job: transferred: $transferred bytes remaining: $remaining bytes total: $total bytes progression: $percent % busy: $busy ready: $ready \n";
6882                 }
6883
6884                 $readycounter++ if $running_mirror_jobs->{$job}->{ready};
6885             }
6886
6887             last if scalar(keys %$jobs) == 0;
6888
6889             if ($readycounter == scalar(keys %$jobs)) {
6890                 print "all mirroring jobs are ready \n";
6891                 last if $skipcomplete; #do the complete later
6892
6893                 if ($vmiddst && $vmiddst != $vmid) {
6894                     my $agent_running = $qga && qga_check_running($vmid);
6895                     if ($agent_running) {
6896                         print "freeze filesystem\n";
6897                         eval { mon_cmd($vmid, "guest-fsfreeze-freeze"); };
6898                     } else {
6899                         print "suspend vm\n";
6900                         eval { PVE::QemuServer::vm_suspend($vmid, 1); };
6901                     }
6902
6903                     # if we clone a disk for a new target vm, we don't switch the disk
6904                     PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs);
6905
6906                     if ($agent_running) {
6907                         print "unfreeze filesystem\n";
6908                         eval { mon_cmd($vmid, "guest-fsfreeze-thaw"); };
6909                     } else {
6910                         print "resume vm\n";
6911                         eval {  PVE::QemuServer::vm_resume($vmid, 1, 1); };
6912                     }
6913
6914                     last;
6915                 } else {
6916
6917                     foreach my $job (keys %$jobs) {
6918                         # try to switch the disk if source and destination are on the same guest
6919                         print "$job: Completing block job...\n";
6920
6921                         eval { mon_cmd($vmid, "block-job-complete", device => $job) };
6922                         if ($@ =~ m/cannot be completed/) {
6923                             print "$job: Block job cannot be completed, try again.\n";
6924                             $err_complete++;
6925                         }else {
6926                             print "$job: Completed successfully.\n";
6927                             $jobs->{$job}->{complete} = 1;
6928                         }
6929                     }
6930                 }
6931             }
6932             sleep 1;
6933         }
6934     };
6935     my $err = $@;
6936
6937     if ($err) {
6938         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6939         die "mirroring error: $err";
6940     }
6941
6942 }
6943
6944 sub qemu_blockjobs_cancel {
6945     my ($vmid, $jobs) = @_;
6946
6947     foreach my $job (keys %$jobs) {
6948         print "$job: Cancelling block job\n";
6949         eval { mon_cmd($vmid, "block-job-cancel", device => $job); };
6950         $jobs->{$job}->{cancel} = 1;
6951     }
6952
6953     while (1) {
6954         my $stats = mon_cmd($vmid, "query-block-jobs");
6955
6956         my $running_jobs = {};
6957         foreach my $stat (@$stats) {
6958             $running_jobs->{$stat->{device}} = $stat;
6959         }
6960
6961         foreach my $job (keys %$jobs) {
6962
6963             if (defined($jobs->{$job}->{cancel}) && !defined($running_jobs->{$job})) {
6964                 print "$job: Done.\n";
6965                 delete $jobs->{$job};
6966             }
6967         }
6968
6969         last if scalar(keys %$jobs) == 0;
6970
6971         sleep 1;
6972     }
6973 }
6974
6975 sub clone_disk {
6976     my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
6977         $newvmid, $storage, $format, $full, $newvollist, $jobs, $skipcomplete, $qga, $bwlimit) = @_;
6978
6979     my $newvolid;
6980
6981     if (!$full) {
6982         print "create linked clone of drive $drivename ($drive->{file})\n";
6983         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
6984         push @$newvollist, $newvolid;
6985     } else {
6986
6987         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6988         $storeid = $storage if $storage;
6989
6990         my $dst_format = resolve_dst_disk_format($storecfg, $storeid, $volname, $format);
6991         my ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 3);
6992
6993         print "create full clone of drive $drivename ($drive->{file})\n";
6994         my $name = undef;
6995         if (drive_is_cloudinit($drive)) {
6996             $name = "vm-$newvmid-cloudinit";
6997             $name .= ".$dst_format" if $dst_format ne 'raw';
6998             $snapname = undef;
6999             $size = PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE;
7000         }
7001         $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $newvmid, $dst_format, $name, ($size/1024));
7002         push @$newvollist, $newvolid;
7003
7004         PVE::Storage::activate_volumes($storecfg, [$newvolid]);
7005
7006         if (drive_is_cloudinit($drive)) {
7007             goto no_data_clone;
7008         }
7009
7010         my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
7011         if (!$running || $snapname) {
7012             # TODO: handle bwlimits
7013             qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit);
7014         } else {
7015
7016             my $kvmver = get_running_qemu_version ($vmid);
7017             if (!min_version($kvmver, 2, 7)) {
7018                 die "drive-mirror with iothread requires qemu version 2.7 or higher\n"
7019                     if $drive->{iothread};
7020             }
7021
7022             qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid, $sparseinit, $jobs, $skipcomplete, $qga, $bwlimit);
7023         }
7024     }
7025
7026 no_data_clone:
7027     my ($size) = PVE::Storage::volume_size_info($storecfg, $newvolid, 3);
7028
7029     my $disk = $drive;
7030     $disk->{format} = undef;
7031     $disk->{file} = $newvolid;
7032     $disk->{size} = $size;
7033
7034     return $disk;
7035 }
7036
7037 sub get_running_qemu_version {
7038     my ($vmid) = @_;
7039     my $res = mon_cmd($vmid, "query-version");
7040     return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
7041 }
7042
7043 sub qemu_use_old_bios_files {
7044     my ($machine_type) = @_;
7045
7046     return if !$machine_type;
7047
7048     my $use_old_bios_files = undef;
7049
7050     if ($machine_type =~ m/^(\S+)\.pxe$/) {
7051         $machine_type = $1;
7052         $use_old_bios_files = 1;
7053     } else {
7054         my $version = PVE::QemuServer::Machine::extract_version($machine_type, kvm_user_version());
7055         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
7056         # load new efi bios files on migration. So this hack is required to allow
7057         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
7058         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
7059         $use_old_bios_files = !min_version($version, 2, 4);
7060     }
7061
7062     return ($use_old_bios_files, $machine_type);
7063 }
7064
7065 sub create_efidisk($$$$$) {
7066     my ($storecfg, $storeid, $vmid, $fmt, $arch) = @_;
7067
7068     my (undef, $ovmf_vars) = get_ovmf_files($arch);
7069     die "EFI vars default image not found\n" if ! -f $ovmf_vars;
7070
7071     my $vars_size_b = -s $ovmf_vars;
7072     my $vars_size = PVE::Tools::convert_size($vars_size_b, 'b' => 'kb');
7073     my $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $fmt, undef, $vars_size);
7074     PVE::Storage::activate_volumes($storecfg, [$volid]);
7075
7076     qemu_img_convert($ovmf_vars, $volid, $vars_size_b, undef, 0);
7077     my ($size) = PVE::Storage::volume_size_info($storecfg, $volid, 3);
7078
7079     return ($volid, $size/1024);
7080 }
7081
7082 sub vm_iothreads_list {
7083     my ($vmid) = @_;
7084
7085     my $res = mon_cmd($vmid, 'query-iothreads');
7086
7087     my $iothreads = {};
7088     foreach my $iothread (@$res) {
7089         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
7090     }
7091
7092     return $iothreads;
7093 }
7094
7095 sub scsihw_infos {
7096     my ($conf, $drive) = @_;
7097
7098     my $maxdev = 0;
7099
7100     if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)) {
7101         $maxdev = 7;
7102     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
7103         $maxdev = 1;
7104     } else {
7105         $maxdev = 256;
7106     }
7107
7108     my $controller = int($drive->{index} / $maxdev);
7109     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single') ? "virtioscsi" : "scsihw";
7110
7111     return ($maxdev, $controller, $controller_prefix);
7112 }
7113
7114 sub add_hyperv_enlightenments {
7115     my ($cpuFlags, $winversion, $machine_version, $bios, $gpu_passthrough, $hv_vendor_id) = @_;
7116
7117     return if $winversion < 6;
7118     return if $bios && $bios eq 'ovmf' && $winversion < 8;
7119
7120     if ($gpu_passthrough || defined($hv_vendor_id)) {
7121         $hv_vendor_id //= 'proxmox';
7122         push @$cpuFlags , "hv_vendor_id=$hv_vendor_id";
7123     }
7124
7125     if (min_version($machine_version, 2, 3)) {
7126         push @$cpuFlags , 'hv_spinlocks=0x1fff';
7127         push @$cpuFlags , 'hv_vapic';
7128         push @$cpuFlags , 'hv_time';
7129     } else {
7130         push @$cpuFlags , 'hv_spinlocks=0xffff';
7131     }
7132
7133     if (min_version($machine_version, 2, 6)) {
7134         push @$cpuFlags , 'hv_reset';
7135         push @$cpuFlags , 'hv_vpindex';
7136         push @$cpuFlags , 'hv_runtime';
7137     }
7138
7139     if ($winversion >= 7) {
7140         push @$cpuFlags , 'hv_relaxed';
7141
7142         if (min_version($machine_version, 2, 12)) {
7143             push @$cpuFlags , 'hv_synic';
7144             push @$cpuFlags , 'hv_stimer';
7145         }
7146
7147         if (min_version($machine_version, 3, 1)) {
7148             push @$cpuFlags , 'hv_ipi';
7149         }
7150     }
7151 }
7152
7153 sub windows_version {
7154     my ($ostype) = @_;
7155
7156     return 0 if !$ostype;
7157
7158     my $winversion = 0;
7159
7160     if($ostype eq 'wxp' || $ostype eq 'w2k3' || $ostype eq 'w2k') {
7161         $winversion = 5;
7162     } elsif($ostype eq 'w2k8' || $ostype eq 'wvista') {
7163         $winversion = 6;
7164     } elsif ($ostype =~ m/^win(\d+)$/) {
7165         $winversion = $1;
7166     }
7167
7168     return $winversion;
7169 }
7170
7171 sub resolve_dst_disk_format {
7172         my ($storecfg, $storeid, $src_volname, $format) = @_;
7173         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
7174
7175         if (!$format) {
7176             # if no target format is specified, use the source disk format as hint
7177             if ($src_volname) {
7178                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
7179                 $format = qemu_img_format($scfg, $src_volname);
7180             } else {
7181                 return $defFormat;
7182             }
7183         }
7184
7185         # test if requested format is supported - else use default
7186         my $supported = grep { $_ eq $format } @$validFormats;
7187         $format = $defFormat if !$supported;
7188         return $format;
7189 }
7190
7191 sub resolve_first_disk {
7192     my $conf = shift;
7193     my @disks = PVE::QemuServer::valid_drive_names();
7194     my $firstdisk;
7195     foreach my $ds (reverse @disks) {
7196         next if !$conf->{$ds};
7197         my $disk = PVE::QemuServer::parse_drive($ds, $conf->{$ds});
7198         next if PVE::QemuServer::drive_is_cdrom($disk);
7199         $firstdisk = $ds;
7200     }
7201     return $firstdisk;
7202 }
7203
7204 sub generate_uuid {
7205     my ($uuid, $uuid_str);
7206     UUID::generate($uuid);
7207     UUID::unparse($uuid, $uuid_str);
7208     return $uuid_str;
7209 }
7210
7211 sub generate_smbios1_uuid {
7212     return "uuid=".generate_uuid();
7213 }
7214
7215 sub nbd_stop {
7216     my ($vmid) = @_;
7217
7218     mon_cmd($vmid, 'nbd-server-stop');
7219 }
7220
7221 sub create_reboot_request {
7222     my ($vmid) = @_;
7223     open(my $fh, '>', "/run/qemu-server/$vmid.reboot")
7224         or die "failed to create reboot trigger file: $!\n";
7225     close($fh);
7226 }
7227
7228 sub clear_reboot_request {
7229     my ($vmid) = @_;
7230     my $path = "/run/qemu-server/$vmid.reboot";
7231     my $res = 0;
7232
7233     $res = unlink($path);
7234     die "could not remove reboot request for $vmid: $!"
7235         if !$res && $! != POSIX::ENOENT;
7236
7237     return $res;
7238 }
7239
7240 # bash completion helper
7241
7242 sub complete_backup_archives {
7243     my ($cmdname, $pname, $cvalue) = @_;
7244
7245     my $cfg = PVE::Storage::config();
7246
7247     my $storeid;
7248
7249     if ($cvalue =~ m/^([^:]+):/) {
7250         $storeid = $1;
7251     }
7252
7253     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
7254
7255     my $res = [];
7256     foreach my $id (keys %$data) {
7257         foreach my $item (@{$data->{$id}}) {
7258             next if $item->{format} !~ m/^vma\.(gz|lzo)$/;
7259             push @$res, $item->{volid} if defined($item->{volid});
7260         }
7261     }
7262
7263     return $res;
7264 }
7265
7266 my $complete_vmid_full = sub {
7267     my ($running) = @_;
7268
7269     my $idlist = vmstatus();
7270
7271     my $res = [];
7272
7273     foreach my $id (keys %$idlist) {
7274         my $d = $idlist->{$id};
7275         if (defined($running)) {
7276             next if $d->{template};
7277             next if $running && $d->{status} ne 'running';
7278             next if !$running && $d->{status} eq 'running';
7279         }
7280         push @$res, $id;
7281
7282     }
7283     return $res;
7284 };
7285
7286 sub complete_vmid {
7287     return &$complete_vmid_full();
7288 }
7289
7290 sub complete_vmid_stopped {
7291     return &$complete_vmid_full(0);
7292 }
7293
7294 sub complete_vmid_running {
7295     return &$complete_vmid_full(1);
7296 }
7297
7298 sub complete_storage {
7299
7300     my $cfg = PVE::Storage::config();
7301     my $ids = $cfg->{ids};
7302
7303     my $res = [];
7304     foreach my $sid (keys %$ids) {
7305         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
7306         next if !$ids->{$sid}->{content}->{images};
7307         push @$res, $sid;
7308     }
7309
7310     return $res;
7311 }
7312
7313 sub complete_migration_storage {
7314     my ($cmd, $param, $current_value, $all_args) = @_;
7315
7316     my $targetnode = @$all_args[1];
7317
7318     my $cfg = PVE::Storage::config();
7319     my $ids = $cfg->{ids};
7320
7321     my $res = [];
7322     foreach my $sid (keys %$ids) {
7323         next if !PVE::Storage::storage_check_enabled($cfg, $sid, $targetnode, 1);
7324         next if !$ids->{$sid}->{content}->{images};
7325         push @$res, $sid;
7326     }
7327
7328     return $res;
7329 }
7330
7331 1;