PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5
   6 use Cwd 'abs_path';
   7 use Digest::SHA;
   8 use Fcntl ':flock';
   9 use Fcntl;
  10 use File::Basename;
  11 use File::Copy qw(copy);
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use IO::Dir;
  16 use IO::File;
  17 use IO::Handle;
  18 use IO::Select;
  19 use IO::Socket::UNIX;
  20 use IPC::Open3;
  21 use JSON;
  22 use MIME::Base64;
  23 use POSIX;
  24 use Storable qw(dclone);
  25 use Time::HiRes qw(gettimeofday);
  26 use URI::Escape;
  27 use UUID;
  28
  29 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file cfs_lock_file);
  30 use PVE::DataCenterConfig;
  31 use PVE::Exception qw(raise raise_param_exc);
  32 use PVE::GuestHelpers;
  33 use PVE::INotify;
  34 use PVE::JSONSchema qw(get_standard_option);
  35 use PVE::ProcFSTools;
  36 use PVE::RPCEnvironment;
  37 use PVE::Storage;
  38 use PVE::SysFSTools;
  39 use PVE::Systemd;
  40 use PVE::Tools qw(run_command lock_file lock_file_full file_read_firstline dir_glob_foreach get_host_arch $IPV6RE);
  41
  42 use PVE::QMPClient;
  43 use PVE::QemuConfig;
  44 use PVE::QemuServer::Helpers qw(min_version);
  45 use PVE::QemuServer::Cloudinit;
  46 use PVE::QemuServer::Machine;
  47 use PVE::QemuServer::Memory;
  48 use PVE::QemuServer::Monitor qw(mon_cmd);
  49 use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr print_pcie_root_port);
  50 use PVE::QemuServer::USB qw(parse_usb_device);
  51
  52 my $EDK2_FW_BASE = '/usr/share/pve-edk2-firmware/';
  53 my $OVMF = {
  54     x86_64 => [
  55         "$EDK2_FW_BASE/OVMF_CODE.fd",
  56         "$EDK2_FW_BASE/OVMF_VARS.fd"
  57     ],
  58     aarch64 => [
  59         "$EDK2_FW_BASE/AAVMF_CODE.fd",
  60         "$EDK2_FW_BASE/AAVMF_VARS.fd"
  61     ],
  62 };
  63
  64 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  65
  66 my $QEMU_FORMAT_RE = qr/raw|cow|qcow|qcow2|qed|vmdk|cloop/;
  67
  68 # Note about locking: we use flock on the config file protect
  69 # against concurent actions.
  70 # Aditionaly, we have a 'lock' setting in the config file. This
  71 # can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
  72 # allowed when such lock is set. But you can ignore this kind of
  73 # lock with the --skiplock flag.
  74
  75 cfs_register_file('/qemu-server/',
  76                   \&parse_vm_config,
  77                   \&write_vm_config);
  78
  79 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
  80     description => "Some command save/restore state from this location.",
  81     type => 'string',
  82     maxLength => 128,
  83     optional => 1,
  84 });
  85
  86 PVE::JSONSchema::register_standard_option('pve-qm-image-format', {
  87     type => 'string',
  88     enum => [qw(raw cow qcow qed qcow2 vmdk cloop)],
  89     description => "The drive's backing file's data format.",
  90     optional => 1,
  91 });
  92
  93 PVE::JSONSchema::register_standard_option('pve-qemu-machine', {
  94         description => "Specifies the Qemu machine type.",
  95         type => 'string',
  96         pattern => '(pc|pc(-i440fx)?-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|q35|pc-q35-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|virt(?:-\d+(\.\d+)+)?(\+pve\d+)?)',
  97         maxLength => 40,
  98         optional => 1,
  99 });
 100
 101 #no warnings 'redefine';
 102
 103 sub cgroups_write {
 104    my ($controller, $vmid, $option, $value) = @_;
 105
 106    my $path = "/sys/fs/cgroup/$controller/qemu.slice/$vmid.scope/$option";
 107    PVE::ProcFSTools::write_proc_entry($path, $value);
 108
 109 }
 110
 111 my $nodename = PVE::INotify::nodename();
 112
 113 my $cpu_vendor_list = {
 114     # Intel CPUs
 115     486 => 'GenuineIntel',
 116     pentium => 'GenuineIntel',
 117     pentium2  => 'GenuineIntel',
 118     pentium3  => 'GenuineIntel',
 119     coreduo => 'GenuineIntel',
 120     core2duo => 'GenuineIntel',
 121     Conroe  => 'GenuineIntel',
 122     Penryn  => 'GenuineIntel',
 123     Nehalem  => 'GenuineIntel',
 124     'Nehalem-IBRS'  => 'GenuineIntel',
 125     Westmere => 'GenuineIntel',
 126     'Westmere-IBRS' => 'GenuineIntel',
 127     SandyBridge => 'GenuineIntel',
 128     'SandyBridge-IBRS' => 'GenuineIntel',
 129     IvyBridge => 'GenuineIntel',
 130     'IvyBridge-IBRS' => 'GenuineIntel',
 131     Haswell => 'GenuineIntel',
 132     'Haswell-IBRS' => 'GenuineIntel',
 133     'Haswell-noTSX' => 'GenuineIntel',
 134     'Haswell-noTSX-IBRS' => 'GenuineIntel',
 135     Broadwell => 'GenuineIntel',
 136     'Broadwell-IBRS' => 'GenuineIntel',
 137     'Broadwell-noTSX' => 'GenuineIntel',
 138     'Broadwell-noTSX-IBRS' => 'GenuineIntel',
 139     'Skylake-Client' => 'GenuineIntel',
 140     'Skylake-Client-IBRS' => 'GenuineIntel',
 141     'Skylake-Server' => 'GenuineIntel',
 142     'Skylake-Server-IBRS' => 'GenuineIntel',
 143     'Cascadelake-Server' => 'GenuineIntel',
 144     KnightsMill => 'GenuineIntel',
 145
 146
 147     # AMD CPUs
 148     athlon => 'AuthenticAMD',
 149     phenom  => 'AuthenticAMD',
 150     Opteron_G1  => 'AuthenticAMD',
 151     Opteron_G2  => 'AuthenticAMD',
 152     Opteron_G3  => 'AuthenticAMD',
 153     Opteron_G4  => 'AuthenticAMD',
 154     Opteron_G5  => 'AuthenticAMD',
 155     EPYC => 'AuthenticAMD',
 156     'EPYC-IBPB' => 'AuthenticAMD',
 157
 158     # generic types, use vendor from host node
 159     host => 'default',
 160     kvm32 => 'default',
 161     kvm64 => 'default',
 162     qemu32 => 'default',
 163     qemu64 => 'default',
 164     max => 'default',
 165 };
 166
 167 my @supported_cpu_flags = (
 168     'pcid',
 169     'spec-ctrl',
 170     'ibpb',
 171     'ssbd',
 172     'virt-ssbd',
 173     'amd-ssbd',
 174     'amd-no-ssb',
 175     'pdpe1gb',
 176     'md-clear',
 177     'hv-tlbflush',
 178     'hv-evmcs',
 179     'aes'
 180 );
 181 my $cpu_flag = qr/[+-](@{[join('|', @supported_cpu_flags)]})/;
 182
 183 my $cpu_fmt = {
 184     cputype => {
 185         description => "Emulated CPU type.",
 186         type => 'string',
 187         enum => [ sort { "\L$a" cmp "\L$b" } keys %$cpu_vendor_list ],
 188         default => 'kvm64',
 189         default_key => 1,
 190     },
 191     hidden => {
 192         description => "Do not identify as a KVM virtual machine.",
 193         type => 'boolean',
 194         optional => 1,
 195         default => 0
 196     },
 197     'hv-vendor-id' => {
 198         type => 'string',
 199         pattern => qr/[a-zA-Z0-9]{1,12}/,
 200         format_description => 'vendor-id',
 201         description => 'The Hyper-V vendor ID. Some drivers or programs inside Windows guests need a specific ID.',
 202         optional => 1,
 203     },
 204     flags => {
 205         description => "List of additional CPU flags separated by ';'."
 206                      . " Use '+FLAG' to enable, '-FLAG' to disable a flag."
 207                      . " Currently supported flags: @{[join(', ', @supported_cpu_flags)]}.",
 208         format_description => '+FLAG[;-FLAG...]',
 209         type => 'string',
 210         pattern => qr/$cpu_flag(;$cpu_flag)*/,
 211         optional => 1,
 212     },
 213 };
 214
 215 my $watchdog_fmt = {
 216     model => {
 217         default_key => 1,
 218         type => 'string',
 219         enum => [qw(i6300esb ib700)],
 220         description => "Watchdog type to emulate.",
 221         default => 'i6300esb',
 222         optional => 1,
 223     },
 224     action => {
 225         type => 'string',
 226         enum => [qw(reset shutdown poweroff pause debug none)],
 227         description => "The action to perform if after activation the guest fails to poll the watchdog in time.",
 228         optional => 1,
 229     },
 230 };
 231 PVE::JSONSchema::register_format('pve-qm-watchdog', $watchdog_fmt);
 232
 233 my $agent_fmt = {
 234     enabled => {
 235         description => "Enable/disable Qemu GuestAgent.",
 236         type => 'boolean',
 237         default => 0,
 238         default_key => 1,
 239     },
 240     fstrim_cloned_disks => {
 241         description => "Run fstrim after cloning/moving a disk.",
 242         type => 'boolean',
 243         optional => 1,
 244         default => 0
 245     },
 246     type => {
 247         description => "Select the agent type",
 248         type => 'string',
 249         default => 'virtio',
 250         optional => 1,
 251         enum => [qw(virtio isa)],
 252     },
 253 };
 254
 255 my $vga_fmt = {
 256     type => {
 257         description => "Select the VGA type.",
 258         type => 'string',
 259         default => 'std',
 260         optional => 1,
 261         default_key => 1,
 262         enum => [qw(cirrus qxl qxl2 qxl3 qxl4 none serial0 serial1 serial2 serial3 std virtio vmware)],
 263     },
 264     memory => {
 265         description => "Sets the VGA memory (in MiB). Has no effect with serial display.",
 266         type => 'integer',
 267         optional => 1,
 268         minimum => 4,
 269         maximum => 512,
 270     },
 271 };
 272
 273 my $ivshmem_fmt = {
 274     size => {
 275         type => 'integer',
 276         minimum => 1,
 277         description => "The size of the file in MB.",
 278     },
 279     name => {
 280         type => 'string',
 281         pattern => '[a-zA-Z0-9\-]+',
 282         optional => 1,
 283         format_description => 'string',
 284         description => "The name of the file. Will be prefixed with 'pve-shm-'. Default is the VMID. Will be deleted when the VM is stopped.",
 285     },
 286 };
 287
 288 my $audio_fmt = {
 289     device => {
 290         type => 'string',
 291         enum => [qw(ich9-intel-hda intel-hda AC97)],
 292         description =>  "Configure an audio device."
 293     },
 294     driver =>  {
 295         type => 'string',
 296         enum => ['spice'],
 297         default => 'spice',
 298         optional => 1,
 299         description => "Driver backend for the audio device."
 300     },
 301 };
 302
 303 my $spice_enhancements_fmt = {
 304     foldersharing => {
 305         type => 'boolean',
 306         optional => 1,
 307         default => '0',
 308         description =>  "Enable folder sharing via SPICE. Needs Spice-WebDAV daemon installed in the VM."
 309     },
 310     videostreaming =>  {
 311         type => 'string',
 312         enum => ['off', 'all', 'filter'],
 313         default => 'off',
 314         optional => 1,
 315         description => "Enable video streaming. Uses compression for detected video streams."
 316     },
 317 };
 318
 319 my $confdesc = {
 320     onboot => {
 321         optional => 1,
 322         type => 'boolean',
 323         description => "Specifies whether a VM will be started during system bootup.",
 324         default => 0,
 325     },
 326     autostart => {
 327         optional => 1,
 328         type => 'boolean',
 329         description => "Automatic restart after crash (currently ignored).",
 330         default => 0,
 331     },
 332     hotplug => {
 333         optional => 1,
 334         type => 'string', format => 'pve-hotplug-features',
 335         description => "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
 336         default => 'network,disk,usb',
 337     },
 338     reboot => {
 339         optional => 1,
 340         type => 'boolean',
 341         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 342         default => 1,
 343     },
 344     lock => {
 345         optional => 1,
 346         type => 'string',
 347         description => "Lock/unlock the VM.",
 348         enum => [qw(backup clone create migrate rollback snapshot snapshot-delete suspending suspended)],
 349     },
 350     cpulimit => {
 351         optional => 1,
 352         type => 'number',
 353         description => "Limit of CPU usage.",
 354         verbose_description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
 355         minimum => 0,
 356         maximum => 128,
 357         default => 0,
 358     },
 359     cpuunits => {
 360         optional => 1,
 361         type => 'integer',
 362         description => "CPU weight for a VM.",
 363         verbose_description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.",
 364         minimum => 2,
 365         maximum => 262144,
 366         default => 1024,
 367     },
 368     memory => {
 369         optional => 1,
 370         type => 'integer',
 371         description => "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
 372         minimum => 16,
 373         default => 512,
 374     },
 375     balloon => {
 376         optional => 1,
 377         type => 'integer',
 378         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 379         minimum => 0,
 380     },
 381     shares => {
 382         optional => 1,
 383         type => 'integer',
 384         description => "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning. Auto-ballooning is done by pvestatd.",
 385         minimum => 0,
 386         maximum => 50000,
 387         default => 1000,
 388     },
 389     keyboard => {
 390         optional => 1,
 391         type => 'string',
 392         description => "Keybord layout for vnc server. Default is read from the '/etc/pve/datacenter.cfg' configuration file.".
 393                        "It should not be necessary to set it.",
 394         enum => PVE::Tools::kvmkeymaplist(),
 395         default => undef,
 396     },
 397     name => {
 398         optional => 1,
 399         type => 'string', format => 'dns-name',
 400         description => "Set a name for the VM. Only used on the configuration web interface.",
 401     },
 402     scsihw => {
 403         optional => 1,
 404         type => 'string',
 405         description => "SCSI controller model",
 406         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 407         default => 'lsi',
 408     },
 409     description => {
 410         optional => 1,
 411         type => 'string',
 412         description => "Description for the VM. Only used on the configuration web interface. This is saved as comment inside the configuration file.",
 413     },
 414     ostype => {
 415         optional => 1,
 416         type => 'string',
 417         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 l24 l26 solaris)],
 418         description => "Specify guest operating system.",
 419         verbose_description => <<EODESC,
 420 Specify guest operating system. This is used to enable special
 421 optimization/features for specific operating systems:
 422
 423 [horizontal]
 424 other;; unspecified OS
 425 wxp;; Microsoft Windows XP
 426 w2k;; Microsoft Windows 2000
 427 w2k3;; Microsoft Windows 2003
 428 w2k8;; Microsoft Windows 2008
 429 wvista;; Microsoft Windows Vista
 430 win7;; Microsoft Windows 7
 431 win8;; Microsoft Windows 8/2012/2012r2
 432 win10;; Microsoft Windows 10/2016
 433 l24;; Linux 2.4 Kernel
 434 l26;; Linux 2.6 - 5.X Kernel
 435 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
 436 EODESC
 437     },
 438     boot => {
 439         optional => 1,
 440         type => 'string',
 441         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n).",
 442         pattern => '[acdn]{1,4}',
 443         default => 'cdn',
 444     },
 445     bootdisk => {
 446         optional => 1,
 447         type => 'string', format => 'pve-qm-bootdisk',
 448         description => "Enable booting from specified disk.",
 449         pattern => '(ide|sata|scsi|virtio)\d+',
 450     },
 451     smp => {
 452         optional => 1,
 453         type => 'integer',
 454         description => "The number of CPUs. Please use option -sockets instead.",
 455         minimum => 1,
 456         default => 1,
 457     },
 458     sockets => {
 459         optional => 1,
 460         type => 'integer',
 461         description => "The number of CPU sockets.",
 462         minimum => 1,
 463         default => 1,
 464     },
 465     cores => {
 466         optional => 1,
 467         type => 'integer',
 468         description => "The number of cores per socket.",
 469         minimum => 1,
 470         default => 1,
 471     },
 472     numa => {
 473         optional => 1,
 474         type => 'boolean',
 475         description => "Enable/disable NUMA.",
 476         default => 0,
 477     },
 478     hugepages => {
 479         optional => 1,
 480         type => 'string',
 481         description => "Enable/disable hugepages memory.",
 482         enum => [qw(any 2 1024)],
 483     },
 484     vcpus => {
 485         optional => 1,
 486         type => 'integer',
 487         description => "Number of hotplugged vcpus.",
 488         minimum => 1,
 489         default => 0,
 490     },
 491     acpi => {
 492         optional => 1,
 493         type => 'boolean',
 494         description => "Enable/disable ACPI.",
 495         default => 1,
 496     },
 497     agent => {
 498         optional => 1,
 499         description => "Enable/disable Qemu GuestAgent and its properties.",
 500         type => 'string',
 501         format => $agent_fmt,
 502     },
 503     kvm => {
 504         optional => 1,
 505         type => 'boolean',
 506         description => "Enable/disable KVM hardware virtualization.",
 507         default => 1,
 508     },
 509     tdf => {
 510         optional => 1,
 511         type => 'boolean',
 512         description => "Enable/disable time drift fix.",
 513         default => 0,
 514     },
 515     localtime => {
 516         optional => 1,
 517         type => 'boolean',
 518         description => "Set the real time clock to local time. This is enabled by default if ostype indicates a Microsoft OS.",
 519     },
 520     freeze => {
 521         optional => 1,
 522         type => 'boolean',
 523         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 524     },
 525     vga => {
 526         optional => 1,
 527         type => 'string', format => $vga_fmt,
 528         description => "Configure the VGA hardware.",
 529         verbose_description => "Configure the VGA Hardware. If you want to use ".
 530             "high resolution modes (>= 1280x1024x16) you may need to increase " .
 531             "the vga memory option. Since QEMU 2.9 the default VGA display type " .
 532             "is 'std' for all OS types besides some Windows versions (XP and " .
 533             "older) which use 'cirrus'. The 'qxl' option enables the SPICE " .
 534             "display server. For win* OS you can select how many independent " .
 535             "displays you want, Linux guests can add displays them self.\n".
 536             "You can also run without any graphic card, using a serial device as terminal.",
 537     },
 538     watchdog => {
 539         optional => 1,
 540         type => 'string', format => 'pve-qm-watchdog',
 541         description => "Create a virtual hardware watchdog device.",
 542         verbose_description => "Create a virtual hardware watchdog device. Once enabled" .
 543             " (by a guest action), the watchdog must be periodically polled " .
 544             "by an agent inside the guest or else the watchdog will reset " .
 545             "the guest (or execute the respective action specified)",
 546     },
 547     startdate => {
 548         optional => 1,
 549         type => 'string',
 550         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 551         description => "Set the initial date of the real time clock. Valid format for date are: 'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 552         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 553         default => 'now',
 554     },
 555     startup =>  get_standard_option('pve-startup-order'),
 556     template => {
 557         optional => 1,
 558         type => 'boolean',
 559         description => "Enable/disable Template.",
 560         default => 0,
 561     },
 562     args => {
 563         optional => 1,
 564         type => 'string',
 565         description => "Arbitrary arguments passed to kvm.",
 566         verbose_description => <<EODESCR,
 567 Arbitrary arguments passed to kvm, for example:
 568
 569 args: -no-reboot -no-hpet
 570
 571 NOTE: this option is for experts only.
 572 EODESCR
 573     },
 574     tablet => {
 575         optional => 1,
 576         type => 'boolean',
 577         default => 1,
 578         description => "Enable/disable the USB tablet device.",
 579         verbose_description => "Enable/disable the USB tablet device. This device is " .
 580             "usually needed to allow absolute mouse positioning with VNC. " .
 581             "Else the mouse runs out of sync with normal VNC clients. " .
 582             "If you're running lots of console-only guests on one host, " .
 583             "you may consider disabling this to save some context switches. " .
 584             "This is turned off by default if you use spice (-vga=qxl).",
 585     },
 586     migrate_speed => {
 587         optional => 1,
 588         type => 'integer',
 589         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 590         minimum => 0,
 591         default => 0,
 592     },
 593     migrate_downtime => {
 594         optional => 1,
 595         type => 'number',
 596         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 597         minimum => 0,
 598         default => 0.1,
 599     },
 600     cdrom => {
 601         optional => 1,
 602         type => 'string', format => 'pve-qm-ide',
 603         typetext => '<volume>',
 604         description => "This is an alias for option -ide2",
 605     },
 606     cpu => {
 607         optional => 1,
 608         description => "Emulated CPU type.",
 609         type => 'string',
 610         format => $cpu_fmt,
 611     },
 612     parent => get_standard_option('pve-snapshot-name', {
 613         optional => 1,
 614         description => "Parent snapshot name. This is used internally, and should not be modified.",
 615     }),
 616     snaptime => {
 617         optional => 1,
 618         description => "Timestamp for snapshots.",
 619         type => 'integer',
 620         minimum => 0,
 621     },
 622     vmstate => {
 623         optional => 1,
 624         type => 'string', format => 'pve-volume-id',
 625         description => "Reference to a volume which stores the VM state. This is used internally for snapshots.",
 626     },
 627     vmstatestorage => get_standard_option('pve-storage-id', {
 628         description => "Default storage for VM state volumes/files.",
 629         optional => 1,
 630     }),
 631     runningmachine => get_standard_option('pve-qemu-machine', {
 632         description => "Specifies the Qemu machine type of the running vm. This is used internally for snapshots.",
 633     }),
 634     machine => get_standard_option('pve-qemu-machine'),
 635     arch => {
 636         description => "Virtual processor architecture. Defaults to the host.",
 637         optional => 1,
 638         type => 'string',
 639         enum => [qw(x86_64 aarch64)],
 640     },
 641     smbios1 => {
 642         description => "Specify SMBIOS type 1 fields.",
 643         type => 'string', format => 'pve-qm-smbios1',
 644         maxLength => 512,
 645         optional => 1,
 646     },
 647     protection => {
 648         optional => 1,
 649         type => 'boolean',
 650         description => "Sets the protection flag of the VM. This will disable the remove VM and remove disk operations.",
 651         default => 0,
 652     },
 653     bios => {
 654         optional => 1,
 655         type => 'string',
 656         enum => [ qw(seabios ovmf) ],
 657         description => "Select BIOS implementation.",
 658         default => 'seabios',
 659     },
 660     vmgenid => {
 661         type => 'string',
 662         pattern => '(?:[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}|[01])',
 663         format_description => 'UUID',
 664         description => "Set VM Generation ID. Use '1' to autogenerate on create or update, pass '0' to disable explicitly.",
 665         verbose_description => "The VM generation ID (vmgenid) device exposes a".
 666             " 128-bit integer value identifier to the guest OS. This allows to".
 667             " notify the guest operating system when the virtual machine is".
 668             " executed with a different configuration (e.g. snapshot execution".
 669             " or creation from a template). The guest operating system notices".
 670             " the change, and is then able to react as appropriate by marking".
 671             " its copies of distributed databases as dirty, re-initializing its".
 672             " random number generator, etc.\n".
 673             "Note that auto-creation only works when done throug API/CLI create".
 674             " or update methods, but not when manually editing the config file.",
 675         default => "1 (autogenerated)",
 676         optional => 1,
 677     },
 678     hookscript => {
 679         type => 'string',
 680         format => 'pve-volume-id',
 681         optional => 1,
 682         description => "Script that will be executed during various steps in the vms lifetime.",
 683     },
 684     ivshmem => {
 685         type => 'string',
 686         format => $ivshmem_fmt,
 687         description => "Inter-VM shared memory. Useful for direct communication between VMs, or to the host.",
 688         optional => 1,
 689     },
 690     audio0 => {
 691         type => 'string',
 692         format => $audio_fmt,
 693         description => "Configure a audio device, useful in combination with QXL/Spice.",
 694         optional => 1
 695     },
 696     spice_enhancements => {
 697         type => 'string',
 698         format => $spice_enhancements_fmt,
 699         description => "Configure additional enhancements for SPICE.",
 700         optional => 1
 701     },
 702     tags => {
 703         type => 'string', format => 'pve-tag-list',
 704         description => 'Tags of the VM. This is only meta information.',
 705         optional => 1,
 706     },
 707 };
 708
 709 my $cicustom_fmt = {
 710     meta => {
 711         type => 'string',
 712         optional => 1,
 713         description => 'Specify a custom file containing all meta data passed to the VM via cloud-init. This is provider specific meaning configdrive2 and nocloud differ.',
 714         format => 'pve-volume-id',
 715         format_description => 'volume',
 716     },
 717     network => {
 718         type => 'string',
 719         optional => 1,
 720         description => 'Specify a custom file containing all network data passed to the VM via cloud-init.',
 721         format => 'pve-volume-id',
 722         format_description => 'volume',
 723     },
 724     user => {
 725         type => 'string',
 726         optional => 1,
 727         description => 'Specify a custom file containing all user data passed to the VM via cloud-init.',
 728         format => 'pve-volume-id',
 729         format_description => 'volume',
 730     },
 731 };
 732 PVE::JSONSchema::register_format('pve-qm-cicustom', $cicustom_fmt);
 733
 734 my $confdesc_cloudinit = {
 735     citype => {
 736         optional => 1,
 737         type => 'string',
 738         description => 'Specifies the cloud-init configuration format. The default depends on the configured operating system type (`ostype`. We use the `nocloud` format for Linux, and `configdrive2` for windows.',
 739         enum => ['configdrive2', 'nocloud'],
 740     },
 741     ciuser => {
 742         optional => 1,
 743         type => 'string',
 744         description => "cloud-init: User name to change ssh keys and password for instead of the image's configured default user.",
 745     },
 746     cipassword => {
 747         optional => 1,
 748         type => 'string',
 749         description => 'cloud-init: Password to assign the user. Using this is generally not recommended. Use ssh keys instead. Also note that older cloud-init versions do not support hashed passwords.',
 750     },
 751     cicustom => {
 752         optional => 1,
 753         type => 'string',
 754         description => 'cloud-init: Specify custom files to replace the automatically generated ones at start.',
 755         format => 'pve-qm-cicustom',
 756     },
 757     searchdomain => {
 758         optional => 1,
 759         type => 'string',
 760         description => "cloud-init: Sets DNS search domains for a container. Create will automatically use the setting from the host if neither searchdomain nor nameserver are set.",
 761     },
 762     nameserver => {
 763         optional => 1,
 764         type => 'string', format => 'address-list',
 765         description => "cloud-init: Sets DNS server IP address for a container. Create will automatically use the setting from the host if neither searchdomain nor nameserver are set.",
 766     },
 767     sshkeys => {
 768         optional => 1,
 769         type => 'string',
 770         format => 'urlencoded',
 771         description => "cloud-init: Setup public SSH keys (one key per line, OpenSSH format).",
 772     },
 773 };
 774
 775 # what about other qemu settings ?
 776 #cpu => 'string',
 777 #machine => 'string',
 778 #fda => 'file',
 779 #fdb => 'file',
 780 #mtdblock => 'file',
 781 #sd => 'file',
 782 #pflash => 'file',
 783 #snapshot => 'bool',
 784 #bootp => 'file',
 785 ##tftp => 'dir',
 786 ##smb => 'dir',
 787 #kernel => 'file',
 788 #append => 'string',
 789 #initrd => 'file',
 790 ##soundhw => 'string',
 791
 792 while (my ($k, $v) = each %$confdesc) {
 793     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 794 }
 795
 796 my $MAX_IDE_DISKS = 4;
 797 my $MAX_SCSI_DISKS = 14;
 798 my $MAX_VIRTIO_DISKS = 16;
 799 my $MAX_SATA_DISKS = 6;
 800 my $MAX_USB_DEVICES = 5;
 801 my $MAX_NETS = 32;
 802 my $MAX_UNUSED_DISKS = 256;
 803 my $MAX_HOSTPCI_DEVICES = 16;
 804 my $MAX_SERIAL_PORTS = 4;
 805 my $MAX_PARALLEL_PORTS = 3;
 806 my $MAX_NUMA = 8;
 807
 808 my $numa_fmt = {
 809     cpus => {
 810         type => "string",
 811         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 812         description => "CPUs accessing this NUMA node.",
 813         format_description => "id[-id];...",
 814     },
 815     memory => {
 816         type => "number",
 817         description => "Amount of memory this NUMA node provides.",
 818         optional => 1,
 819     },
 820     hostnodes => {
 821         type => "string",
 822         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 823         description => "Host NUMA nodes to use.",
 824         format_description => "id[-id];...",
 825         optional => 1,
 826     },
 827     policy => {
 828         type => 'string',
 829         enum => [qw(preferred bind interleave)],
 830         description => "NUMA allocation policy.",
 831         optional => 1,
 832     },
 833 };
 834 PVE::JSONSchema::register_format('pve-qm-numanode', $numa_fmt);
 835 my $numadesc = {
 836     optional => 1,
 837     type => 'string', format => $numa_fmt,
 838     description => "NUMA topology.",
 839 };
 840 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 841
 842 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 843     $confdesc->{"numa$i"} = $numadesc;
 844 }
 845
 846 my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000',  'pcnet',  'virtio',
 847                       'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3',
 848                       'e1000-82540em', 'e1000-82544gc', 'e1000-82545em'];
 849 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 850
 851 my $net_fmt_bridge_descr = <<__EOD__;
 852 Bridge to attach the network device to. The Proxmox VE standard bridge
 853 is called 'vmbr0'.
 854
 855 If you do not specify a bridge, we create a kvm user (NATed) network
 856 device, which provides DHCP and DNS services. The following addresses
 857 are used:
 858
 859  10.0.2.2   Gateway
 860  10.0.2.3   DNS Server
 861  10.0.2.4   SMB Server
 862
 863 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 864 __EOD__
 865
 866 my $net_fmt = {
 867     macaddr  => get_standard_option('mac-addr', {
 868         description => "MAC address. That address must be unique withing your network. This is automatically generated if not specified.",
 869     }),
 870     model => {
 871         type => 'string',
 872         description => "Network Card Model. The 'virtio' model provides the best performance with very low CPU overhead. If your guest does not support this driver, it is usually best to use 'e1000'.",
 873         enum => $nic_model_list,
 874         default_key => 1,
 875     },
 876     (map { $_ => { keyAlias => 'model', alias => 'macaddr' }} @$nic_model_list),
 877     bridge => {
 878         type => 'string',
 879         description => $net_fmt_bridge_descr,
 880         format_description => 'bridge',
 881         optional => 1,
 882     },
 883     queues => {
 884         type => 'integer',
 885         minimum => 0, maximum => 16,
 886         description => 'Number of packet queues to be used on the device.',
 887         optional => 1,
 888     },
 889     rate => {
 890         type => 'number',
 891         minimum => 0,
 892         description => "Rate limit in mbps (megabytes per second) as floating point number.",
 893         optional => 1,
 894     },
 895     tag => {
 896         type => 'integer',
 897         minimum => 1, maximum => 4094,
 898         description => 'VLAN tag to apply to packets on this interface.',
 899         optional => 1,
 900     },
 901     trunks => {
 902         type => 'string',
 903         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 904         description => 'VLAN trunks to pass through this interface.',
 905         format_description => 'vlanid[;vlanid...]',
 906         optional => 1,
 907     },
 908     firewall => {
 909         type => 'boolean',
 910         description => 'Whether this interface should be protected by the firewall.',
 911         optional => 1,
 912     },
 913     link_down => {
 914         type => 'boolean',
 915         description => 'Whether this interface should be disconnected (like pulling the plug).',
 916         optional => 1,
 917     },
 918 };
 919
 920 my $netdesc = {
 921     optional => 1,
 922     type => 'string', format => $net_fmt,
 923     description => "Specify network devices.",
 924 };
 925
 926 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 927
 928 my $ipconfig_fmt = {
 929     ip => {
 930         type => 'string',
 931         format => 'pve-ipv4-config',
 932         format_description => 'IPv4Format/CIDR',
 933         description => 'IPv4 address in CIDR format.',
 934         optional => 1,
 935         default => 'dhcp',
 936     },
 937     gw => {
 938         type => 'string',
 939         format => 'ipv4',
 940         format_description => 'GatewayIPv4',
 941         description => 'Default gateway for IPv4 traffic.',
 942         optional => 1,
 943         requires => 'ip',
 944     },
 945     ip6 => {
 946         type => 'string',
 947         format => 'pve-ipv6-config',
 948         format_description => 'IPv6Format/CIDR',
 949         description => 'IPv6 address in CIDR format.',
 950         optional => 1,
 951         default => 'dhcp',
 952     },
 953     gw6 => {
 954         type => 'string',
 955         format => 'ipv6',
 956         format_description => 'GatewayIPv6',
 957         description => 'Default gateway for IPv6 traffic.',
 958         optional => 1,
 959         requires => 'ip6',
 960     },
 961 };
 962 PVE::JSONSchema::register_format('pve-qm-ipconfig', $ipconfig_fmt);
 963 my $ipconfigdesc = {
 964     optional => 1,
 965     type => 'string', format => 'pve-qm-ipconfig',
 966     description => <<'EODESCR',
 967 cloud-init: Specify IP addresses and gateways for the corresponding interface.
 968
 969 IP addresses use CIDR notation, gateways are optional but need an IP of the same type specified.
 970
 971 The special string 'dhcp' can be used for IP addresses to use DHCP, in which case no explicit gateway should be provided.
 972 For IPv6 the special string 'auto' can be used to use stateless autoconfiguration.
 973
 974 If cloud-init is enabled and neither an IPv4 nor an IPv6 address is specified, it defaults to using dhcp on IPv4.
 975 EODESCR
 976 };
 977 PVE::JSONSchema::register_standard_option("pve-qm-ipconfig", $netdesc);
 978
 979 for (my $i = 0; $i < $MAX_NETS; $i++)  {
 980     $confdesc->{"net$i"} = $netdesc;
 981     $confdesc_cloudinit->{"ipconfig$i"} = $ipconfigdesc;
 982 }
 983
 984 foreach my $key (keys %$confdesc_cloudinit) {
 985     $confdesc->{$key} = $confdesc_cloudinit->{$key};
 986 }
 987
 988 PVE::JSONSchema::register_format('pve-volume-id-or-qm-path', \&verify_volume_id_or_qm_path);
 989 sub verify_volume_id_or_qm_path {
 990     my ($volid, $noerr) = @_;
 991
 992     if ($volid eq 'none' || $volid eq 'cdrom' || $volid =~ m|^/|) {
 993         return $volid;
 994     }
 995
 996     # if its neither 'none' nor 'cdrom' nor a path, check if its a volume-id
 997     $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') };
 998     if ($@) {
 999         return undef if $noerr;
1000         die $@;
1001     }
1002     return $volid;
1003 }
1004
1005 my $drivename_hash;
1006
1007 my %drivedesc_base = (
1008     volume => { alias => 'file' },
1009     file => {
1010         type => 'string',
1011         format => 'pve-volume-id-or-qm-path',
1012         default_key => 1,
1013         format_description => 'volume',
1014         description => "The drive's backing volume.",
1015     },
1016     media => {
1017         type => 'string',
1018         enum => [qw(cdrom disk)],
1019         description => "The drive's media type.",
1020         default => 'disk',
1021         optional => 1
1022     },
1023     cyls => {
1024         type => 'integer',
1025         description => "Force the drive's physical geometry to have a specific cylinder count.",
1026         optional => 1
1027     },
1028     heads => {
1029         type => 'integer',
1030         description => "Force the drive's physical geometry to have a specific head count.",
1031         optional => 1
1032     },
1033     secs => {
1034         type => 'integer',
1035         description => "Force the drive's physical geometry to have a specific sector count.",
1036         optional => 1
1037     },
1038     trans => {
1039         type => 'string',
1040         enum => [qw(none lba auto)],
1041         description => "Force disk geometry bios translation mode.",
1042         optional => 1,
1043     },
1044     snapshot => {
1045         type => 'boolean',
1046         description => "Controls qemu's snapshot mode feature."
1047             . " If activated, changes made to the disk are temporary and will"
1048             . " be discarded when the VM is shutdown.",
1049         optional => 1,
1050     },
1051     cache => {
1052         type => 'string',
1053         enum => [qw(none writethrough writeback unsafe directsync)],
1054         description => "The drive's cache mode",
1055         optional => 1,
1056     },
1057     format => get_standard_option('pve-qm-image-format'),
1058     size => {
1059         type => 'string',
1060         format => 'disk-size',
1061         format_description => 'DiskSize',
1062         description => "Disk size. This is purely informational and has no effect.",
1063         optional => 1,
1064     },
1065     backup => {
1066         type => 'boolean',
1067         description => "Whether the drive should be included when making backups.",
1068         optional => 1,
1069     },
1070     replicate => {
1071         type => 'boolean',
1072         description => 'Whether the drive should considered for replication jobs.',
1073         optional => 1,
1074         default => 1,
1075     },
1076     rerror => {
1077         type => 'string',
1078         enum => [qw(ignore report stop)],
1079         description => 'Read error action.',
1080         optional => 1,
1081     },
1082     werror => {
1083         type => 'string',
1084         enum => [qw(enospc ignore report stop)],
1085         description => 'Write error action.',
1086         optional => 1,
1087     },
1088     aio => {
1089         type => 'string',
1090         enum => [qw(native threads)],
1091         description => 'AIO type to use.',
1092         optional => 1,
1093     },
1094     discard => {
1095         type => 'string',
1096         enum => [qw(ignore on)],
1097         description => 'Controls whether to pass discard/trim requests to the underlying storage.',
1098         optional => 1,
1099     },
1100     detect_zeroes => {
1101         type => 'boolean',
1102         description => 'Controls whether to detect and try to optimize writes of zeroes.',
1103         optional => 1,
1104     },
1105     serial => {
1106         type => 'string',
1107         format => 'urlencoded',
1108         format_description => 'serial',
1109         maxLength => 20*3, # *3 since it's %xx url enoded
1110         description => "The drive's reported serial number, url-encoded, up to 20 bytes long.",
1111         optional => 1,
1112     },
1113     shared => {
1114         type => 'boolean',
1115         description => 'Mark this locally-managed volume as available on all nodes',
1116         verbose_description => "Mark this locally-managed volume as available on all nodes.\n\nWARNING: This option does not share the volume automatically, it assumes it is shared already!",
1117         optional => 1,
1118         default => 0,
1119     }
1120 );
1121
1122 my %iothread_fmt = ( iothread => {
1123         type => 'boolean',
1124         description => "Whether to use iothreads for this drive",
1125         optional => 1,
1126 });
1127
1128 my %model_fmt = (
1129     model => {
1130         type => 'string',
1131         format => 'urlencoded',
1132         format_description => 'model',
1133         maxLength => 40*3, # *3 since it's %xx url enoded
1134         description => "The drive's reported model name, url-encoded, up to 40 bytes long.",
1135         optional => 1,
1136     },
1137 );
1138
1139 my %queues_fmt = (
1140     queues => {
1141         type => 'integer',
1142         description => "Number of queues.",
1143         minimum => 2,
1144         optional => 1
1145     }
1146 );
1147
1148 my %scsiblock_fmt = (
1149     scsiblock => {
1150         type => 'boolean',
1151         description => "whether to use scsi-block for full passthrough of host block device\n\nWARNING: can lead to I/O errors in combination with low memory or high memory fragmentation on host",
1152         optional => 1,
1153         default => 0,
1154     },
1155 );
1156
1157 my %ssd_fmt = (
1158     ssd => {
1159         type => 'boolean',
1160         description => "Whether to expose this drive as an SSD, rather than a rotational hard disk.",
1161         optional => 1,
1162     },
1163 );
1164
1165 my %wwn_fmt = (
1166     wwn => {
1167         type => 'string',
1168         pattern => qr/^(0x)[0-9a-fA-F]{16}/,
1169         format_description => 'wwn',
1170         description => "The drive's worldwide name, encoded as 16 bytes hex string, prefixed by '0x'.",
1171         optional => 1,
1172     },
1173 );
1174
1175 my $add_throttle_desc = sub {
1176     my ($key, $type, $what, $unit, $longunit, $minimum) = @_;
1177     my $d = {
1178         type => $type,
1179         format_description => $unit,
1180         description => "Maximum $what in $longunit.",
1181         optional => 1,
1182     };
1183     $d->{minimum} = $minimum if defined($minimum);
1184     $drivedesc_base{$key} = $d;
1185 };
1186 # throughput: (leaky bucket)
1187 $add_throttle_desc->('bps',     'integer', 'r/w speed',   'bps',  'bytes per second');
1188 $add_throttle_desc->('bps_rd',  'integer', 'read speed',  'bps',  'bytes per second');
1189 $add_throttle_desc->('bps_wr',  'integer', 'write speed', 'bps',  'bytes per second');
1190 $add_throttle_desc->('mbps',    'number',  'r/w speed',   'mbps', 'megabytes per second');
1191 $add_throttle_desc->('mbps_rd', 'number',  'read speed',  'mbps', 'megabytes per second');
1192 $add_throttle_desc->('mbps_wr', 'number',  'write speed', 'mbps', 'megabytes per second');
1193 $add_throttle_desc->('iops',    'integer', 'r/w I/O',     'iops', 'operations per second');
1194 $add_throttle_desc->('iops_rd', 'integer', 'read I/O',    'iops', 'operations per second');
1195 $add_throttle_desc->('iops_wr', 'integer', 'write I/O',   'iops', 'operations per second');
1196
1197 # pools: (pool of IO before throttling starts taking effect)
1198 $add_throttle_desc->('mbps_max',    'number',  'unthrottled r/w pool',       'mbps', 'megabytes per second');
1199 $add_throttle_desc->('mbps_rd_max', 'number',  'unthrottled read pool',      'mbps', 'megabytes per second');
1200 $add_throttle_desc->('mbps_wr_max', 'number',  'unthrottled write pool',     'mbps', 'megabytes per second');
1201 $add_throttle_desc->('iops_max',    'integer', 'unthrottled r/w I/O pool',   'iops', 'operations per second');
1202 $add_throttle_desc->('iops_rd_max', 'integer', 'unthrottled read I/O pool',  'iops', 'operations per second');
1203 $add_throttle_desc->('iops_wr_max', 'integer', 'unthrottled write I/O pool', 'iops', 'operations per second');
1204
1205 # burst lengths
1206 $add_throttle_desc->('bps_max_length',     'integer', 'length of I/O bursts',       'seconds', 'seconds', 1);
1207 $add_throttle_desc->('bps_rd_max_length',  'integer', 'length of read I/O bursts',  'seconds', 'seconds', 1);
1208 $add_throttle_desc->('bps_wr_max_length',  'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
1209 $add_throttle_desc->('iops_max_length',    'integer', 'length of I/O bursts',       'seconds', 'seconds', 1);
1210 $add_throttle_desc->('iops_rd_max_length', 'integer', 'length of read I/O bursts',  'seconds', 'seconds', 1);
1211 $add_throttle_desc->('iops_wr_max_length', 'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
1212
1213 # legacy support
1214 $drivedesc_base{'bps_rd_length'} = { alias => 'bps_rd_max_length' };
1215 $drivedesc_base{'bps_wr_length'} = { alias => 'bps_wr_max_length' };
1216 $drivedesc_base{'iops_rd_length'} = { alias => 'iops_rd_max_length' };
1217 $drivedesc_base{'iops_wr_length'} = { alias => 'iops_wr_max_length' };
1218
1219 my $ide_fmt = {
1220     %drivedesc_base,
1221     %model_fmt,
1222     %ssd_fmt,
1223     %wwn_fmt,
1224 };
1225 PVE::JSONSchema::register_format("pve-qm-ide", $ide_fmt);
1226
1227 my $idedesc = {
1228     optional => 1,
1229     type => 'string', format => $ide_fmt,
1230     description => "Use volume as IDE hard disk or CD-ROM (n is 0 to " .($MAX_IDE_DISKS -1) . ").",
1231 };
1232 PVE::JSONSchema::register_standard_option("pve-qm-ide", $idedesc);
1233
1234 my $scsi_fmt = {
1235     %drivedesc_base,
1236     %iothread_fmt,
1237     %queues_fmt,
1238     %scsiblock_fmt,
1239     %ssd_fmt,
1240     %wwn_fmt,
1241 };
1242 my $scsidesc = {
1243     optional => 1,
1244     type => 'string', format => $scsi_fmt,
1245     description => "Use volume as SCSI hard disk or CD-ROM (n is 0 to " . ($MAX_SCSI_DISKS - 1) . ").",
1246 };
1247 PVE::JSONSchema::register_standard_option("pve-qm-scsi", $scsidesc);
1248
1249 my $sata_fmt = {
1250     %drivedesc_base,
1251     %ssd_fmt,
1252     %wwn_fmt,
1253 };
1254 my $satadesc = {
1255     optional => 1,
1256     type => 'string', format => $sata_fmt,
1257     description => "Use volume as SATA hard disk or CD-ROM (n is 0 to " . ($MAX_SATA_DISKS - 1). ").",
1258 };
1259 PVE::JSONSchema::register_standard_option("pve-qm-sata", $satadesc);
1260
1261 my $virtio_fmt = {
1262     %drivedesc_base,
1263     %iothread_fmt,
1264 };
1265 my $virtiodesc = {
1266     optional => 1,
1267     type => 'string', format => $virtio_fmt,
1268     description => "Use volume as VIRTIO hard disk (n is 0 to " . ($MAX_VIRTIO_DISKS - 1) . ").",
1269 };
1270 PVE::JSONSchema::register_standard_option("pve-qm-virtio", $virtiodesc);
1271
1272 my $alldrive_fmt = {
1273     %drivedesc_base,
1274     %iothread_fmt,
1275     %model_fmt,
1276     %queues_fmt,
1277     %scsiblock_fmt,
1278     %ssd_fmt,
1279     %wwn_fmt,
1280 };
1281
1282 my $efidisk_fmt = {
1283     volume => { alias => 'file' },
1284     file => {
1285         type => 'string',
1286         format => 'pve-volume-id-or-qm-path',
1287         default_key => 1,
1288         format_description => 'volume',
1289         description => "The drive's backing volume.",
1290     },
1291     format => get_standard_option('pve-qm-image-format'),
1292     size => {
1293         type => 'string',
1294         format => 'disk-size',
1295         format_description => 'DiskSize',
1296         description => "Disk size. This is purely informational and has no effect.",
1297         optional => 1,
1298     },
1299 };
1300
1301 my $efidisk_desc = {
1302     optional => 1,
1303     type => 'string', format => $efidisk_fmt,
1304     description => "Configure a Disk for storing EFI vars",
1305 };
1306
1307 PVE::JSONSchema::register_standard_option("pve-qm-efidisk", $efidisk_desc);
1308
1309 my $usb_fmt = {
1310     host => {
1311         default_key => 1,
1312         type => 'string', format => 'pve-qm-usb-device',
1313         format_description => 'HOSTUSBDEVICE|spice',
1314         description => <<EODESCR,
1315 The Host USB device or port or the value 'spice'. HOSTUSBDEVICE syntax is:
1316
1317  'bus-port(.port)*' (decimal numbers) or
1318  'vendor_id:product_id' (hexadeciaml numbers) or
1319  'spice'
1320
1321 You can use the 'lsusb -t' command to list existing usb devices.
1322
1323 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1324
1325 The value 'spice' can be used to add a usb redirection devices for spice.
1326 EODESCR
1327     },
1328     usb3 => {
1329         optional => 1,
1330         type => 'boolean',
1331         description => "Specifies whether if given host option is a USB3 device or port.",
1332         default => 0,
1333     },
1334 };
1335
1336 my $usbdesc = {
1337     optional => 1,
1338     type => 'string', format => $usb_fmt,
1339     description => "Configure an USB device (n is 0 to 4).",
1340 };
1341 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
1342
1343 my $PCIRE = qr/([a-f0-9]{4}:)?[a-f0-9]{2}:[a-f0-9]{2}(?:\.[a-f0-9])?/;
1344 my $hostpci_fmt = {
1345     host => {
1346         default_key => 1,
1347         type => 'string',
1348         pattern => qr/$PCIRE(;$PCIRE)*/,
1349         format_description => 'HOSTPCIID[;HOSTPCIID2...]',
1350         description => <<EODESCR,
1351 Host PCI device pass through. The PCI ID of a host's PCI device or a list
1352 of PCI virtual functions of the host. HOSTPCIID syntax is:
1353
1354 'bus:dev.func' (hexadecimal numbers)
1355
1356 You can us the 'lspci' command to list existing PCI devices.
1357 EODESCR
1358     },
1359     rombar => {
1360         type => 'boolean',
1361         description =>  "Specify whether or not the device's ROM will be visible in the guest's memory map.",
1362         optional => 1,
1363         default => 1,
1364     },
1365     romfile => {
1366         type => 'string',
1367         pattern => '[^,;]+',
1368         format_description => 'string',
1369         description => "Custom pci device rom filename (must be located in /usr/share/kvm/).",
1370         optional => 1,
1371     },
1372     pcie => {
1373         type => 'boolean',
1374         description =>  "Choose the PCI-express bus (needs the 'q35' machine model).",
1375         optional => 1,
1376         default => 0,
1377     },
1378     'x-vga' => {
1379         type => 'boolean',
1380         description =>  "Enable vfio-vga device support.",
1381         optional => 1,
1382         default => 0,
1383     },
1384     'mdev' => {
1385         type => 'string',
1386         format_description => 'string',
1387         pattern => '[^/\.:]+',
1388         optional => 1,
1389         description => <<EODESCR
1390 The type of mediated device to use.
1391 An instance of this type will be created on startup of the VM and
1392 will be cleaned up when the VM stops.
1393 EODESCR
1394     }
1395 };
1396 PVE::JSONSchema::register_format('pve-qm-hostpci', $hostpci_fmt);
1397
1398 my $hostpcidesc = {
1399         optional => 1,
1400         type => 'string', format => 'pve-qm-hostpci',
1401         description => "Map host PCI devices into guest.",
1402         verbose_description =>  <<EODESCR,
1403 Map host PCI devices into guest.
1404
1405 NOTE: This option allows direct access to host hardware. So it is no longer
1406 possible to migrate such machines - use with special care.
1407
1408 CAUTION: Experimental! User reported problems with this option.
1409 EODESCR
1410 };
1411 PVE::JSONSchema::register_standard_option("pve-qm-hostpci", $hostpcidesc);
1412
1413 my $serialdesc = {
1414         optional => 1,
1415         type => 'string',
1416         pattern => '(/dev/.+|socket)',
1417         description =>  "Create a serial device inside the VM (n is 0 to 3)",
1418         verbose_description =>  <<EODESCR,
1419 Create a serial device inside the VM (n is 0 to 3), and pass through a
1420 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1421 host side (use 'qm terminal' to open a terminal connection).
1422
1423 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines - use with special care.
1424
1425 CAUTION: Experimental! User reported problems with this option.
1426 EODESCR
1427 };
1428
1429 my $paralleldesc= {
1430         optional => 1,
1431         type => 'string',
1432         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
1433         description =>  "Map host parallel devices (n is 0 to 2).",
1434         verbose_description =>  <<EODESCR,
1435 Map host parallel devices (n is 0 to 2).
1436
1437 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1438
1439 CAUTION: Experimental! User reported problems with this option.
1440 EODESCR
1441 };
1442
1443 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
1444     $confdesc->{"parallel$i"} = $paralleldesc;
1445 }
1446
1447 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
1448     $confdesc->{"serial$i"} = $serialdesc;
1449 }
1450
1451 for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
1452     $confdesc->{"hostpci$i"} = $hostpcidesc;
1453 }
1454
1455 for (my $i = 0; $i < $MAX_IDE_DISKS; $i++)  {
1456     $drivename_hash->{"ide$i"} = 1;
1457     $confdesc->{"ide$i"} = $idedesc;
1458 }
1459
1460 for (my $i = 0; $i < $MAX_SATA_DISKS; $i++)  {
1461     $drivename_hash->{"sata$i"} = 1;
1462     $confdesc->{"sata$i"} = $satadesc;
1463 }
1464
1465 for (my $i = 0; $i < $MAX_SCSI_DISKS; $i++)  {
1466     $drivename_hash->{"scsi$i"} = 1;
1467     $confdesc->{"scsi$i"} = $scsidesc ;
1468 }
1469
1470 for (my $i = 0; $i < $MAX_VIRTIO_DISKS; $i++)  {
1471     $drivename_hash->{"virtio$i"} = 1;
1472     $confdesc->{"virtio$i"} = $virtiodesc;
1473 }
1474
1475 $drivename_hash->{efidisk0} = 1;
1476 $confdesc->{efidisk0} = $efidisk_desc;
1477
1478 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
1479     $confdesc->{"usb$i"} = $usbdesc;
1480 }
1481
1482 my $unuseddesc = {
1483     optional => 1,
1484     type => 'string', format => 'pve-volume-id',
1485     description => "Reference to unused volumes. This is used internally, and should not be modified manually.",
1486 };
1487
1488 for (my $i = 0; $i < $MAX_UNUSED_DISKS; $i++)  {
1489     $confdesc->{"unused$i"} = $unuseddesc;
1490 }
1491
1492 my $kvm_api_version = 0;
1493
1494 sub kvm_version {
1495     return $kvm_api_version if $kvm_api_version;
1496
1497     open my $fh, '<', '/dev/kvm'
1498         or return undef;
1499
1500     # 0xae00 => KVM_GET_API_VERSION
1501     $kvm_api_version = ioctl($fh, 0xae00, 0);
1502
1503     return $kvm_api_version;
1504 }
1505
1506 my $kvm_user_version = {};
1507 my $kvm_mtime = {};
1508
1509 sub kvm_user_version {
1510     my ($binary) = @_;
1511
1512     $binary //= get_command_for_arch(get_host_arch()); # get the native arch by default
1513     my $st = stat($binary);
1514
1515     my $cachedmtime = $kvm_mtime->{$binary} // -1;
1516     return $kvm_user_version->{$binary} if $kvm_user_version->{$binary} &&
1517         $cachedmtime == $st->mtime;
1518
1519     $kvm_user_version->{$binary} = 'unknown';
1520     $kvm_mtime->{$binary} = $st->mtime;
1521
1522     my $code = sub {
1523         my $line = shift;
1524         if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1525             $kvm_user_version->{$binary} = $2;
1526         }
1527     };
1528
1529     eval { run_command([$binary, '--version'], outfunc => $code); };
1530     warn $@ if $@;
1531
1532     return $kvm_user_version->{$binary};
1533
1534 }
1535
1536 sub kernel_has_vhost_net {
1537     return -c '/dev/vhost-net';
1538 }
1539
1540 sub valid_drive_names {
1541     # order is important - used to autoselect boot disk
1542     return ((map { "ide$_" } (0 .. ($MAX_IDE_DISKS - 1))),
1543             (map { "scsi$_" } (0 .. ($MAX_SCSI_DISKS - 1))),
1544             (map { "virtio$_" } (0 .. ($MAX_VIRTIO_DISKS - 1))),
1545             (map { "sata$_" } (0 .. ($MAX_SATA_DISKS - 1))),
1546             'efidisk0');
1547 }
1548
1549 sub is_valid_drivename {
1550     my $dev = shift;
1551
1552     return defined($drivename_hash->{$dev});
1553 }
1554
1555 sub option_exists {
1556     my $key = shift;
1557     return defined($confdesc->{$key});
1558 }
1559
1560 my $cdrom_path;
1561 sub get_cdrom_path {
1562
1563     return  $cdrom_path if $cdrom_path;
1564
1565     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
1566     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
1567     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
1568 }
1569
1570 sub get_iso_path {
1571     my ($storecfg, $vmid, $cdrom) = @_;
1572
1573     if ($cdrom eq 'cdrom') {
1574         return get_cdrom_path();
1575     } elsif ($cdrom eq 'none') {
1576         return '';
1577     } elsif ($cdrom =~ m|^/|) {
1578         return $cdrom;
1579     } else {
1580         return PVE::Storage::path($storecfg, $cdrom);
1581     }
1582 }
1583
1584 # try to convert old style file names to volume IDs
1585 sub filename_to_volume_id {
1586     my ($vmid, $file, $media) = @_;
1587
1588      if (!($file eq 'none' || $file eq 'cdrom' ||
1589           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1590
1591         return undef if $file =~ m|/|;
1592
1593         if ($media && $media eq 'cdrom') {
1594             $file = "local:iso/$file";
1595         } else {
1596             $file = "local:$vmid/$file";
1597         }
1598     }
1599
1600     return $file;
1601 }
1602
1603 sub verify_media_type {
1604     my ($opt, $vtype, $media) = @_;
1605
1606     return if !$media;
1607
1608     my $etype;
1609     if ($media eq 'disk') {
1610         $etype = 'images';
1611     } elsif ($media eq 'cdrom') {
1612         $etype = 'iso';
1613     } else {
1614         die "internal error";
1615     }
1616
1617     return if ($vtype eq $etype);
1618
1619     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1620 }
1621
1622 sub cleanup_drive_path {
1623     my ($opt, $storecfg, $drive) = @_;
1624
1625     # try to convert filesystem paths to volume IDs
1626
1627     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1628         ($drive->{file} !~ m|^/dev/.+|) &&
1629         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1630         ($drive->{file} !~ m/^\d+$/)) {
1631         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1632         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"}) if !$vtype;
1633         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1634         verify_media_type($opt, $vtype, $drive->{media});
1635         $drive->{file} = $volid;
1636     }
1637
1638     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1639 }
1640
1641 sub parse_hotplug_features {
1642     my ($data) = @_;
1643
1644     my $res = {};
1645
1646     return $res if $data eq '0';
1647
1648     $data = $confdesc->{hotplug}->{default} if $data eq '1';
1649
1650     foreach my $feature (PVE::Tools::split_list($data)) {
1651         if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
1652             $res->{$1} = 1;
1653         } else {
1654             die "invalid hotplug feature '$feature'\n";
1655         }
1656     }
1657     return $res;
1658 }
1659
1660 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1661 sub pve_verify_hotplug_features {
1662     my ($value, $noerr) = @_;
1663
1664     return $value if parse_hotplug_features($value);
1665
1666     return undef if $noerr;
1667
1668     die "unable to parse hotplug option\n";
1669 }
1670
1671 # ideX = [volume=]volume-id[,media=d][,cyls=c,heads=h,secs=s[,trans=t]]
1672 #        [,snapshot=on|off][,cache=on|off][,format=f][,backup=yes|no]
1673 #        [,rerror=ignore|report|stop][,werror=enospc|ignore|report|stop]
1674 #        [,aio=native|threads][,discard=ignore|on][,detect_zeroes=on|off]
1675 #        [,iothread=on][,serial=serial][,model=model]
1676
1677 sub parse_drive {
1678     my ($key, $data) = @_;
1679
1680     my ($interface, $index);
1681
1682     if ($key =~ m/^([^\d]+)(\d+)$/) {
1683         $interface = $1;
1684         $index = $2;
1685     } else {
1686         return undef;
1687     }
1688
1689     my $desc = $key =~ /^unused\d+$/ ? $alldrive_fmt
1690                                      : $confdesc->{$key}->{format};
1691     if (!$desc) {
1692         warn "invalid drive key: $key\n";
1693         return undef;
1694     }
1695     my $res = eval { PVE::JSONSchema::parse_property_string($desc, $data) };
1696     return undef if !$res;
1697     $res->{interface} = $interface;
1698     $res->{index} = $index;
1699
1700     my $error = 0;
1701     foreach my $opt (qw(bps bps_rd bps_wr)) {
1702         if (my $bps = defined(delete $res->{$opt})) {
1703             if (defined($res->{"m$opt"})) {
1704                 warn "both $opt and m$opt specified\n";
1705                 ++$error;
1706                 next;
1707             }
1708             $res->{"m$opt"} = sprintf("%.3f", $bps / (1024*1024.0));
1709         }
1710     }
1711
1712     # can't use the schema's 'requires' because of the mbps* => bps* "transforming aliases"
1713     for my $requirement (
1714         [mbps_max => 'mbps'],
1715         [mbps_rd_max => 'mbps_rd'],
1716         [mbps_wr_max => 'mbps_wr'],
1717         [miops_max => 'miops'],
1718         [miops_rd_max => 'miops_rd'],
1719         [miops_wr_max => 'miops_wr'],
1720         [bps_max_length => 'mbps_max'],
1721         [bps_rd_max_length => 'mbps_rd_max'],
1722         [bps_wr_max_length => 'mbps_wr_max'],
1723         [iops_max_length => 'iops_max'],
1724         [iops_rd_max_length => 'iops_rd_max'],
1725         [iops_wr_max_length => 'iops_wr_max']) {
1726         my ($option, $requires) = @$requirement;
1727         if ($res->{$option} && !$res->{$requires}) {
1728             warn "$option requires $requires\n";
1729             ++$error;
1730         }
1731     }
1732
1733     return undef if $error;
1734
1735     return undef if $res->{mbps_rd} && $res->{mbps};
1736     return undef if $res->{mbps_wr} && $res->{mbps};
1737     return undef if $res->{iops_rd} && $res->{iops};
1738     return undef if $res->{iops_wr} && $res->{iops};
1739
1740     if ($res->{media} && ($res->{media} eq 'cdrom')) {
1741         return undef if $res->{snapshot} || $res->{trans} || $res->{format};
1742         return undef if $res->{heads} || $res->{secs} || $res->{cyls};
1743         return undef if $res->{interface} eq 'virtio';
1744     }
1745
1746     if (my $size = $res->{size}) {
1747         return undef if !defined($res->{size} = PVE::JSONSchema::parse_size($size));
1748     }
1749
1750     return $res;
1751 }
1752
1753 sub print_drive {
1754     my ($drive) = @_;
1755     my $data = { %$drive };
1756     delete $data->{$_} for qw(index interface);
1757     return PVE::JSONSchema::print_property_string($data, $alldrive_fmt);
1758 }
1759
1760 sub scsi_inquiry {
1761     my($fh, $noerr) = @_;
1762
1763     my $SG_IO = 0x2285;
1764     my $SG_GET_VERSION_NUM = 0x2282;
1765
1766     my $versionbuf = "\x00" x 8;
1767     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1768     if (!$ret) {
1769         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1770         return undef;
1771     }
1772     my $version = unpack("I", $versionbuf);
1773     if ($version < 30000) {
1774         die "scsi generic interface too old\n"  if !$noerr;
1775         return undef;
1776     }
1777
1778     my $buf = "\x00" x 36;
1779     my $sensebuf = "\x00" x 8;
1780     my $cmd = pack("C x3 C x1", 0x12, 36);
1781
1782     # see /usr/include/scsi/sg.h
1783     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1784
1785     my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1786                       length($sensebuf), 0, length($buf), $buf,
1787                       $cmd, $sensebuf, 6000);
1788
1789     $ret = ioctl($fh, $SG_IO, $packet);
1790     if (!$ret) {
1791         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1792         return undef;
1793     }
1794
1795     my @res = unpack($sg_io_hdr_t, $packet);
1796     if ($res[17] || $res[18]) {
1797         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1798         return undef;
1799     }
1800
1801     my $res = {};
1802     (my $byte0, my $byte1, $res->{vendor},
1803      $res->{product}, $res->{revision}) = unpack("C C x6 A8 A16 A4", $buf);
1804
1805     $res->{removable} = $byte1 & 128 ? 1 : 0;
1806     $res->{type} = $byte0 & 31;
1807
1808     return $res;
1809 }
1810
1811 sub path_is_scsi {
1812     my ($path) = @_;
1813
1814     my $fh = IO::File->new("+<$path") || return undef;
1815     my $res = scsi_inquiry($fh, 1);
1816     close($fh);
1817
1818     return $res;
1819 }
1820
1821 sub print_tabletdevice_full {
1822     my ($conf, $arch) = @_;
1823
1824     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
1825
1826     # we use uhci for old VMs because tablet driver was buggy in older qemu
1827     my $usbbus;
1828     if (PVE::QemuServer::Machine::machine_type_is_q35($conf) || $arch eq 'aarch64') {
1829         $usbbus = 'ehci';
1830     } else {
1831         $usbbus = 'uhci';
1832     }
1833
1834     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1835 }
1836
1837 sub print_keyboarddevice_full {
1838     my ($conf, $arch, $machine) = @_;
1839
1840     return undef if $arch ne 'aarch64';
1841
1842     return "usb-kbd,id=keyboard,bus=ehci.0,port=2";
1843 }
1844
1845 sub print_drivedevice_full {
1846     my ($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type) = @_;
1847
1848     my $device = '';
1849     my $maxdev = 0;
1850
1851     if ($drive->{interface} eq 'virtio') {
1852         my $pciaddr = print_pci_addr("$drive->{interface}$drive->{index}", $bridges, $arch, $machine_type);
1853         $device = "virtio-blk-pci,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}$pciaddr";
1854         $device .= ",iothread=iothread-$drive->{interface}$drive->{index}" if $drive->{iothread};
1855     } elsif ($drive->{interface} eq 'scsi') {
1856
1857         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1858         my $unit = $drive->{index} % $maxdev;
1859         my $devicetype = 'hd';
1860         my $path = '';
1861         if (drive_is_cdrom($drive)) {
1862             $devicetype = 'cd';
1863         } else {
1864             if ($drive->{file} =~ m|^/|) {
1865                 $path = $drive->{file};
1866                 if (my $info = path_is_scsi($path)) {
1867                     if ($info->{type} == 0 && $drive->{scsiblock}) {
1868                         $devicetype = 'block';
1869                     } elsif ($info->{type} == 1) { # tape
1870                         $devicetype = 'generic';
1871                     }
1872                 }
1873             } else {
1874                  $path = PVE::Storage::path($storecfg, $drive->{file});
1875             }
1876
1877             # for compatibility only, we prefer scsi-hd (#2408, #2355, #2380)
1878             my $version = PVE::QemuServer::Machine::extract_version($machine_type, kvm_user_version());
1879             if ($path =~ m/^iscsi\:\/\// &&
1880                !min_version($version, 4, 1)) {
1881                 $devicetype = 'generic';
1882             }
1883         }
1884
1885         if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)){
1886             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1887         } else {
1888             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0,lun=$drive->{index},drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1889         }
1890
1891         if ($drive->{ssd} && ($devicetype eq 'block' || $devicetype eq 'hd')) {
1892             $device .= ",rotation_rate=1";
1893         }
1894         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1895
1896     } elsif ($drive->{interface} eq 'ide' || $drive->{interface} eq 'sata') {
1897         my $maxdev = ($drive->{interface} eq 'sata') ? $MAX_SATA_DISKS : 2;
1898         my $controller = int($drive->{index} / $maxdev);
1899         my $unit = $drive->{index} % $maxdev;
1900         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1901
1902         $device = "ide-$devicetype";
1903         if ($drive->{interface} eq 'ide') {
1904             $device .= ",bus=ide.$controller,unit=$unit";
1905         } else {
1906             $device .= ",bus=ahci$controller.$unit";
1907         }
1908         $device .= ",drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1909
1910         if ($devicetype eq 'hd') {
1911             if (my $model = $drive->{model}) {
1912                 $model = URI::Escape::uri_unescape($model);
1913                 $device .= ",model=$model";
1914             }
1915             if ($drive->{ssd}) {
1916                 $device .= ",rotation_rate=1";
1917             }
1918         }
1919         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1920     } elsif ($drive->{interface} eq 'usb') {
1921         die "implement me";
1922         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1923     } else {
1924         die "unsupported interface type";
1925     }
1926
1927     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1928
1929     if (my $serial = $drive->{serial}) {
1930         $serial = URI::Escape::uri_unescape($serial);
1931         $device .= ",serial=$serial";
1932     }
1933
1934
1935     return $device;
1936 }
1937
1938 sub get_initiator_name {
1939     my $initiator;
1940
1941     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return undef;
1942     while (defined(my $line = <$fh>)) {
1943         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1944         $initiator = $1;
1945         last;
1946     }
1947     $fh->close();
1948
1949     return $initiator;
1950 }
1951
1952 sub print_drive_full {
1953     my ($storecfg, $vmid, $drive) = @_;
1954
1955     my $path;
1956     my $volid = $drive->{file};
1957     my $format;
1958
1959     if (drive_is_cdrom($drive)) {
1960         $path = get_iso_path($storecfg, $vmid, $volid);
1961     } else {
1962         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1963         if ($storeid) {
1964             $path = PVE::Storage::path($storecfg, $volid);
1965             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
1966             $format = qemu_img_format($scfg, $volname);
1967         } else {
1968             $path = $volid;
1969             $format = "raw";
1970         }
1971    }
1972
1973     my $opts = '';
1974     my @qemu_drive_options = qw(heads secs cyls trans media format cache rerror werror aio discard);
1975     foreach my $o (@qemu_drive_options) {
1976         $opts .= ",$o=$drive->{$o}" if defined($drive->{$o});
1977     }
1978
1979     # snapshot only accepts on|off
1980     if (defined($drive->{snapshot})) {
1981         my $v = $drive->{snapshot} ? 'on' : 'off';
1982         $opts .= ",snapshot=$v";
1983     }
1984
1985     foreach my $type (['', '-total'], [_rd => '-read'], [_wr => '-write']) {
1986         my ($dir, $qmpname) = @$type;
1987         if (my $v = $drive->{"mbps$dir"}) {
1988             $opts .= ",throttling.bps$qmpname=".int($v*1024*1024);
1989         }
1990         if (my $v = $drive->{"mbps${dir}_max"}) {
1991             $opts .= ",throttling.bps$qmpname-max=".int($v*1024*1024);
1992         }
1993         if (my $v = $drive->{"bps${dir}_max_length"}) {
1994             $opts .= ",throttling.bps$qmpname-max-length=$v";
1995         }
1996         if (my $v = $drive->{"iops${dir}"}) {
1997             $opts .= ",throttling.iops$qmpname=$v";
1998         }
1999         if (my $v = $drive->{"iops${dir}_max"}) {
2000             $opts .= ",throttling.iops$qmpname-max=$v";
2001         }
2002         if (my $v = $drive->{"iops${dir}_max_length"}) {
2003             $opts .= ",throttling.iops$qmpname-max-length=$v";
2004         }
2005     }
2006
2007     $opts .= ",format=$format" if $format && !$drive->{format};
2008
2009     my $cache_direct = 0;
2010
2011     if (my $cache = $drive->{cache}) {
2012         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
2013     } elsif (!drive_is_cdrom($drive)) {
2014         $opts .= ",cache=none";
2015         $cache_direct = 1;
2016     }
2017
2018     # aio native works only with O_DIRECT
2019     if (!$drive->{aio}) {
2020         if($cache_direct) {
2021             $opts .= ",aio=native";
2022         } else {
2023             $opts .= ",aio=threads";
2024         }
2025     }
2026
2027     if (!drive_is_cdrom($drive)) {
2028         my $detectzeroes;
2029         if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
2030             $detectzeroes = 'off';
2031         } elsif ($drive->{discard}) {
2032             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
2033         } else {
2034             # This used to be our default with discard not being specified:
2035             $detectzeroes = 'on';
2036         }
2037         $opts .= ",detect-zeroes=$detectzeroes" if $detectzeroes;
2038     }
2039
2040     my $pathinfo = $path ? "file=$path," : '';
2041
2042     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
2043 }
2044
2045 sub print_netdevice_full {
2046     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files, $arch, $machine_type) = @_;
2047
2048     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
2049
2050     my $device = $net->{model};
2051     if ($net->{model} eq 'virtio') {
2052          $device = 'virtio-net-pci';
2053      };
2054
2055     my $pciaddr = print_pci_addr("$netid", $bridges, $arch, $machine_type);
2056     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
2057     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
2058         #Consider we have N queues, the number of vectors needed is 2*N + 2 (plus one config interrupt and control vq)
2059         my $vectors = $net->{queues} * 2 + 2;
2060         $tmpstr .= ",vectors=$vectors,mq=on";
2061     }
2062     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
2063
2064     if ($use_old_bios_files) {
2065         my $romfile;
2066         if ($device eq 'virtio-net-pci') {
2067             $romfile = 'pxe-virtio.rom';
2068         } elsif ($device eq 'e1000') {
2069             $romfile = 'pxe-e1000.rom';
2070         } elsif ($device eq 'ne2k') {
2071             $romfile = 'pxe-ne2k_pci.rom';
2072         } elsif ($device eq 'pcnet') {
2073             $romfile = 'pxe-pcnet.rom';
2074         } elsif ($device eq 'rtl8139') {
2075             $romfile = 'pxe-rtl8139.rom';
2076         }
2077         $tmpstr .= ",romfile=$romfile" if $romfile;
2078     }
2079
2080     return $tmpstr;
2081 }
2082
2083 sub print_netdev_full {
2084     my ($vmid, $conf, $arch, $net, $netid, $hotplug) = @_;
2085
2086     my $i = '';
2087     if ($netid =~ m/^net(\d+)$/) {
2088         $i = int($1);
2089     }
2090
2091     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
2092
2093     my $ifname = "tap${vmid}i$i";
2094
2095     # kvm uses TUNSETIFF ioctl, and that limits ifname length
2096     die "interface name '$ifname' is too long (max 15 character)\n"
2097         if length($ifname) >= 16;
2098
2099     my $vhostparam = '';
2100     if (is_native($arch)) {
2101         $vhostparam = ',vhost=on' if kernel_has_vhost_net() && $net->{model} eq 'virtio';
2102     }
2103
2104     my $vmname = $conf->{name} || "vm$vmid";
2105
2106     my $netdev = "";
2107     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
2108
2109     if ($net->{bridge}) {
2110         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script,downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
2111     } else {
2112         $netdev = "type=user,id=$netid,hostname=$vmname";
2113     }
2114
2115     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
2116
2117     return $netdev;
2118 }
2119
2120
2121 sub print_cpu_device {
2122     my ($conf, $id) = @_;
2123
2124     my $kvm = $conf->{kvm} // 1;
2125     my $cpu = $kvm ? "kvm64" : "qemu64";
2126     if (my $cputype = $conf->{cpu}) {
2127         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
2128             or die "Cannot parse cpu description: $cputype\n";
2129         $cpu = $cpuconf->{cputype};
2130     }
2131
2132     my $cores = $conf->{cores} || 1;
2133
2134     my $current_core = ($id - 1) % $cores;
2135     my $current_socket = int(($id - 1 - $current_core)/$cores);
2136
2137     return "$cpu-x86_64-cpu,id=cpu$id,socket-id=$current_socket,core-id=$current_core,thread-id=0";
2138 }
2139
2140 my $vga_map = {
2141     'cirrus' => 'cirrus-vga',
2142     'std' => 'VGA',
2143     'vmware' => 'vmware-svga',
2144     'virtio' => 'virtio-vga',
2145 };
2146
2147 sub print_vga_device {
2148     my ($conf, $vga, $arch, $machine_version, $machine, $id, $qxlnum, $bridges) = @_;
2149
2150     my $type = $vga_map->{$vga->{type}};
2151     if ($arch eq 'aarch64' && defined($type) && $type eq 'virtio-vga') {
2152         $type = 'virtio-gpu';
2153     }
2154     my $vgamem_mb = $vga->{memory};
2155
2156     my $max_outputs = '';
2157     if ($qxlnum) {
2158         $type = $id ? 'qxl' : 'qxl-vga';
2159
2160         if (!$conf->{ostype} || $conf->{ostype} =~ m/^(?:l\d\d)|(?:other)$/) {
2161             # set max outputs so linux can have up to 4 qxl displays with one device
2162             if (min_version($machine_version, 4, 1)) {
2163                 $max_outputs = ",max_outputs=4";
2164             }
2165         }
2166     }
2167
2168     die "no devicetype for $vga->{type}\n" if !$type;
2169
2170     my $memory = "";
2171     if ($vgamem_mb) {
2172         if ($vga->{type} eq 'virtio') {
2173             my $bytes = PVE::Tools::convert_size($vgamem_mb, "mb" => "b");
2174             $memory = ",max_hostmem=$bytes";
2175         } elsif ($qxlnum) {
2176             # from https://www.spice-space.org/multiple-monitors.html
2177             $memory = ",vgamem_mb=$vga->{memory}";
2178             my $ram = $vgamem_mb * 4;
2179             my $vram = $vgamem_mb * 2;
2180             $memory .= ",ram_size_mb=$ram,vram_size_mb=$vram";
2181         } else {
2182             $memory = ",vgamem_mb=$vga->{memory}";
2183         }
2184     } elsif ($qxlnum && $id) {
2185         $memory = ",ram_size=67108864,vram_size=33554432";
2186     }
2187
2188     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
2189     my $vgaid = "vga" . ($id // '');
2190     my $pciaddr;
2191
2192     if ($q35 && $vgaid eq 'vga') {
2193         # the first display uses pcie.0 bus on q35 machines
2194         $pciaddr = print_pcie_addr($vgaid, $bridges, $arch, $machine);
2195     } else {
2196         $pciaddr = print_pci_addr($vgaid, $bridges, $arch, $machine);
2197     }
2198
2199     return "$type,id=${vgaid}${memory}${max_outputs}${pciaddr}";
2200 }
2201
2202 sub drive_is_cloudinit {
2203     my ($drive) = @_;
2204     return $drive->{file} =~ m@[:/]vm-\d+-cloudinit(?:\.$QEMU_FORMAT_RE)?$@;
2205 }
2206
2207 sub drive_is_cdrom {
2208     my ($drive, $exclude_cloudinit) = @_;
2209
2210     return 0 if $exclude_cloudinit && drive_is_cloudinit($drive);
2211
2212     return $drive && $drive->{media} && ($drive->{media} eq 'cdrom');
2213
2214 }
2215
2216 sub parse_number_sets {
2217     my ($set) = @_;
2218     my $res = [];
2219     foreach my $part (split(/;/, $set)) {
2220         if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
2221             die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
2222             push @$res, [ $1, $2 ];
2223         } else {
2224             die "invalid range: $part\n";
2225         }
2226     }
2227     return $res;
2228 }
2229
2230 sub parse_numa {
2231     my ($data) = @_;
2232
2233     my $res = PVE::JSONSchema::parse_property_string($numa_fmt, $data);
2234     $res->{cpus} = parse_number_sets($res->{cpus}) if defined($res->{cpus});
2235     $res->{hostnodes} = parse_number_sets($res->{hostnodes}) if defined($res->{hostnodes});
2236     return $res;
2237 }
2238
2239 sub parse_hostpci {
2240     my ($value) = @_;
2241
2242     return undef if !$value;
2243
2244     my $res = PVE::JSONSchema::parse_property_string($hostpci_fmt, $value);
2245
2246     my @idlist = split(/;/, $res->{host});
2247     delete $res->{host};
2248     foreach my $id (@idlist) {
2249         my $devs = PVE::SysFSTools::lspci($id);
2250         die "no PCI device found for '$id'\n" if !scalar(@$devs);
2251         push @{$res->{pciid}}, @$devs;
2252     }
2253     return $res;
2254 }
2255
2256 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
2257 sub parse_net {
2258     my ($data) = @_;
2259
2260     my $res = eval { PVE::JSONSchema::parse_property_string($net_fmt, $data) };
2261     if ($@) {
2262         warn $@;
2263         return undef;
2264     }
2265     if (!defined($res->{macaddr})) {
2266         my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
2267         $res->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix});
2268     }
2269     return $res;
2270 }
2271
2272 # ipconfigX ip=cidr,gw=ip,ip6=cidr,gw6=ip
2273 sub parse_ipconfig {
2274     my ($data) = @_;
2275
2276     my $res = eval { PVE::JSONSchema::parse_property_string($ipconfig_fmt, $data) };
2277     if ($@) {
2278         warn $@;
2279         return undef;
2280     }
2281
2282     if ($res->{gw} && !$res->{ip}) {
2283         warn 'gateway specified without specifying an IP address';
2284         return undef;
2285     }
2286     if ($res->{gw6} && !$res->{ip6}) {
2287         warn 'IPv6 gateway specified without specifying an IPv6 address';
2288         return undef;
2289     }
2290     if ($res->{gw} && $res->{ip} eq 'dhcp') {
2291         warn 'gateway specified together with DHCP';
2292         return undef;
2293     }
2294     if ($res->{gw6} && $res->{ip6} !~ /^$IPV6RE/) {
2295         # gw6 + auto/dhcp
2296         warn "IPv6 gateway specified together with $res->{ip6} address";
2297         return undef;
2298     }
2299
2300     if (!$res->{ip} && !$res->{ip6}) {
2301         return { ip => 'dhcp', ip6 => 'dhcp' };
2302     }
2303
2304     return $res;
2305 }
2306
2307 sub print_net {
2308     my $net = shift;
2309
2310     return PVE::JSONSchema::print_property_string($net, $net_fmt);
2311 }
2312
2313 sub add_random_macs {
2314     my ($settings) = @_;
2315
2316     foreach my $opt (keys %$settings) {
2317         next if $opt !~ m/^net(\d+)$/;
2318         my $net = parse_net($settings->{$opt});
2319         next if !$net;
2320         $settings->{$opt} = print_net($net);
2321     }
2322 }
2323
2324 sub vm_is_volid_owner {
2325     my ($storecfg, $vmid, $volid) = @_;
2326
2327     if ($volid !~  m|^/|) {
2328         my ($path, $owner);
2329         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
2330         if ($owner && ($owner == $vmid)) {
2331             return 1;
2332         }
2333     }
2334
2335     return undef;
2336 }
2337
2338 sub vmconfig_register_unused_drive {
2339     my ($storecfg, $vmid, $conf, $drive) = @_;
2340
2341     if (drive_is_cloudinit($drive)) {
2342         eval { PVE::Storage::vdisk_free($storecfg, $drive->{file}) };
2343         warn $@ if $@;
2344     } elsif (!drive_is_cdrom($drive)) {
2345         my $volid = $drive->{file};
2346         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
2347             PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
2348         }
2349     }
2350 }
2351
2352 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str][,base64=bool]
2353 my $smbios1_fmt = {
2354     uuid => {
2355         type => 'string',
2356         pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
2357         format_description => 'UUID',
2358         description => "Set SMBIOS1 UUID.",
2359         optional => 1,
2360     },
2361     version => {
2362         type => 'string',
2363         pattern => '[A-Za-z0-9+\/]+={0,2}',
2364         format_description => 'Base64 encoded string',
2365         description => "Set SMBIOS1 version.",
2366         optional => 1,
2367     },
2368     serial => {
2369         type => 'string',
2370         pattern => '[A-Za-z0-9+\/]+={0,2}',
2371         format_description => 'Base64 encoded string',
2372         description => "Set SMBIOS1 serial number.",
2373         optional => 1,
2374     },
2375     manufacturer => {
2376         type => 'string',
2377         pattern => '[A-Za-z0-9+\/]+={0,2}',
2378         format_description => 'Base64 encoded string',
2379         description => "Set SMBIOS1 manufacturer.",
2380         optional => 1,
2381     },
2382     product => {
2383         type => 'string',
2384         pattern => '[A-Za-z0-9+\/]+={0,2}',
2385         format_description => 'Base64 encoded string',
2386         description => "Set SMBIOS1 product ID.",
2387         optional => 1,
2388     },
2389     sku => {
2390         type => 'string',
2391         pattern => '[A-Za-z0-9+\/]+={0,2}',
2392         format_description => 'Base64 encoded string',
2393         description => "Set SMBIOS1 SKU string.",
2394         optional => 1,
2395     },
2396     family => {
2397         type => 'string',
2398         pattern => '[A-Za-z0-9+\/]+={0,2}',
2399         format_description => 'Base64 encoded string',
2400         description => "Set SMBIOS1 family string.",
2401         optional => 1,
2402     },
2403     base64 => {
2404         type => 'boolean',
2405         description => 'Flag to indicate that the SMBIOS values are base64 encoded',
2406         optional => 1,
2407     },
2408 };
2409
2410 sub parse_smbios1 {
2411     my ($data) = @_;
2412
2413     my $res = eval { PVE::JSONSchema::parse_property_string($smbios1_fmt, $data) };
2414     warn $@ if $@;
2415     return $res;
2416 }
2417
2418 sub print_smbios1 {
2419     my ($smbios1) = @_;
2420     return PVE::JSONSchema::print_property_string($smbios1, $smbios1_fmt);
2421 }
2422
2423 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_fmt);
2424
2425 PVE::JSONSchema::register_format('pve-qm-bootdisk', \&verify_bootdisk);
2426 sub verify_bootdisk {
2427     my ($value, $noerr) = @_;
2428
2429     return $value if is_valid_drivename($value);
2430
2431     return undef if $noerr;
2432
2433     die "invalid boot disk '$value'\n";
2434 }
2435
2436 sub parse_watchdog {
2437     my ($value) = @_;
2438
2439     return undef if !$value;
2440
2441     my $res = eval { PVE::JSONSchema::parse_property_string($watchdog_fmt, $value) };
2442     warn $@ if $@;
2443     return $res;
2444 }
2445
2446 sub parse_guest_agent {
2447     my ($value) = @_;
2448
2449     return {} if !defined($value->{agent});
2450
2451     my $res = eval { PVE::JSONSchema::parse_property_string($agent_fmt, $value->{agent}) };
2452     warn $@ if $@;
2453
2454     # if the agent is disabled ignore the other potentially set properties
2455     return {} if !$res->{enabled};
2456     return $res;
2457 }
2458
2459 sub parse_vga {
2460     my ($value) = @_;
2461
2462     return {} if !$value;
2463     my $res = eval { PVE::JSONSchema::parse_property_string($vga_fmt, $value) };
2464     warn $@ if $@;
2465     return $res;
2466 }
2467
2468 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
2469 sub verify_usb_device {
2470     my ($value, $noerr) = @_;
2471
2472     return $value if parse_usb_device($value);
2473
2474     return undef if $noerr;
2475
2476     die "unable to parse usb device\n";
2477 }
2478
2479 # add JSON properties for create and set function
2480 sub json_config_properties {
2481     my $prop = shift;
2482
2483     foreach my $opt (keys %$confdesc) {
2484         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate' || $opt eq 'runningmachine';
2485         $prop->{$opt} = $confdesc->{$opt};
2486     }
2487
2488     return $prop;
2489 }
2490
2491 # return copy of $confdesc_cloudinit to generate documentation
2492 sub cloudinit_config_properties {
2493
2494     return dclone($confdesc_cloudinit);
2495 }
2496
2497 sub check_type {
2498     my ($key, $value) = @_;
2499
2500     die "unknown setting '$key'\n" if !$confdesc->{$key};
2501
2502     my $type = $confdesc->{$key}->{type};
2503
2504     if (!defined($value)) {
2505         die "got undefined value\n";
2506     }
2507
2508     if ($value =~ m/[\n\r]/) {
2509         die "property contains a line feed\n";
2510     }
2511
2512     if ($type eq 'boolean') {
2513         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
2514         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
2515         die "type check ('boolean') failed - got '$value'\n";
2516     } elsif ($type eq 'integer') {
2517         return int($1) if $value =~ m/^(\d+)$/;
2518         die "type check ('integer') failed - got '$value'\n";
2519     } elsif ($type eq 'number') {
2520         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
2521         die "type check ('number') failed - got '$value'\n";
2522     } elsif ($type eq 'string') {
2523         if (my $fmt = $confdesc->{$key}->{format}) {
2524             PVE::JSONSchema::check_format($fmt, $value);
2525             return $value;
2526         }
2527         $value =~ s/^\"(.*)\"$/$1/;
2528         return $value;
2529     } else {
2530         die "internal error"
2531     }
2532 }
2533
2534 sub destroy_vm {
2535     my ($storecfg, $vmid, $skiplock, $replacement_conf) = @_;
2536
2537     my $conf = PVE::QemuConfig->load_config($vmid);
2538
2539     PVE::QemuConfig->check_lock($conf) if !$skiplock;
2540
2541     if ($conf->{template}) {
2542         # check if any base image is still used by a linked clone
2543         foreach_drive($conf, sub {
2544                 my ($ds, $drive) = @_;
2545                 return if drive_is_cdrom($drive);
2546
2547                 my $volid = $drive->{file};
2548                 return if !$volid || $volid =~ m|^/|;
2549
2550                 die "base volume '$volid' is still in use by linked cloned\n"
2551                     if PVE::Storage::volume_is_base_and_used($storecfg, $volid);
2552
2553         });
2554     }
2555
2556     # only remove disks owned by this VM
2557     foreach_drive($conf, sub {
2558         my ($ds, $drive) = @_;
2559         return if drive_is_cdrom($drive, 1);
2560
2561         my $volid = $drive->{file};
2562         return if !$volid || $volid =~ m|^/|;
2563
2564         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
2565         return if !$path || !$owner || ($owner != $vmid);
2566
2567         eval { PVE::Storage::vdisk_free($storecfg, $volid) };
2568         warn "Could not remove disk '$volid', check manually: $@" if $@;
2569     });
2570
2571     # also remove unused disk
2572     my $vmdisks = PVE::Storage::vdisk_list($storecfg, undef, $vmid);
2573     PVE::Storage::foreach_volid($vmdisks, sub {
2574         my ($volid, $sid, $volname, $d) = @_;
2575         eval { PVE::Storage::vdisk_free($storecfg, $volid) };
2576         warn $@ if $@;
2577     });
2578
2579     if (defined $replacement_conf) {
2580         PVE::QemuConfig->write_config($vmid, $replacement_conf);
2581     } else {
2582         PVE::QemuConfig->destroy_config($vmid);
2583     }
2584 }
2585
2586 sub parse_vm_config {
2587     my ($filename, $raw) = @_;
2588
2589     return undef if !defined($raw);
2590
2591     my $res = {
2592         digest => Digest::SHA::sha1_hex($raw),
2593         snapshots => {},
2594         pending => {},
2595     };
2596
2597     $filename =~ m|/qemu-server/(\d+)\.conf$|
2598         || die "got strange filename '$filename'";
2599
2600     my $vmid = $1;
2601
2602     my $conf = $res;
2603     my $descr;
2604     my $section = '';
2605
2606     my @lines = split(/\n/, $raw);
2607     foreach my $line (@lines) {
2608         next if $line =~ m/^\s*$/;
2609
2610         if ($line =~ m/^\[PENDING\]\s*$/i) {
2611             $section = 'pending';
2612             if (defined($descr)) {
2613                 $descr =~ s/\s+$//;
2614                 $conf->{description} = $descr;
2615             }
2616             $descr = undef;
2617             $conf = $res->{$section} = {};
2618             next;
2619
2620         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2621             $section = $1;
2622             if (defined($descr)) {
2623                 $descr =~ s/\s+$//;
2624                 $conf->{description} = $descr;
2625             }
2626             $descr = undef;
2627             $conf = $res->{snapshots}->{$section} = {};
2628             next;
2629         }
2630
2631         if ($line =~ m/^\#(.*)\s*$/) {
2632             $descr = '' if !defined($descr);
2633             $descr .= PVE::Tools::decode_text($1) . "\n";
2634             next;
2635         }
2636
2637         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2638             $descr = '' if !defined($descr);
2639             $descr .= PVE::Tools::decode_text($2);
2640         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2641             $conf->{snapstate} = $1;
2642         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2643             my $key = $1;
2644             my $value = $2;
2645             $conf->{$key} = $value;
2646         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2647             my $value = $1;
2648             if ($section eq 'pending') {
2649                 $conf->{delete} = $value; # we parse this later
2650             } else {
2651                 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
2652             }
2653         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(.+?)\s*$/) {
2654             my $key = $1;
2655             my $value = $2;
2656             eval { $value = check_type($key, $value); };
2657             if ($@) {
2658                 warn "vm $vmid - unable to parse value of '$key' - $@";
2659             } else {
2660                 $key = 'ide2' if $key eq 'cdrom';
2661                 my $fmt = $confdesc->{$key}->{format};
2662                 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2663                     my $v = parse_drive($key, $value);
2664                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2665                         $v->{file} = $volid;
2666                         $value = print_drive($v);
2667                     } else {
2668                         warn "vm $vmid - unable to parse value of '$key'\n";
2669                         next;
2670                     }
2671                 }
2672
2673                 $conf->{$key} = $value;
2674             }
2675         }
2676     }
2677
2678     if (defined($descr)) {
2679         $descr =~ s/\s+$//;
2680         $conf->{description} = $descr;
2681     }
2682     delete $res->{snapstate}; # just to be sure
2683
2684     return $res;
2685 }
2686
2687 sub write_vm_config {
2688     my ($filename, $conf) = @_;
2689
2690     delete $conf->{snapstate}; # just to be sure
2691
2692     if ($conf->{cdrom}) {
2693         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2694         $conf->{ide2} = $conf->{cdrom};
2695         delete $conf->{cdrom};
2696     }
2697
2698     # we do not use 'smp' any longer
2699     if ($conf->{sockets}) {
2700         delete $conf->{smp};
2701     } elsif ($conf->{smp}) {
2702         $conf->{sockets} = $conf->{smp};
2703         delete $conf->{cores};
2704         delete $conf->{smp};
2705     }
2706
2707     my $used_volids = {};
2708
2709     my $cleanup_config = sub {
2710         my ($cref, $pending, $snapname) = @_;
2711
2712         foreach my $key (keys %$cref) {
2713             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2714                 $key eq 'snapstate' || $key eq 'pending';
2715             my $value = $cref->{$key};
2716             if ($key eq 'delete') {
2717                 die "propertry 'delete' is only allowed in [PENDING]\n"
2718                     if !$pending;
2719                 # fixme: check syntax?
2720                 next;
2721             }
2722             eval { $value = check_type($key, $value); };
2723             die "unable to parse value of '$key' - $@" if $@;
2724
2725             $cref->{$key} = $value;
2726
2727             if (!$snapname && is_valid_drivename($key)) {
2728                 my $drive = parse_drive($key, $value);
2729                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2730             }
2731         }
2732     };
2733
2734     &$cleanup_config($conf);
2735
2736     &$cleanup_config($conf->{pending}, 1);
2737
2738     foreach my $snapname (keys %{$conf->{snapshots}}) {
2739         die "internal error: snapshot name '$snapname' is forbidden" if lc($snapname) eq 'pending';
2740         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2741     }
2742
2743     # remove 'unusedX' settings if we re-add a volume
2744     foreach my $key (keys %$conf) {
2745         my $value = $conf->{$key};
2746         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2747             delete $conf->{$key};
2748         }
2749     }
2750
2751     my $generate_raw_config = sub {
2752         my ($conf, $pending) = @_;
2753
2754         my $raw = '';
2755
2756         # add description as comment to top of file
2757         if (defined(my $descr = $conf->{description})) {
2758             if ($descr) {
2759                 foreach my $cl (split(/\n/, $descr)) {
2760                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2761                 }
2762             } else {
2763                 $raw .= "#\n" if $pending;
2764             }
2765         }
2766
2767         foreach my $key (sort keys %$conf) {
2768             next if $key eq 'digest' || $key eq 'description' || $key eq 'pending' || $key eq 'snapshots';
2769             $raw .= "$key: $conf->{$key}\n";
2770         }
2771         return $raw;
2772     };
2773
2774     my $raw = &$generate_raw_config($conf);
2775
2776     if (scalar(keys %{$conf->{pending}})){
2777         $raw .= "\n[PENDING]\n";
2778         $raw .= &$generate_raw_config($conf->{pending}, 1);
2779     }
2780
2781     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2782         $raw .= "\n[$snapname]\n";
2783         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2784     }
2785
2786     return $raw;
2787 }
2788
2789 sub load_defaults {
2790
2791     my $res = {};
2792
2793     # we use static defaults from our JSON schema configuration
2794     foreach my $key (keys %$confdesc) {
2795         if (defined(my $default = $confdesc->{$key}->{default})) {
2796             $res->{$key} = $default;
2797         }
2798     }
2799
2800     return $res;
2801 }
2802
2803 sub config_list {
2804     my $vmlist = PVE::Cluster::get_vmlist();
2805     my $res = {};
2806     return $res if !$vmlist || !$vmlist->{ids};
2807     my $ids = $vmlist->{ids};
2808
2809     foreach my $vmid (keys %$ids) {
2810         my $d = $ids->{$vmid};
2811         next if !$d->{node} || $d->{node} ne $nodename;
2812         next if !$d->{type} || $d->{type} ne 'qemu';
2813         $res->{$vmid}->{exists} = 1;
2814     }
2815     return $res;
2816 }
2817
2818 # test if VM uses local resources (to prevent migration)
2819 sub check_local_resources {
2820     my ($conf, $noerr) = @_;
2821
2822     my @loc_res = ();
2823
2824     push @loc_res, "hostusb" if $conf->{hostusb}; # old syntax
2825     push @loc_res, "hostpci" if $conf->{hostpci}; # old syntax
2826
2827     push @loc_res, "ivshmem" if $conf->{ivshmem};
2828
2829     foreach my $k (keys %$conf) {
2830         next if $k =~ m/^usb/ && ($conf->{$k} =~ m/^spice(?![^,])/);
2831         # sockets are safe: they will recreated be on the target side post-migrate
2832         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2833         push @loc_res, $k if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2834     }
2835
2836     die "VM uses local resources\n" if scalar @loc_res && !$noerr;
2837
2838     return \@loc_res;
2839 }
2840
2841 # check if used storages are available on all nodes (use by migrate)
2842 sub check_storage_availability {
2843     my ($storecfg, $conf, $node) = @_;
2844
2845     foreach_drive($conf, sub {
2846         my ($ds, $drive) = @_;
2847
2848         my $volid = $drive->{file};
2849         return if !$volid;
2850
2851         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2852         return if !$sid;
2853
2854         # check if storage is available on both nodes
2855         my $scfg = PVE::Storage::storage_check_node($storecfg, $sid);
2856         PVE::Storage::storage_check_node($storecfg, $sid, $node);
2857    });
2858 }
2859
2860 # list nodes where all VM images are available (used by has_feature API)
2861 sub shared_nodes {
2862     my ($conf, $storecfg) = @_;
2863
2864     my $nodelist = PVE::Cluster::get_nodelist();
2865     my $nodehash = { map { $_ => 1 } @$nodelist };
2866     my $nodename = PVE::INotify::nodename();
2867
2868     foreach_drive($conf, sub {
2869         my ($ds, $drive) = @_;
2870
2871         my $volid = $drive->{file};
2872         return if !$volid;
2873
2874         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2875         if ($storeid) {
2876             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2877             if ($scfg->{disable}) {
2878                 $nodehash = {};
2879             } elsif (my $avail = $scfg->{nodes}) {
2880                 foreach my $node (keys %$nodehash) {
2881                     delete $nodehash->{$node} if !$avail->{$node};
2882                 }
2883             } elsif (!$scfg->{shared}) {
2884                 foreach my $node (keys %$nodehash) {
2885                     delete $nodehash->{$node} if $node ne $nodename
2886                 }
2887             }
2888         }
2889     });
2890
2891     return $nodehash
2892 }
2893
2894 sub check_local_storage_availability {
2895     my ($conf, $storecfg) = @_;
2896
2897     my $nodelist = PVE::Cluster::get_nodelist();
2898     my $nodehash = { map { $_ => {} } @$nodelist };
2899
2900     foreach_drive($conf, sub {
2901         my ($ds, $drive) = @_;
2902
2903         my $volid = $drive->{file};
2904         return if !$volid;
2905
2906         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2907         if ($storeid) {
2908             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2909
2910             if ($scfg->{disable}) {
2911                 foreach my $node (keys %$nodehash) {
2912                     $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2913                 }
2914             } elsif (my $avail = $scfg->{nodes}) {
2915                 foreach my $node (keys %$nodehash) {
2916                     if (!$avail->{$node}) {
2917                         $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2918                     }
2919                 }
2920             }
2921         }
2922     });
2923
2924     foreach my $node (values %$nodehash) {
2925         if (my $unavail = $node->{unavailable_storages}) {
2926             $node->{unavailable_storages} = [ sort keys %$unavail ];
2927         }
2928     }
2929
2930     return $nodehash
2931 }
2932
2933 # Compat only, use assert_config_exists_on_node and vm_running_locally where possible
2934 sub check_running {
2935     my ($vmid, $nocheck, $node) = @_;
2936
2937     PVE::QemuConfig::assert_config_exists_on_node($vmid, $node) if !$nocheck;
2938     return PVE::QemuServer::Helpers::vm_running_locally($vmid);
2939 }
2940
2941 sub vzlist {
2942
2943     my $vzlist = config_list();
2944
2945     my $fd = IO::Dir->new($PVE::QemuServer::Helpers::var_run_tmpdir) || return $vzlist;
2946
2947     while (defined(my $de = $fd->read)) {
2948         next if $de !~ m/^(\d+)\.pid$/;
2949         my $vmid = $1;
2950         next if !defined($vzlist->{$vmid});
2951         if (my $pid = check_running($vmid)) {
2952             $vzlist->{$vmid}->{pid} = $pid;
2953         }
2954     }
2955
2956     return $vzlist;
2957 }
2958
2959 sub disksize {
2960     my ($storecfg, $conf) = @_;
2961
2962     my $bootdisk = $conf->{bootdisk};
2963     return undef if !$bootdisk;
2964     return undef if !is_valid_drivename($bootdisk);
2965
2966     return undef if !$conf->{$bootdisk};
2967
2968     my $drive = parse_drive($bootdisk, $conf->{$bootdisk});
2969     return undef if !defined($drive);
2970
2971     return undef if drive_is_cdrom($drive);
2972
2973     my $volid = $drive->{file};
2974     return undef if !$volid;
2975
2976     return $drive->{size};
2977 }
2978
2979 our $vmstatus_return_properties = {
2980     vmid => get_standard_option('pve-vmid'),
2981     status => {
2982         description => "Qemu process status.",
2983         type => 'string',
2984         enum => ['stopped', 'running'],
2985     },
2986     maxmem => {
2987         description => "Maximum memory in bytes.",
2988         type => 'integer',
2989         optional => 1,
2990         renderer => 'bytes',
2991     },
2992     maxdisk => {
2993         description => "Root disk size in bytes.",
2994         type => 'integer',
2995         optional => 1,
2996         renderer => 'bytes',
2997     },
2998     name => {
2999         description => "VM name.",
3000         type => 'string',
3001         optional => 1,
3002     },
3003     qmpstatus => {
3004         description => "Qemu QMP agent status.",
3005         type => 'string',
3006         optional => 1,
3007     },
3008     pid => {
3009         description => "PID of running qemu process.",
3010         type => 'integer',
3011         optional => 1,
3012     },
3013     uptime => {
3014         description => "Uptime.",
3015         type => 'integer',
3016         optional => 1,
3017         renderer => 'duration',
3018     },
3019     cpus => {
3020         description => "Maximum usable CPUs.",
3021         type => 'number',
3022         optional => 1,
3023     },
3024     lock => {
3025         description => "The current config lock, if any.",
3026         type => 'string',
3027         optional => 1,
3028     },
3029     tags => {
3030         description  => "The current configured tags, if any",
3031         type => 'string',
3032         optional => 1,
3033     },
3034 };
3035
3036 my $last_proc_pid_stat;
3037
3038 # get VM status information
3039 # This must be fast and should not block ($full == false)
3040 # We only query KVM using QMP if $full == true (this can be slow)
3041 sub vmstatus {
3042     my ($opt_vmid, $full) = @_;
3043
3044     my $res = {};
3045
3046     my $storecfg = PVE::Storage::config();
3047
3048     my $list = vzlist();
3049     my $defaults = load_defaults();
3050
3051     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
3052
3053     my $cpucount = $cpuinfo->{cpus} || 1;
3054
3055     foreach my $vmid (keys %$list) {
3056         next if $opt_vmid && ($vmid ne $opt_vmid);
3057
3058         my $conf = PVE::QemuConfig->load_config($vmid);
3059
3060         my $d = { vmid => $vmid };
3061         $d->{pid} = $list->{$vmid}->{pid};
3062
3063         # fixme: better status?
3064         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
3065
3066         my $size = disksize($storecfg, $conf);
3067         if (defined($size)) {
3068             $d->{disk} = 0; # no info available
3069             $d->{maxdisk} = $size;
3070         } else {
3071             $d->{disk} = 0;
3072             $d->{maxdisk} = 0;
3073         }
3074
3075         $d->{cpus} = ($conf->{sockets} || $defaults->{sockets})
3076             * ($conf->{cores} || $defaults->{cores});
3077         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
3078         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
3079
3080         $d->{name} = $conf->{name} || "VM $vmid";
3081         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024)
3082             : $defaults->{memory}*(1024*1024);
3083
3084         if ($conf->{balloon}) {
3085             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
3086             $d->{shares} = defined($conf->{shares}) ? $conf->{shares}
3087                 : $defaults->{shares};
3088         }
3089
3090         $d->{uptime} = 0;
3091         $d->{cpu} = 0;
3092         $d->{mem} = 0;
3093
3094         $d->{netout} = 0;
3095         $d->{netin} = 0;
3096
3097         $d->{diskread} = 0;
3098         $d->{diskwrite} = 0;
3099
3100         $d->{template} = PVE::QemuConfig->is_template($conf);
3101
3102         $d->{serial} = 1 if conf_has_serial($conf);
3103         $d->{lock} = $conf->{lock} if $conf->{lock};
3104         $d->{tags} = $conf->{tags} if defined($conf->{tags});
3105
3106         $res->{$vmid} = $d;
3107     }
3108
3109     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
3110     foreach my $dev (keys %$netdev) {
3111         next if $dev !~ m/^tap([1-9]\d*)i/;
3112         my $vmid = $1;
3113         my $d = $res->{$vmid};
3114         next if !$d;
3115
3116         $d->{netout} += $netdev->{$dev}->{receive};
3117         $d->{netin} += $netdev->{$dev}->{transmit};
3118
3119         if ($full) {
3120             $d->{nics}->{$dev}->{netout} = $netdev->{$dev}->{receive};
3121             $d->{nics}->{$dev}->{netin} = $netdev->{$dev}->{transmit};
3122         }
3123
3124     }
3125
3126     my $ctime = gettimeofday;
3127
3128     foreach my $vmid (keys %$list) {
3129
3130         my $d = $res->{$vmid};
3131         my $pid = $d->{pid};
3132         next if !$pid;
3133
3134         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
3135         next if !$pstat; # not running
3136
3137         my $used = $pstat->{utime} + $pstat->{stime};
3138
3139         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
3140
3141         if ($pstat->{vsize}) {
3142             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
3143         }
3144
3145         my $old = $last_proc_pid_stat->{$pid};
3146         if (!$old) {
3147             $last_proc_pid_stat->{$pid} = {
3148                 time => $ctime,
3149                 used => $used,
3150                 cpu => 0,
3151             };
3152             next;
3153         }
3154
3155         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
3156
3157         if ($dtime > 1000) {
3158             my $dutime = $used -  $old->{used};
3159
3160             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
3161             $last_proc_pid_stat->{$pid} = {
3162                 time => $ctime,
3163                 used => $used,
3164                 cpu => $d->{cpu},
3165             };
3166         } else {
3167             $d->{cpu} = $old->{cpu};
3168         }
3169     }
3170
3171     return $res if !$full;
3172
3173     my $qmpclient = PVE::QMPClient->new();
3174
3175     my $ballooncb = sub {
3176         my ($vmid, $resp) = @_;
3177
3178         my $info = $resp->{'return'};
3179         return if !$info->{max_mem};
3180
3181         my $d = $res->{$vmid};
3182
3183         # use memory assigned to VM
3184         $d->{maxmem} = $info->{max_mem};
3185         $d->{balloon} = $info->{actual};
3186
3187         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
3188             $d->{mem} = $info->{total_mem} - $info->{free_mem};
3189             $d->{freemem} = $info->{free_mem};
3190         }
3191
3192         $d->{ballooninfo} = $info;
3193     };
3194
3195     my $blockstatscb = sub {
3196         my ($vmid, $resp) = @_;
3197         my $data = $resp->{'return'} || [];
3198         my $totalrdbytes = 0;
3199         my $totalwrbytes = 0;
3200
3201         for my $blockstat (@$data) {
3202             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
3203             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
3204
3205             $blockstat->{device} =~ s/drive-//;
3206             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
3207         }
3208         $res->{$vmid}->{diskread} = $totalrdbytes;
3209         $res->{$vmid}->{diskwrite} = $totalwrbytes;
3210     };
3211
3212     my $statuscb = sub {
3213         my ($vmid, $resp) = @_;
3214
3215         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
3216         # this fails if ballon driver is not loaded, so this must be
3217         # the last commnand (following command are aborted if this fails).
3218         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
3219
3220         my $status = 'unknown';
3221         if (!defined($status = $resp->{'return'}->{status})) {
3222             warn "unable to get VM status\n";
3223             return;
3224         }
3225
3226         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
3227     };
3228
3229     foreach my $vmid (keys %$list) {
3230         next if $opt_vmid && ($vmid ne $opt_vmid);
3231         next if !$res->{$vmid}->{pid}; # not running
3232         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
3233     }
3234
3235     $qmpclient->queue_execute(undef, 2);
3236
3237     foreach my $vmid (keys %$list) {
3238         next if $opt_vmid && ($vmid ne $opt_vmid);
3239         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
3240     }
3241
3242     return $res;
3243 }
3244
3245 sub foreach_drive {
3246     my ($conf, $func, @param) = @_;
3247
3248     foreach my $ds (valid_drive_names()) {
3249         next if !defined($conf->{$ds});
3250
3251         my $drive = parse_drive($ds, $conf->{$ds});
3252         next if !$drive;
3253
3254         &$func($ds, $drive, @param);
3255     }
3256 }
3257
3258 sub foreach_volid {
3259     my ($conf, $func, @param) = @_;
3260
3261     my $volhash = {};
3262
3263     my $test_volid = sub {
3264         my ($volid, $is_cdrom, $replicate, $shared, $snapname, $size) = @_;
3265
3266         return if !$volid;
3267
3268         $volhash->{$volid}->{cdrom} //= 1;
3269         $volhash->{$volid}->{cdrom} = 0 if !$is_cdrom;
3270
3271         $volhash->{$volid}->{replicate} //= 0;
3272         $volhash->{$volid}->{replicate} = 1 if $replicate;
3273
3274         $volhash->{$volid}->{shared} //= 0;
3275         $volhash->{$volid}->{shared} = 1 if $shared;
3276
3277         $volhash->{$volid}->{referenced_in_config} //= 0;
3278         $volhash->{$volid}->{referenced_in_config} = 1 if !defined($snapname);
3279
3280         $volhash->{$volid}->{referenced_in_snapshot}->{$snapname} = 1
3281             if defined($snapname);
3282         $volhash->{$volid}->{size} = $size if $size;
3283     };
3284
3285     foreach_drive($conf, sub {
3286         my ($ds, $drive) = @_;
3287         $test_volid->($drive->{file}, drive_is_cdrom($drive), $drive->{replicate} // 1, $drive->{shared}, undef, $drive->{size});
3288     });
3289
3290     foreach my $snapname (keys %{$conf->{snapshots}}) {
3291         my $snap = $conf->{snapshots}->{$snapname};
3292         $test_volid->($snap->{vmstate}, 0, 1, $snapname);
3293         foreach_drive($snap, sub {
3294             my ($ds, $drive) = @_;
3295             $test_volid->($drive->{file}, drive_is_cdrom($drive), $drive->{replicate} // 1, $drive->{shared}, $snapname);
3296         });
3297     }
3298
3299     foreach my $volid (keys %$volhash) {
3300         &$func($volid, $volhash->{$volid}, @param);
3301     }
3302 }
3303
3304 sub conf_has_serial {
3305     my ($conf) = @_;
3306
3307     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3308         if ($conf->{"serial$i"}) {
3309             return 1;
3310         }
3311     }
3312
3313     return 0;
3314 }
3315
3316 sub conf_has_audio {
3317     my ($conf, $id) = @_;
3318
3319     $id //= 0;
3320     my $audio = $conf->{"audio$id"};
3321     return undef if !defined($audio);
3322
3323     my $audioproperties = PVE::JSONSchema::parse_property_string($audio_fmt, $audio);
3324     my $audiodriver = $audioproperties->{driver} // 'spice';
3325
3326     return {
3327         dev => $audioproperties->{device},
3328         dev_id => "audiodev$id",
3329         backend => $audiodriver,
3330         backend_id => "$audiodriver-backend${id}",
3331     };
3332 }
3333
3334 sub vga_conf_has_spice {
3335     my ($vga) = @_;
3336
3337     my $vgaconf = parse_vga($vga);
3338     my $vgatype = $vgaconf->{type};
3339     return 0 if !$vgatype || $vgatype !~ m/^qxl([234])?$/;
3340
3341     return $1 || 1;
3342 }
3343
3344 sub is_native($) {
3345     my ($arch) = @_;
3346     return get_host_arch() eq $arch;
3347 }
3348
3349 sub get_vm_arch {
3350     my ($conf) = @_;
3351     return $conf->{arch} // get_host_arch();
3352 }
3353
3354 my $default_machines = {
3355     x86_64 => 'pc',
3356     aarch64 => 'virt',
3357 };
3358
3359 sub get_vm_machine {
3360     my ($conf, $forcemachine, $arch, $add_pve_version) = @_;
3361
3362     my $machine = $forcemachine || $conf->{machine};
3363
3364     if (!$machine || $machine =~ m/^(?:pc|q35|virt)$/) {
3365         $arch //= 'x86_64';
3366         $machine ||= $default_machines->{$arch};
3367         $machine .= "+pve$PVE::QemuServer::Machine::PVE_MACHINE_VERSION" if $add_pve_version;
3368     }
3369
3370     return $machine;
3371 }
3372
3373 sub get_ovmf_files($) {
3374     my ($arch) = @_;
3375
3376     my $ovmf = $OVMF->{$arch}
3377         or die "no OVMF images known for architecture '$arch'\n";
3378
3379     return @$ovmf;
3380 }
3381
3382 my $Arch2Qemu = {
3383     aarch64 => '/usr/bin/qemu-system-aarch64',
3384     x86_64 => '/usr/bin/qemu-system-x86_64',
3385 };
3386 sub get_command_for_arch($) {
3387     my ($arch) = @_;
3388     return '/usr/bin/kvm' if is_native($arch);
3389
3390     my $cmd = $Arch2Qemu->{$arch}
3391         or die "don't know how to emulate architecture '$arch'\n";
3392     return $cmd;
3393 }
3394
3395 sub get_cpu_options {
3396     my ($conf, $arch, $kvm, $kvm_off, $machine_version, $winversion, $gpu_passthrough) = @_;
3397
3398     my $cpuFlags = [];
3399     my $ostype = $conf->{ostype};
3400
3401     my $cpu = $kvm ? "kvm64" : "qemu64";
3402     if ($arch eq 'aarch64') {
3403         $cpu = 'cortex-a57';
3404     }
3405     my $hv_vendor_id;
3406     if (my $cputype = $conf->{cpu}) {
3407         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
3408             or die "Cannot parse cpu description: $cputype\n";
3409         $cpu = $cpuconf->{cputype};
3410         $kvm_off = 1 if $cpuconf->{hidden};
3411         $hv_vendor_id = $cpuconf->{'hv-vendor-id'};
3412
3413         if (defined(my $flags = $cpuconf->{flags})) {
3414             push @$cpuFlags, split(";", $flags);
3415         }
3416     }
3417
3418     push @$cpuFlags , '+lahf_lm' if $cpu eq 'kvm64' && $arch eq 'x86_64';
3419
3420     push @$cpuFlags , '-x2apic' if $ostype && $ostype eq 'solaris';
3421
3422     push @$cpuFlags, '+sep' if $cpu eq 'kvm64' || $cpu eq 'kvm32';
3423
3424     push @$cpuFlags, '-rdtscp' if $cpu =~ m/^Opteron/;
3425
3426     if (min_version($machine_version, 2, 3) && $arch eq 'x86_64') {
3427
3428         push @$cpuFlags , '+kvm_pv_unhalt' if $kvm;
3429         push @$cpuFlags , '+kvm_pv_eoi' if $kvm;
3430     }
3431
3432     add_hyperv_enlightenments($cpuFlags, $winversion, $machine_version, $conf->{bios}, $gpu_passthrough, $hv_vendor_id) if $kvm;
3433
3434     push @$cpuFlags, 'enforce' if $cpu ne 'host' && $kvm && $arch eq 'x86_64';
3435
3436     push @$cpuFlags, 'kvm=off' if $kvm_off;
3437
3438     if (my $cpu_vendor = $cpu_vendor_list->{$cpu}) {
3439         push @$cpuFlags, "vendor=${cpu_vendor}"
3440             if $cpu_vendor ne 'default';
3441     } elsif ($arch ne 'aarch64') {
3442         die "internal error"; # should not happen
3443     }
3444
3445     $cpu .= "," . join(',', @$cpuFlags) if scalar(@$cpuFlags);
3446
3447     return ('-cpu', $cpu);
3448 }
3449
3450 sub config_to_command {
3451     my ($storecfg, $vmid, $conf, $defaults, $forcemachine) = @_;
3452
3453     my $cmd = [];
3454     my $globalFlags = [];
3455     my $machineFlags = [];
3456     my $rtcFlags = [];
3457     my $devices = [];
3458     my $pciaddr = '';
3459     my $bridges = {};
3460     my $vernum = 0; # unknown
3461     my $ostype = $conf->{ostype};
3462     my $winversion = windows_version($ostype);
3463     my $kvm = $conf->{kvm};
3464
3465     my $arch = get_vm_arch($conf);
3466     my $kvm_binary = get_command_for_arch($arch);
3467     my $kvmver = kvm_user_version($kvm_binary);
3468
3469     my $add_pve_version = min_version($kvmver, 4, 1);
3470
3471     my $machine_type = get_vm_machine($conf, $forcemachine, $arch, $add_pve_version);
3472     my $machine_version = PVE::QemuServer::Machine::extract_version($machine_type, $kvmver);
3473     $kvm //= 1 if is_native($arch);
3474
3475     $machine_version =~ m/(\d+)\.(\d+)/;
3476     die "Installed QEMU version '$kvmver' is too old to run machine type '$machine_type', please upgrade node '$nodename'\n"
3477         if !PVE::QemuServer::min_version($kvmver, $1, $2);
3478
3479     if ($kvm) {
3480         die "KVM virtualisation configured, but not available. Either disable in VM configuration or enable in BIOS.\n"
3481             if !defined kvm_version();
3482     }
3483
3484     if ($kvmver =~ m/^(\d+)\.(\d+)$/) {
3485         $vernum = $1*1000000+$2*1000;
3486     } elsif ($kvmver =~ m/^(\d+)\.(\d+)\.(\d+)$/) {
3487         $vernum = $1*1000000+$2*1000+$3;
3488     }
3489
3490     die "detected old qemu-kvm binary ($kvmver)\n" if $vernum < 15000;
3491
3492     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
3493     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
3494     my $use_old_bios_files = undef;
3495     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3496
3497     my $cpuunits = defined($conf->{cpuunits}) ?
3498             $conf->{cpuunits} : $defaults->{cpuunits};
3499
3500     push @$cmd, $kvm_binary;
3501
3502     push @$cmd, '-id', $vmid;
3503
3504     my $vmname = $conf->{name} || "vm$vmid";
3505
3506     push @$cmd, '-name', $vmname;
3507
3508     my $use_virtio = 0;
3509
3510     my $qmpsocket = PVE::QemuServer::Helpers::qmp_socket($vmid);
3511     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server,nowait";
3512     push @$cmd, '-mon', "chardev=qmp,mode=control";
3513
3514     if (min_version($machine_version, 2, 12)) {
3515         push @$cmd, '-chardev', "socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5";
3516         push @$cmd, '-mon', "chardev=qmp-event,mode=control";
3517     }
3518
3519     push @$cmd, '-pidfile' , PVE::QemuServer::Helpers::pidfile_name($vmid);
3520
3521     push @$cmd, '-daemonize';
3522
3523     if ($conf->{smbios1}) {
3524         my $smbios_conf = parse_smbios1($conf->{smbios1});
3525         if ($smbios_conf->{base64}) {
3526             # Do not pass base64 flag to qemu
3527             delete $smbios_conf->{base64};
3528             my $smbios_string = "";
3529             foreach my $key (keys %$smbios_conf) {
3530                 my $value;
3531                 if ($key eq "uuid") {
3532                     $value = $smbios_conf->{uuid}
3533                 } else {
3534                     $value = decode_base64($smbios_conf->{$key});
3535                 }
3536                 # qemu accepts any binary data, only commas need escaping by double comma
3537                 $value =~ s/,/,,/g;
3538                 $smbios_string .= "," . $key . "=" . $value if $value;
3539             }
3540             push @$cmd, '-smbios', "type=1" . $smbios_string;
3541         } else {
3542             push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
3543         }
3544     }
3545
3546     if ($conf->{vmgenid}) {
3547         push @$devices, '-device', 'vmgenid,guid='.$conf->{vmgenid};
3548     }
3549
3550     my ($ovmf_code, $ovmf_vars) = get_ovmf_files($arch);
3551     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3552         die "uefi base image not found\n" if ! -f $ovmf_code;
3553
3554         my $path;
3555         my $format;
3556         if (my $efidisk = $conf->{efidisk0}) {
3557             my $d = PVE::JSONSchema::parse_property_string($efidisk_fmt, $efidisk);
3558             my ($storeid, $volname) = PVE::Storage::parse_volume_id($d->{file}, 1);
3559             $format = $d->{format};
3560             if ($storeid) {
3561                 $path = PVE::Storage::path($storecfg, $d->{file});
3562                 if (!defined($format)) {
3563                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
3564                     $format = qemu_img_format($scfg, $volname);
3565                 }
3566             } else {
3567                 $path = $d->{file};
3568                 die "efidisk format must be specified\n"
3569                     if !defined($format);
3570             }
3571         } else {
3572             warn "no efidisk configured! Using temporary efivars disk.\n";
3573             $path = "/tmp/$vmid-ovmf.fd";
3574             PVE::Tools::file_copy($ovmf_vars, $path, -s $ovmf_vars);
3575             $format = 'raw';
3576         }
3577
3578         push @$cmd, '-drive', "if=pflash,unit=0,format=raw,readonly,file=$ovmf_code";
3579         push @$cmd, '-drive', "if=pflash,unit=1,format=$format,id=drive-efidisk0,file=$path";
3580     }
3581
3582     # load q35 config
3583     if ($q35) {
3584         # we use different pcie-port hardware for qemu >= 4.0 for passthrough
3585         if (min_version($machine_version, 4, 0)) {
3586             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35-4.0.cfg';
3587         } else {
3588             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35.cfg';
3589         }
3590     }
3591
3592     # add usb controllers
3593     my @usbcontrollers = PVE::QemuServer::USB::get_usb_controllers($conf, $bridges, $arch, $machine_type, $usbdesc->{format}, $MAX_USB_DEVICES);
3594     push @$devices, @usbcontrollers if @usbcontrollers;
3595     my $vga = parse_vga($conf->{vga});
3596
3597     my $qxlnum = vga_conf_has_spice($conf->{vga});
3598     $vga->{type} = 'qxl' if $qxlnum;
3599
3600     if (!$vga->{type}) {
3601         if ($arch eq 'aarch64') {
3602             $vga->{type} = 'virtio';
3603         } elsif (min_version($machine_version, 2, 9)) {
3604             $vga->{type} = (!$winversion || $winversion >= 6) ? 'std' : 'cirrus';
3605         } else {
3606             $vga->{type} = ($winversion >= 6) ? 'std' : 'cirrus';
3607         }
3608     }
3609
3610     # enable absolute mouse coordinates (needed by vnc)
3611     my $tablet;
3612     if (defined($conf->{tablet})) {
3613         $tablet = $conf->{tablet};
3614     } else {
3615         $tablet = $defaults->{tablet};
3616         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
3617         $tablet = 0 if $vga->{type} =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
3618     }
3619
3620     if ($tablet) {
3621         push @$devices, '-device', print_tabletdevice_full($conf, $arch) if $tablet;
3622         my $kbd = print_keyboarddevice_full($conf, $arch);
3623         push @$devices, '-device', $kbd if defined($kbd);
3624     }
3625
3626     my $kvm_off = 0;
3627     my $gpu_passthrough;
3628
3629     # host pci devices
3630     for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
3631         my $id = "hostpci$i";
3632         my $d = parse_hostpci($conf->{$id});
3633         next if !$d;
3634
3635         if (my $pcie = $d->{pcie}) {
3636             die "q35 machine model is not enabled" if !$q35;
3637             # win7 wants to have the pcie devices directly on the pcie bus
3638             # instead of in the root port
3639             if ($winversion == 7) {
3640                 $pciaddr = print_pcie_addr("${id}bus0");
3641             } else {
3642                 # add more root ports if needed, 4 are present by default
3643                 # by pve-q35 cfgs, rest added here on demand.
3644                 if ($i > 3) {
3645                     push @$devices, '-device', print_pcie_root_port($i);
3646                 }
3647                 $pciaddr = print_pcie_addr($id);
3648             }
3649         } else {
3650             $pciaddr = print_pci_addr($id, $bridges, $arch, $machine_type);
3651         }
3652
3653         my $xvga = '';
3654         if ($d->{'x-vga'}) {
3655             $xvga = ',x-vga=on' if !($conf->{bios} && $conf->{bios} eq 'ovmf');
3656             $kvm_off = 1;
3657             $vga->{type} = 'none' if !defined($conf->{vga});
3658             $gpu_passthrough = 1;
3659         }
3660
3661         my $pcidevices = $d->{pciid};
3662         my $multifunction = 1 if @$pcidevices > 1;
3663
3664         my $sysfspath;
3665         if ($d->{mdev} && scalar(@$pcidevices) == 1) {
3666             my $pci_id = $pcidevices->[0]->{id};
3667             my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $i);
3668             $sysfspath = "/sys/bus/pci/devices/$pci_id/$uuid";
3669         } elsif ($d->{mdev}) {
3670             warn "ignoring mediated device '$id' with multifunction device\n";
3671         }
3672
3673         my $j=0;
3674         foreach my $pcidevice (@$pcidevices) {
3675             my $devicestr = "vfio-pci";
3676
3677             if ($sysfspath) {
3678                 $devicestr .= ",sysfsdev=$sysfspath";
3679             } else {
3680                 $devicestr .= ",host=$pcidevice->{id}";
3681             }
3682
3683             my $mf_addr = $multifunction ? ".$j" : '';
3684             $devicestr .= ",id=${id}${mf_addr}${pciaddr}${mf_addr}";
3685
3686             if ($j == 0) {
3687                 $devicestr .= ',rombar=0' if defined($d->{rombar}) && !$d->{rombar};
3688                 $devicestr .= "$xvga";
3689                 $devicestr .= ",multifunction=on" if $multifunction;
3690                 $devicestr .= ",romfile=/usr/share/kvm/$d->{romfile}" if $d->{romfile};
3691             }
3692
3693             push @$devices, '-device', $devicestr;
3694             $j++;
3695         }
3696     }
3697
3698     # usb devices
3699     my $usb_dev_features = {};
3700     $usb_dev_features->{spice_usb3} = 1 if min_version($machine_version, 4, 0);
3701
3702     my @usbdevices = PVE::QemuServer::USB::get_usb_devices($conf, $usbdesc->{format}, $MAX_USB_DEVICES, $usb_dev_features);
3703     push @$devices, @usbdevices if @usbdevices;
3704     # serial devices
3705     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3706         if (my $path = $conf->{"serial$i"}) {
3707             if ($path eq 'socket') {
3708                 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
3709                 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server,nowait";
3710                 # On aarch64, serial0 is the UART device. Qemu only allows
3711                 # connecting UART devices via the '-serial' command line, as
3712                 # the device has a fixed slot on the hardware...
3713                 if ($arch eq 'aarch64' && $i == 0) {
3714                     push @$devices, '-serial', "chardev:serial$i";
3715                 } else {
3716                     push @$devices, '-device', "isa-serial,chardev=serial$i";
3717                 }
3718             } else {
3719                 die "no such serial device\n" if ! -c $path;
3720                 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
3721                 push @$devices, '-device', "isa-serial,chardev=serial$i";
3722             }
3723         }
3724     }
3725
3726     # parallel devices
3727     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
3728         if (my $path = $conf->{"parallel$i"}) {
3729             die "no such parallel device\n" if ! -c $path;
3730             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
3731             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
3732             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
3733         }
3734     }
3735
3736     if (my $audio = conf_has_audio($conf)) {
3737
3738         my $audiopciaddr = print_pci_addr("audio0", $bridges, $arch, $machine_type);
3739
3740         my $id = $audio->{dev_id};
3741         if ($audio->{dev} eq 'AC97') {
3742             push @$devices, '-device', "AC97,id=${id}${audiopciaddr}";
3743         } elsif ($audio->{dev} =~ /intel\-hda$/) {
3744             push @$devices, '-device', "$audio->{dev},id=${id}${audiopciaddr}";
3745             push @$devices, '-device', "hda-micro,id=${id}-codec0,bus=${id}.0,cad=0";
3746             push @$devices, '-device', "hda-duplex,id=${id}-codec1,bus=${id}.0,cad=1";
3747         } else {
3748             die "unkown audio device '$audio->{dev}', implement me!";
3749         }
3750
3751         push @$devices, '-audiodev', "$audio->{backend},id=$audio->{backend_id}";
3752     }
3753
3754     my $sockets = 1;
3755     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3756     $sockets = $conf->{sockets} if  $conf->{sockets};
3757
3758     my $cores = $conf->{cores} || 1;
3759
3760     my $maxcpus = $sockets * $cores;
3761
3762     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
3763
3764     my $allowed_vcpus = $cpuinfo->{cpus};
3765
3766     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n"
3767         if ($allowed_vcpus < $maxcpus);
3768
3769     if($hotplug_features->{cpu} && min_version($machine_version, 2, 7)) {
3770
3771         push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3772         for (my $i = 2; $i <= $vcpus; $i++)  {
3773             my $cpustr = print_cpu_device($conf,$i);
3774             push @$cmd, '-device', $cpustr;
3775         }
3776
3777     } else {
3778
3779         push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3780     }
3781     push @$cmd, '-nodefaults';
3782
3783     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
3784
3785     my $bootindex_hash = {};
3786     my $i = 1;
3787     foreach my $o (split(//, $bootorder)) {
3788         $bootindex_hash->{$o} = $i*100;
3789         $i++;
3790     }
3791
3792     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
3793
3794     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
3795
3796     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
3797
3798     if ($vga->{type} && $vga->{type} !~ m/^serial\d+$/ && $vga->{type} ne 'none'){
3799         push @$devices, '-device', print_vga_device($conf, $vga, $arch, $machine_version, $machine_type, undef, $qxlnum, $bridges);
3800         my $socket = PVE::QemuServer::Helpers::vnc_socket($vmid);
3801         push @$cmd,  '-vnc', "unix:$socket,password";
3802     } else {
3803         push @$cmd, '-vga', 'none' if $vga->{type} eq 'none';
3804         push @$cmd, '-nographic';
3805     }
3806
3807     # time drift fix
3808     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
3809
3810     my $useLocaltime = $conf->{localtime};
3811
3812     if ($winversion >= 5) { # windows
3813         $useLocaltime = 1 if !defined($conf->{localtime});
3814
3815         # use time drift fix when acpi is enabled
3816         if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
3817             $tdf = 1 if !defined($conf->{tdf});
3818         }
3819     }
3820
3821     if ($winversion >= 6) {
3822         push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3823         push @$cmd, '-no-hpet';
3824     }
3825
3826     push @$rtcFlags, 'driftfix=slew' if $tdf;
3827
3828     if (!$kvm) {
3829         push @$machineFlags, 'accel=tcg';
3830     }
3831
3832     if ($machine_type) {
3833         push @$machineFlags, "type=${machine_type}";
3834     }
3835
3836     if (($conf->{startdate}) && ($conf->{startdate} ne 'now')) {
3837         push @$rtcFlags, "base=$conf->{startdate}";
3838     } elsif ($useLocaltime) {
3839         push @$rtcFlags, 'base=localtime';
3840     }
3841
3842     push @$cmd, get_cpu_options($conf, $arch, $kvm, $kvm_off, $machine_version, $winversion, $gpu_passthrough);
3843
3844     PVE::QemuServer::Memory::config($conf, $vmid, $sockets, $cores, $defaults, $hotplug_features, $cmd);
3845
3846     push @$cmd, '-S' if $conf->{freeze};
3847
3848     push @$cmd, '-k', $conf->{keyboard} if defined($conf->{keyboard});
3849
3850     my $guest_agent = parse_guest_agent($conf);
3851
3852     if ($guest_agent->{enabled}) {
3853         my $qgasocket = PVE::QemuServer::Helpers::qmp_socket($vmid, 1);
3854         push @$devices, '-chardev', "socket,path=$qgasocket,server,nowait,id=qga0";
3855
3856         if (!$guest_agent->{type} || $guest_agent->{type} eq 'virtio') {
3857             my $pciaddr = print_pci_addr("qga0", $bridges, $arch, $machine_type);
3858             push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3859             push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3860         } elsif ($guest_agent->{type} eq 'isa') {
3861             push @$devices, '-device', "isa-serial,chardev=qga0";
3862         }
3863     }
3864
3865     my $spice_port;
3866
3867     if ($qxlnum) {
3868         if ($qxlnum > 1) {
3869             if ($winversion){
3870                 for(my $i = 1; $i < $qxlnum; $i++){
3871                     push @$devices, '-device', print_vga_device($conf, $vga, $arch, $machine_version, $machine_type, $i, $qxlnum, $bridges);
3872                 }
3873             } else {
3874                 # assume other OS works like Linux
3875                 my ($ram, $vram) = ("134217728", "67108864");
3876                 if ($vga->{memory}) {
3877                     $ram = PVE::Tools::convert_size($qxlnum*4*$vga->{memory}, 'mb' => 'b');
3878                     $vram = PVE::Tools::convert_size($qxlnum*2*$vga->{memory}, 'mb' => 'b');
3879                 }
3880                 push @$cmd, '-global', "qxl-vga.ram_size=$ram";
3881                 push @$cmd, '-global', "qxl-vga.vram_size=$vram";
3882             }
3883         }
3884
3885         my $pciaddr = print_pci_addr("spice", $bridges, $arch, $machine_type);
3886
3887         my $nodename = PVE::INotify::nodename();
3888         my $pfamily = PVE::Tools::get_host_address_family($nodename);
3889         my @nodeaddrs = PVE::Tools::getaddrinfo_all('localhost', family => $pfamily);
3890         die "failed to get an ip address of type $pfamily for 'localhost'\n" if !@nodeaddrs;
3891
3892         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3893         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3894         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3895
3896         my $localhost = PVE::Network::addr_to_ip($nodeaddrs[0]->{addr});
3897         $spice_port = PVE::Tools::next_spice_port($pfamily, $localhost);
3898
3899         my $spice_enhancement = PVE::JSONSchema::parse_property_string($spice_enhancements_fmt, $conf->{spice_enhancements} // '');
3900         if ($spice_enhancement->{foldersharing}) {
3901             push @$devices, '-chardev', "spiceport,id=foldershare,name=org.spice-space.webdav.0";
3902             push @$devices, '-device', "virtserialport,chardev=foldershare,name=org.spice-space.webdav.0";
3903         }
3904
3905         my $spice_opts = "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";
3906         $spice_opts .= ",streaming-video=$spice_enhancement->{videostreaming}" if $spice_enhancement->{videostreaming};
3907         push @$devices, '-spice', "$spice_opts";
3908     }
3909
3910     # enable balloon by default, unless explicitly disabled
3911     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3912         $pciaddr = print_pci_addr("balloon0", $bridges, $arch, $machine_type);
3913         push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
3914     }
3915
3916     if ($conf->{watchdog}) {
3917         my $wdopts = parse_watchdog($conf->{watchdog});
3918         $pciaddr = print_pci_addr("watchdog", $bridges, $arch, $machine_type);
3919         my $watchdog = $wdopts->{model} || 'i6300esb';
3920         push @$devices, '-device', "$watchdog$pciaddr";
3921         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3922     }
3923
3924     my $vollist = [];
3925     my $scsicontroller = {};
3926     my $ahcicontroller = {};
3927     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3928
3929     # Add iscsi initiator name if available
3930     if (my $initiator = get_initiator_name()) {
3931         push @$devices, '-iscsi', "initiator-name=$initiator";
3932     }
3933
3934     foreach_drive($conf, sub {
3935         my ($ds, $drive) = @_;
3936
3937         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3938             push @$vollist, $drive->{file};
3939         }
3940
3941         # ignore efidisk here, already added in bios/fw handling code above
3942         return if $drive->{interface} eq 'efidisk';
3943
3944         $use_virtio = 1 if $ds =~ m/^virtio/;
3945
3946         if (drive_is_cdrom ($drive)) {
3947             if ($bootindex_hash->{d}) {
3948                 $drive->{bootindex} = $bootindex_hash->{d};
3949                 $bootindex_hash->{d} += 1;
3950             }
3951         } else {
3952             if ($bootindex_hash->{c}) {
3953                 $drive->{bootindex} = $bootindex_hash->{c} if $conf->{bootdisk} && ($conf->{bootdisk} eq $ds);
3954                 $bootindex_hash->{c} += 1;
3955             }
3956         }
3957
3958         if($drive->{interface} eq 'virtio'){
3959            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3960         }
3961
3962         if ($drive->{interface} eq 'scsi') {
3963
3964             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
3965
3966             $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges, $arch, $machine_type);
3967             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
3968
3969             my $iothread = '';
3970             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
3971                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3972                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3973             } elsif ($drive->{iothread}) {
3974                 warn "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n";
3975             }
3976
3977             my $queues = '';
3978             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
3979                 $queues = ",num_queues=$drive->{queues}";
3980             }
3981
3982             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues" if !$scsicontroller->{$controller};
3983             $scsicontroller->{$controller}=1;
3984         }
3985
3986         if ($drive->{interface} eq 'sata') {
3987            my $controller = int($drive->{index} / $MAX_SATA_DISKS);
3988            $pciaddr = print_pci_addr("ahci$controller", $bridges, $arch, $machine_type);
3989            push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr" if !$ahcicontroller->{$controller};
3990            $ahcicontroller->{$controller}=1;
3991         }
3992
3993         my $drive_cmd = print_drive_full($storecfg, $vmid, $drive);
3994         push @$devices, '-drive',$drive_cmd;
3995         push @$devices, '-device', print_drivedevice_full($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type);
3996     });
3997
3998     for (my $i = 0; $i < $MAX_NETS; $i++) {
3999          next if !$conf->{"net$i"};
4000          my $d = parse_net($conf->{"net$i"});
4001          next if !$d;
4002
4003          $use_virtio = 1 if $d->{model} eq 'virtio';
4004
4005          if ($bootindex_hash->{n}) {
4006             $d->{bootindex} = $bootindex_hash->{n};
4007             $bootindex_hash->{n} += 1;
4008          }
4009
4010          my $netdevfull = print_netdev_full($vmid, $conf, $arch, $d, "net$i");
4011          push @$devices, '-netdev', $netdevfull;
4012
4013          my $netdevicefull = print_netdevice_full($vmid, $conf, $d, "net$i", $bridges, $use_old_bios_files, $arch, $machine_type);
4014          push @$devices, '-device', $netdevicefull;
4015     }
4016
4017     if ($conf->{ivshmem}) {
4018         my $ivshmem = PVE::JSONSchema::parse_property_string($ivshmem_fmt, $conf->{ivshmem});
4019
4020         my $bus;
4021         if ($q35) {
4022             $bus = print_pcie_addr("ivshmem");
4023         } else {
4024             $bus = print_pci_addr("ivshmem", $bridges, $arch, $machine_type);
4025         }
4026
4027         my $ivshmem_name = $ivshmem->{name} // $vmid;
4028         my $path = '/dev/shm/pve-shm-' . $ivshmem_name;
4029
4030         push @$devices, '-device', "ivshmem-plain,memdev=ivshmem$bus,";
4031         push @$devices, '-object', "memory-backend-file,id=ivshmem,share=on,mem-path=$path,size=$ivshmem->{size}M";
4032     }
4033
4034     if (!$q35) {
4035         # add pci bridges
4036         if (min_version($machine_version, 2, 3)) {
4037            $bridges->{1} = 1;
4038            $bridges->{2} = 1;
4039         }
4040
4041         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
4042
4043         for my $k (sort {$b cmp $a} keys %$bridges) {
4044             $pciaddr = print_pci_addr("pci.$k", undef, $arch, $machine_type);
4045             unshift @$devices, '-device', "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr" if $k > 0;
4046         }
4047     }
4048
4049     push @$cmd, @$devices;
4050     push @$cmd, '-rtc', join(',', @$rtcFlags)
4051         if scalar(@$rtcFlags);
4052     push @$cmd, '-machine', join(',', @$machineFlags)
4053         if scalar(@$machineFlags);
4054     push @$cmd, '-global', join(',', @$globalFlags)
4055         if scalar(@$globalFlags);
4056
4057     if (my $vmstate = $conf->{vmstate}) {
4058         my $statepath = PVE::Storage::path($storecfg, $vmstate);
4059         push @$vollist, $vmstate;
4060         push @$cmd, '-loadstate', $statepath;
4061         print "activating and using '$vmstate' as vmstate\n";
4062     }
4063
4064     # add custom args
4065     if ($conf->{args}) {
4066         my $aa = PVE::Tools::split_args($conf->{args});
4067         push @$cmd, @$aa;
4068     }
4069
4070     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
4071 }
4072
4073 sub spice_port {
4074     my ($vmid) = @_;
4075
4076     my $res = mon_cmd($vmid, 'query-spice');
4077
4078     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
4079 }
4080
4081 sub vm_devices_list {
4082     my ($vmid) = @_;
4083
4084     my $res = mon_cmd($vmid, 'query-pci');
4085     my $devices_to_check = [];
4086     my $devices = {};
4087     foreach my $pcibus (@$res) {
4088         push @$devices_to_check, @{$pcibus->{devices}},
4089     }
4090
4091     while (@$devices_to_check) {
4092         my $to_check = [];
4093         for my $d (@$devices_to_check) {
4094             $devices->{$d->{'qdev_id'}} = 1 if $d->{'qdev_id'};
4095             next if !$d->{'pci_bridge'};
4096
4097             $devices->{$d->{'qdev_id'}} += scalar(@{$d->{'pci_bridge'}->{devices}});
4098             push @$to_check, @{$d->{'pci_bridge'}->{devices}};
4099         }
4100         $devices_to_check = $to_check;
4101     }
4102
4103     my $resblock = mon_cmd($vmid, 'query-block');
4104     foreach my $block (@$resblock) {
4105         if($block->{device} =~ m/^drive-(\S+)/){
4106                 $devices->{$1} = 1;
4107         }
4108     }
4109
4110     my $resmice = mon_cmd($vmid, 'query-mice');
4111     foreach my $mice (@$resmice) {
4112         if ($mice->{name} eq 'QEMU HID Tablet') {
4113             $devices->{tablet} = 1;
4114             last;
4115         }
4116     }
4117
4118     # for usb devices there is no query-usb
4119     # but we can iterate over the entries in
4120     # qom-list path=/machine/peripheral
4121     my $resperipheral = mon_cmd($vmid, 'qom-list', path => '/machine/peripheral');
4122     foreach my $per (@$resperipheral) {
4123         if ($per->{name} =~ m/^usb\d+$/) {
4124             $devices->{$per->{name}} = 1;
4125         }
4126     }
4127
4128     return $devices;
4129 }
4130
4131 sub vm_deviceplug {
4132     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4133
4134     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
4135
4136     my $devices_list = vm_devices_list($vmid);
4137     return 1 if defined($devices_list->{$deviceid});
4138
4139     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid, $arch, $machine_type); # add PCI bridge if we need it for the device
4140
4141     if ($deviceid eq 'tablet') {
4142
4143         qemu_deviceadd($vmid, print_tabletdevice_full($conf, $arch));
4144
4145     } elsif ($deviceid eq 'keyboard') {
4146
4147         qemu_deviceadd($vmid, print_keyboarddevice_full($conf, $arch));
4148
4149     } elsif ($deviceid =~ m/^usb(\d+)$/) {
4150
4151         die "usb hotplug currently not reliable\n";
4152         # since we can't reliably hot unplug all added usb devices
4153         # and usb passthrough disables live migration
4154         # we disable usb hotplugging for now
4155         qemu_deviceadd($vmid, PVE::QemuServer::USB::print_usbdevice_full($conf, $deviceid, $device));
4156
4157     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4158
4159         qemu_iothread_add($vmid, $deviceid, $device);
4160
4161         qemu_driveadd($storecfg, $vmid, $device);
4162         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, $arch, $machine_type);
4163
4164         qemu_deviceadd($vmid, $devicefull);
4165         eval { qemu_deviceaddverify($vmid, $deviceid); };
4166         if (my $err = $@) {
4167             eval { qemu_drivedel($vmid, $deviceid); };
4168             warn $@ if $@;
4169             die $err;
4170         }
4171
4172     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4173
4174
4175         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
4176         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
4177         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
4178
4179         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
4180
4181         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
4182             qemu_iothread_add($vmid, $deviceid, $device);
4183             $devicefull .= ",iothread=iothread-$deviceid";
4184         }
4185
4186         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
4187             $devicefull .= ",num_queues=$device->{queues}";
4188         }
4189
4190         qemu_deviceadd($vmid, $devicefull);
4191         qemu_deviceaddverify($vmid, $deviceid);
4192
4193     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4194
4195         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device, $arch, $machine_type);
4196         qemu_driveadd($storecfg, $vmid, $device);
4197
4198         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, $arch, $machine_type);
4199         eval { qemu_deviceadd($vmid, $devicefull); };
4200         if (my $err = $@) {
4201             eval { qemu_drivedel($vmid, $deviceid); };
4202             warn $@ if $@;
4203             die $err;
4204         }
4205
4206     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4207
4208         return undef if !qemu_netdevadd($vmid, $conf, $arch, $device, $deviceid);
4209
4210         my $machine_type = PVE::QemuServer::Machine::qemu_machine_pxe($vmid, $conf);
4211         my $use_old_bios_files = undef;
4212         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
4213
4214         my $netdevicefull = print_netdevice_full($vmid, $conf, $device, $deviceid, undef, $use_old_bios_files, $arch, $machine_type);
4215         qemu_deviceadd($vmid, $netdevicefull);
4216         eval {
4217             qemu_deviceaddverify($vmid, $deviceid);
4218             qemu_set_link_status($vmid, $deviceid, !$device->{link_down});
4219         };
4220         if (my $err = $@) {
4221             eval { qemu_netdevdel($vmid, $deviceid); };
4222             warn $@ if $@;
4223             die $err;
4224         }
4225
4226     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
4227
4228         my $bridgeid = $2;
4229         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
4230         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
4231
4232         qemu_deviceadd($vmid, $devicefull);
4233         qemu_deviceaddverify($vmid, $deviceid);
4234
4235     } else {
4236         die "can't hotplug device '$deviceid'\n";
4237     }
4238
4239     return 1;
4240 }
4241
4242 # fixme: this should raise exceptions on error!
4243 sub vm_deviceunplug {
4244     my ($vmid, $conf, $deviceid) = @_;
4245
4246     my $devices_list = vm_devices_list($vmid);
4247     return 1 if !defined($devices_list->{$deviceid});
4248
4249     die "can't unplug bootdisk" if $conf->{bootdisk} && $conf->{bootdisk} eq $deviceid;
4250
4251     if ($deviceid eq 'tablet' || $deviceid eq 'keyboard') {
4252
4253         qemu_devicedel($vmid, $deviceid);
4254
4255     } elsif ($deviceid =~ m/^usb\d+$/) {
4256
4257         die "usb hotplug currently not reliable\n";
4258         # when unplugging usb devices this way,
4259         # there may be remaining usb controllers/hubs
4260         # so we disable it for now
4261         qemu_devicedel($vmid, $deviceid);
4262         qemu_devicedelverify($vmid, $deviceid);
4263
4264     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4265
4266         qemu_devicedel($vmid, $deviceid);
4267         qemu_devicedelverify($vmid, $deviceid);
4268         qemu_drivedel($vmid, $deviceid);
4269         qemu_iothread_del($conf, $vmid, $deviceid);
4270
4271     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4272
4273         qemu_devicedel($vmid, $deviceid);
4274         qemu_devicedelverify($vmid, $deviceid);
4275         qemu_iothread_del($conf, $vmid, $deviceid);
4276
4277     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4278
4279         qemu_devicedel($vmid, $deviceid);
4280         qemu_drivedel($vmid, $deviceid);
4281         qemu_deletescsihw($conf, $vmid, $deviceid);
4282
4283     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4284
4285         qemu_devicedel($vmid, $deviceid);
4286         qemu_devicedelverify($vmid, $deviceid);
4287         qemu_netdevdel($vmid, $deviceid);
4288
4289     } else {
4290         die "can't unplug device '$deviceid'\n";
4291     }
4292
4293     return 1;
4294 }
4295
4296 sub qemu_deviceadd {
4297     my ($vmid, $devicefull) = @_;
4298
4299     $devicefull = "driver=".$devicefull;
4300     my %options =  split(/[=,]/, $devicefull);
4301
4302     mon_cmd($vmid, "device_add" , %options);
4303 }
4304
4305 sub qemu_devicedel {
4306     my ($vmid, $deviceid) = @_;
4307
4308     my $ret = mon_cmd($vmid, "device_del", id => $deviceid);
4309 }
4310
4311 sub qemu_iothread_add {
4312     my($vmid, $deviceid, $device) = @_;
4313
4314     if ($device->{iothread}) {
4315         my $iothreads = vm_iothreads_list($vmid);
4316         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
4317     }
4318 }
4319
4320 sub qemu_iothread_del {
4321     my($conf, $vmid, $deviceid) = @_;
4322
4323     my $confid = $deviceid;
4324     if ($deviceid =~ m/^(?:virtioscsi|scsihw)(\d+)$/) {
4325         $confid = 'scsi' . $1;
4326     }
4327     my $device = parse_drive($confid, $conf->{$confid});
4328     if ($device->{iothread}) {
4329         my $iothreads = vm_iothreads_list($vmid);
4330         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
4331     }
4332 }
4333
4334 sub qemu_objectadd {
4335     my($vmid, $objectid, $qomtype) = @_;
4336
4337     mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
4338
4339     return 1;
4340 }
4341
4342 sub qemu_objectdel {
4343     my($vmid, $objectid) = @_;
4344
4345     mon_cmd($vmid, "object-del", id => $objectid);
4346
4347     return 1;
4348 }
4349
4350 sub qemu_driveadd {
4351     my ($storecfg, $vmid, $device) = @_;
4352
4353     my $drive = print_drive_full($storecfg, $vmid, $device);
4354     $drive =~ s/\\/\\\\/g;
4355     my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_add auto \"$drive\"");
4356
4357     # If the command succeeds qemu prints: "OK"
4358     return 1 if $ret =~ m/OK/s;
4359
4360     die "adding drive failed: $ret\n";
4361 }
4362
4363 sub qemu_drivedel {
4364     my($vmid, $deviceid) = @_;
4365
4366     my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_del drive-$deviceid");
4367     $ret =~ s/^\s+//;
4368
4369     return 1 if $ret eq "";
4370
4371     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
4372     return 1 if $ret =~ m/Device \'.*?\' not found/s;
4373
4374     die "deleting drive $deviceid failed : $ret\n";
4375 }
4376
4377 sub qemu_deviceaddverify {
4378     my ($vmid, $deviceid) = @_;
4379
4380     for (my $i = 0; $i <= 5; $i++) {
4381          my $devices_list = vm_devices_list($vmid);
4382          return 1 if defined($devices_list->{$deviceid});
4383          sleep 1;
4384     }
4385
4386     die "error on hotplug device '$deviceid'\n";
4387 }
4388
4389
4390 sub qemu_devicedelverify {
4391     my ($vmid, $deviceid) = @_;
4392
4393     # need to verify that the device is correctly removed as device_del
4394     # is async and empty return is not reliable
4395
4396     for (my $i = 0; $i <= 5; $i++) {
4397          my $devices_list = vm_devices_list($vmid);
4398          return 1 if !defined($devices_list->{$deviceid});
4399          sleep 1;
4400     }
4401
4402     die "error on hot-unplugging device '$deviceid'\n";
4403 }
4404
4405 sub qemu_findorcreatescsihw {
4406     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4407
4408     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4409
4410     my $scsihwid="$controller_prefix$controller";
4411     my $devices_list = vm_devices_list($vmid);
4412
4413     if(!defined($devices_list->{$scsihwid})) {
4414         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device, $arch, $machine_type);
4415     }
4416
4417     return 1;
4418 }
4419
4420 sub qemu_deletescsihw {
4421     my ($conf, $vmid, $opt) = @_;
4422
4423     my $device = parse_drive($opt, $conf->{$opt});
4424
4425     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
4426         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
4427         return 1;
4428     }
4429
4430     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4431
4432     my $devices_list = vm_devices_list($vmid);
4433     foreach my $opt (keys %{$devices_list}) {
4434         if (PVE::QemuServer::is_valid_drivename($opt)) {
4435             my $drive = PVE::QemuServer::parse_drive($opt, $conf->{$opt});
4436             if($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
4437                 return 1;
4438             }
4439         }
4440     }
4441
4442     my $scsihwid="scsihw$controller";
4443
4444     vm_deviceunplug($vmid, $conf, $scsihwid);
4445
4446     return 1;
4447 }
4448
4449 sub qemu_add_pci_bridge {
4450     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4451
4452     my $bridges = {};
4453
4454     my $bridgeid;
4455
4456     print_pci_addr($device, $bridges, $arch, $machine_type);
4457
4458     while (my ($k, $v) = each %$bridges) {
4459         $bridgeid = $k;
4460     }
4461     return 1 if !defined($bridgeid) || $bridgeid < 1;
4462
4463     my $bridge = "pci.$bridgeid";
4464     my $devices_list = vm_devices_list($vmid);
4465
4466     if (!defined($devices_list->{$bridge})) {
4467         vm_deviceplug($storecfg, $conf, $vmid, $bridge, $arch, $machine_type);
4468     }
4469
4470     return 1;
4471 }
4472
4473 sub qemu_set_link_status {
4474     my ($vmid, $device, $up) = @_;
4475
4476     mon_cmd($vmid, "set_link", name => $device,
4477                up => $up ? JSON::true : JSON::false);
4478 }
4479
4480 sub qemu_netdevadd {
4481     my ($vmid, $conf, $arch, $device, $deviceid) = @_;
4482
4483     my $netdev = print_netdev_full($vmid, $conf, $arch, $device, $deviceid, 1);
4484     my %options =  split(/[=,]/, $netdev);
4485
4486     mon_cmd($vmid, "netdev_add",  %options);
4487     return 1;
4488 }
4489
4490 sub qemu_netdevdel {
4491     my ($vmid, $deviceid) = @_;
4492
4493     mon_cmd($vmid, "netdev_del", id => $deviceid);
4494 }
4495
4496 sub qemu_usb_hotplug {
4497     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4498
4499     return if !$device;
4500
4501     # remove the old one first
4502     vm_deviceunplug($vmid, $conf, $deviceid);
4503
4504     # check if xhci controller is necessary and available
4505     if ($device->{usb3}) {
4506
4507         my $devicelist = vm_devices_list($vmid);
4508
4509         if (!$devicelist->{xhci}) {
4510             my $pciaddr = print_pci_addr("xhci", undef, $arch, $machine_type);
4511             qemu_deviceadd($vmid, "nec-usb-xhci,id=xhci$pciaddr");
4512         }
4513     }
4514     my $d = parse_usb_device($device->{host});
4515     $d->{usb3} = $device->{usb3};
4516
4517     # add the new one
4518     vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $d, $arch, $machine_type);
4519 }
4520
4521 sub qemu_cpu_hotplug {
4522     my ($vmid, $conf, $vcpus) = @_;
4523
4524     my $machine_type = PVE::QemuServer::Machine::get_current_qemu_machine($vmid);
4525
4526     my $sockets = 1;
4527     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
4528     $sockets = $conf->{sockets} if  $conf->{sockets};
4529     my $cores = $conf->{cores} || 1;
4530     my $maxcpus = $sockets * $cores;
4531
4532     $vcpus = $maxcpus if !$vcpus;
4533
4534     die "you can't add more vcpus than maxcpus\n"
4535         if $vcpus > $maxcpus;
4536
4537     my $currentvcpus = $conf->{vcpus} || $maxcpus;
4538
4539     if ($vcpus < $currentvcpus) {
4540
4541         if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4542
4543             for (my $i = $currentvcpus; $i > $vcpus; $i--) {
4544                 qemu_devicedel($vmid, "cpu$i");
4545                 my $retry = 0;
4546                 my $currentrunningvcpus = undef;
4547                 while (1) {
4548                     $currentrunningvcpus = mon_cmd($vmid, "query-cpus");
4549                     last if scalar(@{$currentrunningvcpus}) == $i-1;
4550                     raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
4551                     $retry++;
4552                     sleep 1;
4553                 }
4554                 #update conf after each succesfull cpu unplug
4555                 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4556                 PVE::QemuConfig->write_config($vmid, $conf);
4557             }
4558         } else {
4559             die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
4560         }
4561
4562         return;
4563     }
4564
4565     my $currentrunningvcpus = mon_cmd($vmid, "query-cpus");
4566     die "vcpus in running vm does not match its configuration\n"
4567         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
4568
4569     if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4570
4571         for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
4572             my $cpustr = print_cpu_device($conf, $i);
4573             qemu_deviceadd($vmid, $cpustr);
4574
4575             my $retry = 0;
4576             my $currentrunningvcpus = undef;
4577             while (1) {
4578                 $currentrunningvcpus = mon_cmd($vmid, "query-cpus");
4579                 last if scalar(@{$currentrunningvcpus}) == $i;
4580                 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
4581                 sleep 1;
4582                 $retry++;
4583             }
4584             #update conf after each succesfull cpu hotplug
4585             $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4586             PVE::QemuConfig->write_config($vmid, $conf);
4587         }
4588     } else {
4589
4590         for (my $i = $currentvcpus; $i < $vcpus; $i++) {
4591             mon_cmd($vmid, "cpu-add", id => int($i));
4592         }
4593     }
4594 }
4595
4596 sub qemu_block_set_io_throttle {
4597     my ($vmid, $deviceid,
4598         $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
4599         $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max,
4600         $bps_max_length, $bps_rd_max_length, $bps_wr_max_length,
4601         $iops_max_length, $iops_rd_max_length, $iops_wr_max_length) = @_;
4602
4603     return if !check_running($vmid) ;
4604
4605     mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
4606         bps => int($bps),
4607         bps_rd => int($bps_rd),
4608         bps_wr => int($bps_wr),
4609         iops => int($iops),
4610         iops_rd => int($iops_rd),
4611         iops_wr => int($iops_wr),
4612         bps_max => int($bps_max),
4613         bps_rd_max => int($bps_rd_max),
4614         bps_wr_max => int($bps_wr_max),
4615         iops_max => int($iops_max),
4616         iops_rd_max => int($iops_rd_max),
4617         iops_wr_max => int($iops_wr_max),
4618         bps_max_length => int($bps_max_length),
4619         bps_rd_max_length => int($bps_rd_max_length),
4620         bps_wr_max_length => int($bps_wr_max_length),
4621         iops_max_length => int($iops_max_length),
4622         iops_rd_max_length => int($iops_rd_max_length),
4623         iops_wr_max_length => int($iops_wr_max_length),
4624     );
4625
4626 }
4627
4628 # old code, only used to shutdown old VM after update
4629 sub __read_avail {
4630     my ($fh, $timeout) = @_;
4631
4632     my $sel = new IO::Select;
4633     $sel->add($fh);
4634
4635     my $res = '';
4636     my $buf;
4637
4638     my @ready;
4639     while (scalar (@ready = $sel->can_read($timeout))) {
4640         my $count;
4641         if ($count = $fh->sysread($buf, 8192)) {
4642             if ($buf =~ /^(.*)\(qemu\) $/s) {
4643                 $res .= $1;
4644                 last;
4645             } else {
4646                 $res .= $buf;
4647             }
4648         } else {
4649             if (!defined($count)) {
4650                 die "$!\n";
4651             }
4652             last;
4653         }
4654     }
4655
4656     die "monitor read timeout\n" if !scalar(@ready);
4657
4658     return $res;
4659 }
4660
4661 sub qemu_block_resize {
4662     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4663
4664     my $running = check_running($vmid);
4665
4666     $size = 0 if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4667
4668     return if !$running;
4669
4670     mon_cmd($vmid, "block_resize", device => $deviceid, size => int($size));
4671
4672 }
4673
4674 sub qemu_volume_snapshot {
4675     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4676
4677     my $running = check_running($vmid);
4678
4679     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4680         mon_cmd($vmid, 'blockdev-snapshot-internal-sync', device => $deviceid, name => $snap);
4681     } else {
4682         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4683     }
4684 }
4685
4686 sub qemu_volume_snapshot_delete {
4687     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4688
4689     my $running = check_running($vmid);
4690
4691     if($running) {
4692
4693         $running = undef;
4694         my $conf = PVE::QemuConfig->load_config($vmid);
4695         foreach_drive($conf, sub {
4696             my ($ds, $drive) = @_;
4697             $running = 1 if $drive->{file} eq $volid;
4698         });
4699     }
4700
4701     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4702         mon_cmd($vmid, 'blockdev-snapshot-delete-internal-sync', device => $deviceid, name => $snap);
4703     } else {
4704         PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4705     }
4706 }
4707
4708 sub set_migration_caps {
4709     my ($vmid) = @_;
4710
4711     my $cap_ref = [];
4712
4713     my $enabled_cap = {
4714         "auto-converge" => 1,
4715         "xbzrle" => 1,
4716         "x-rdma-pin-all" => 0,
4717         "zero-blocks" => 0,
4718         "compress" => 0
4719     };
4720
4721     my $supported_capabilities = mon_cmd($vmid, "query-migrate-capabilities");
4722
4723     for my $supported_capability (@$supported_capabilities) {
4724         push @$cap_ref, {
4725             capability => $supported_capability->{capability},
4726             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4727         };
4728     }
4729
4730     mon_cmd($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4731 }
4732
4733 my $fast_plug_option = {
4734     'lock' => 1,
4735     'name' => 1,
4736     'onboot' => 1,
4737     'shares' => 1,
4738     'startup' => 1,
4739     'description' => 1,
4740     'protection' => 1,
4741     'vmstatestorage' => 1,
4742     'hookscript' => 1,
4743     'tags' => 1,
4744 };
4745
4746 # hotplug changes in [PENDING]
4747 # $selection hash can be used to only apply specified options, for
4748 # example: { cores => 1 } (only apply changed 'cores')
4749 # $errors ref is used to return error messages
4750 sub vmconfig_hotplug_pending {
4751     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4752
4753     my $defaults = load_defaults();
4754     my $arch = get_vm_arch($conf);
4755     my $machine_type = get_vm_machine($conf, undef, $arch);
4756
4757     # commit values which do not have any impact on running VM first
4758     # Note: those option cannot raise errors, we we do not care about
4759     # $selection and always apply them.
4760
4761     my $add_error = sub {
4762         my ($opt, $msg) = @_;
4763         $errors->{$opt} = "hotplug problem - $msg";
4764     };
4765
4766     my $changes = 0;
4767     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4768         if ($fast_plug_option->{$opt}) {
4769             $conf->{$opt} = $conf->{pending}->{$opt};
4770             delete $conf->{pending}->{$opt};
4771             $changes = 1;
4772         }
4773     }
4774
4775     if ($changes) {
4776         PVE::QemuConfig->write_config($vmid, $conf);
4777         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4778     }
4779
4780     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4781
4782     my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4783     foreach my $opt (sort keys %$pending_delete_hash) {
4784         next if $selection && !$selection->{$opt};
4785         my $force = $pending_delete_hash->{$opt}->{force};
4786         eval {
4787             if ($opt eq 'hotplug') {
4788                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
4789             } elsif ($opt eq 'tablet') {
4790                 die "skip\n" if !$hotplug_features->{usb};
4791                 if ($defaults->{tablet}) {
4792                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4793                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4794                         if $arch eq 'aarch64';
4795                 } else {
4796                     vm_deviceunplug($vmid, $conf, 'tablet');
4797                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4798                 }
4799             } elsif ($opt =~ m/^usb\d+/) {
4800                 die "skip\n";
4801                 # since we cannot reliably hot unplug usb devices
4802                 # we are disabling it
4803                 die "skip\n" if !$hotplug_features->{usb} || $conf->{$opt} =~ m/spice/i;
4804                 vm_deviceunplug($vmid, $conf, $opt);
4805             } elsif ($opt eq 'vcpus') {
4806                 die "skip\n" if !$hotplug_features->{cpu};
4807                 qemu_cpu_hotplug($vmid, $conf, undef);
4808             } elsif ($opt eq 'balloon') {
4809                 # enable balloon device is not hotpluggable
4810                 die "skip\n" if defined($conf->{balloon}) && $conf->{balloon} == 0;
4811                 # here we reset the ballooning value to memory
4812                 my $balloon = $conf->{memory} || $defaults->{memory};
4813                 mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4814             } elsif ($fast_plug_option->{$opt}) {
4815                 # do nothing
4816             } elsif ($opt =~ m/^net(\d+)$/) {
4817                 die "skip\n" if !$hotplug_features->{network};
4818                 vm_deviceunplug($vmid, $conf, $opt);
4819             } elsif (is_valid_drivename($opt)) {
4820                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4821                 vm_deviceunplug($vmid, $conf, $opt);
4822                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4823             } elsif ($opt =~ m/^memory$/) {
4824                 die "skip\n" if !$hotplug_features->{memory};
4825                 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
4826             } elsif ($opt eq 'cpuunits') {
4827                 cgroups_write("cpu", $vmid, "cpu.shares", $defaults->{cpuunits});
4828             } elsif ($opt eq 'cpulimit') {
4829                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", -1);
4830             } else {
4831                 die "skip\n";
4832             }
4833         };
4834         if (my $err = $@) {
4835             &$add_error($opt, $err) if $err ne "skip\n";
4836         } else {
4837             # save new config if hotplug was successful
4838             delete $conf->{$opt};
4839             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4840             PVE::QemuConfig->write_config($vmid, $conf);
4841             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4842         }
4843     }
4844
4845     my ($apply_pending_cloudinit, $apply_pending_cloudinit_done);
4846     $apply_pending_cloudinit = sub {
4847         return if $apply_pending_cloudinit_done; # once is enough
4848         $apply_pending_cloudinit_done = 1; # once is enough
4849
4850         my ($key, $value) = @_;
4851
4852         my @cloudinit_opts = keys %$confdesc_cloudinit;
4853         foreach my $opt (keys %{$conf->{pending}}) {
4854             next if !grep { $_ eq $opt } @cloudinit_opts;
4855             $conf->{$opt} = delete $conf->{pending}->{$opt};
4856         }
4857
4858         my $new_conf = { %$conf };
4859         $new_conf->{$key} = $value;
4860         PVE::QemuServer::Cloudinit::generate_cloudinitconfig($new_conf, $vmid);
4861     };
4862
4863     foreach my $opt (keys %{$conf->{pending}}) {
4864         next if $selection && !$selection->{$opt};
4865         my $value = $conf->{pending}->{$opt};
4866         eval {
4867             if ($opt eq 'hotplug') {
4868                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
4869             } elsif ($opt eq 'tablet') {
4870                 die "skip\n" if !$hotplug_features->{usb};
4871                 if ($value == 1) {
4872                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4873                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4874                         if $arch eq 'aarch64';
4875                 } elsif ($value == 0) {
4876                     vm_deviceunplug($vmid, $conf, 'tablet');
4877                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4878                 }
4879             } elsif ($opt =~ m/^usb\d+$/) {
4880                 die "skip\n";
4881                 # since we cannot reliably hot unplug usb devices
4882                 # we are disabling it
4883                 die "skip\n" if !$hotplug_features->{usb} || $value =~ m/spice/i;
4884                 my $d = eval { PVE::JSONSchema::parse_property_string($usbdesc->{format}, $value) };
4885                 die "skip\n" if !$d;
4886                 qemu_usb_hotplug($storecfg, $conf, $vmid, $opt, $d, $arch, $machine_type);
4887             } elsif ($opt eq 'vcpus') {
4888                 die "skip\n" if !$hotplug_features->{cpu};
4889                 qemu_cpu_hotplug($vmid, $conf, $value);
4890             } elsif ($opt eq 'balloon') {
4891                 # enable/disable balloning device is not hotpluggable
4892                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
4893                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
4894                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
4895
4896                 # allow manual ballooning if shares is set to zero
4897                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
4898                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
4899                     mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4900                 }
4901             } elsif ($opt =~ m/^net(\d+)$/) {
4902                 # some changes can be done without hotplug
4903                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
4904                                     $vmid, $opt, $value, $arch, $machine_type);
4905             } elsif (is_valid_drivename($opt)) {
4906                 die "skip\n" if $opt eq 'efidisk0';
4907                 # some changes can be done without hotplug
4908                 my $drive = parse_drive($opt, $value);
4909                 if (drive_is_cloudinit($drive)) {
4910                     &$apply_pending_cloudinit($opt, $value);
4911                 }
4912                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
4913                                      $vmid, $opt, $value, 1, $arch, $machine_type);
4914             } elsif ($opt =~ m/^memory$/) { #dimms
4915                 die "skip\n" if !$hotplug_features->{memory};
4916                 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
4917             } elsif ($opt eq 'cpuunits') {
4918                 cgroups_write("cpu", $vmid, "cpu.shares", $conf->{pending}->{$opt});
4919             } elsif ($opt eq 'cpulimit') {
4920                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
4921                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", $cpulimit);
4922             } else {
4923                 die "skip\n";  # skip non-hot-pluggable options
4924             }
4925         };
4926         if (my $err = $@) {
4927             &$add_error($opt, $err) if $err ne "skip\n";
4928         } else {
4929             # save new config if hotplug was successful
4930             $conf->{$opt} = $value;
4931             delete $conf->{pending}->{$opt};
4932             PVE::QemuConfig->write_config($vmid, $conf);
4933             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4934         }
4935     }
4936 }
4937
4938 sub try_deallocate_drive {
4939     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
4940
4941     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
4942         my $volid = $drive->{file};
4943         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
4944             my $sid = PVE::Storage::parse_volume_id($volid);
4945             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
4946
4947             # check if the disk is really unused
4948             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
4949                 if is_volume_in_use($storecfg, $conf, $key, $volid);
4950             PVE::Storage::vdisk_free($storecfg, $volid);
4951             return 1;
4952         } else {
4953             # If vm is not owner of this disk remove from config
4954             return 1;
4955         }
4956     }
4957
4958     return undef;
4959 }
4960
4961 sub vmconfig_delete_or_detach_drive {
4962     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
4963
4964     my $drive = parse_drive($opt, $conf->{$opt});
4965
4966     my $rpcenv = PVE::RPCEnvironment::get();
4967     my $authuser = $rpcenv->get_user();
4968
4969     if ($force) {
4970         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
4971         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
4972     } else {
4973         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
4974     }
4975 }
4976
4977
4978
4979 sub vmconfig_apply_pending {
4980     my ($vmid, $conf, $storecfg) = @_;
4981
4982     # cold plug
4983
4984     my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4985     foreach my $opt (sort keys %$pending_delete_hash) {
4986         die "internal error" if $opt =~ m/^unused/;
4987         my $force = $pending_delete_hash->{$opt}->{force};
4988         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4989         if (!defined($conf->{$opt})) {
4990             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4991             PVE::QemuConfig->write_config($vmid, $conf);
4992         } elsif (is_valid_drivename($opt)) {
4993             vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4994             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4995             delete $conf->{$opt};
4996             PVE::QemuConfig->write_config($vmid, $conf);
4997         } else {
4998             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4999             delete $conf->{$opt};
5000             PVE::QemuConfig->write_config($vmid, $conf);
5001         }
5002     }
5003
5004     $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5005
5006     foreach my $opt (keys %{$conf->{pending}}) { # add/change
5007         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5008
5009         if (defined($conf->{$opt}) && ($conf->{$opt} eq $conf->{pending}->{$opt})) {
5010             # skip if nothing changed
5011         } elsif (is_valid_drivename($opt)) {
5012             vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
5013                 if defined($conf->{$opt});
5014             $conf->{$opt} = $conf->{pending}->{$opt};
5015         } else {
5016             $conf->{$opt} = $conf->{pending}->{$opt};
5017         }
5018
5019         delete $conf->{pending}->{$opt};
5020         PVE::QemuConfig->write_config($vmid, $conf);
5021     }
5022 }
5023
5024 my $safe_num_ne = sub {
5025     my ($a, $b) = @_;
5026
5027     return 0 if !defined($a) && !defined($b);
5028     return 1 if !defined($a);
5029     return 1 if !defined($b);
5030
5031     return $a != $b;
5032 };
5033
5034 my $safe_string_ne = sub {
5035     my ($a, $b) = @_;
5036
5037     return 0 if !defined($a) && !defined($b);
5038     return 1 if !defined($a);
5039     return 1 if !defined($b);
5040
5041     return $a ne $b;
5042 };
5043
5044 sub vmconfig_update_net {
5045     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
5046
5047     my $newnet = parse_net($value);
5048
5049     if ($conf->{$opt}) {
5050         my $oldnet = parse_net($conf->{$opt});
5051
5052         if (&$safe_string_ne($oldnet->{model}, $newnet->{model}) ||
5053             &$safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
5054             &$safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
5055             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
5056
5057             # for non online change, we try to hot-unplug
5058             die "skip\n" if !$hotplug;
5059             vm_deviceunplug($vmid, $conf, $opt);
5060         } else {
5061
5062             die "internal error" if $opt !~ m/net(\d+)/;
5063             my $iface = "tap${vmid}i$1";
5064
5065             if (&$safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
5066                 &$safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
5067                 &$safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
5068                 &$safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
5069                 PVE::Network::tap_unplug($iface);
5070                 PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
5071             } elsif (&$safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
5072                 # Rate can be applied on its own but any change above needs to
5073                 # include the rate in tap_plug since OVS resets everything.
5074                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
5075             }
5076
5077             if (&$safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
5078                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
5079             }
5080
5081             return 1;
5082         }
5083     }
5084
5085     if ($hotplug) {
5086         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet, $arch, $machine_type);
5087     } else {
5088         die "skip\n";
5089     }
5090 }
5091
5092 sub vmconfig_update_disk {
5093     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $force, $arch, $machine_type) = @_;
5094
5095     # fixme: do we need force?
5096
5097     my $drive = parse_drive($opt, $value);
5098
5099     if ($conf->{$opt}) {
5100
5101         if (my $old_drive = parse_drive($opt, $conf->{$opt}))  {
5102
5103             my $media = $drive->{media} || 'disk';
5104             my $oldmedia = $old_drive->{media} || 'disk';
5105             die "unable to change media type\n" if $media ne $oldmedia;
5106
5107             if (!drive_is_cdrom($old_drive)) {
5108
5109                 if ($drive->{file} ne $old_drive->{file}) {
5110
5111                     die "skip\n" if !$hotplug;
5112
5113                     # unplug and register as unused
5114                     vm_deviceunplug($vmid, $conf, $opt);
5115                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
5116
5117                 } else {
5118                     # update existing disk
5119
5120                     # skip non hotpluggable value
5121                     if (&$safe_string_ne($drive->{discard}, $old_drive->{discard}) ||
5122                         &$safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
5123                         &$safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
5124                         &$safe_string_ne($drive->{cache}, $old_drive->{cache})) {
5125                         die "skip\n";
5126                     }
5127
5128                     # apply throttle
5129                     if (&$safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
5130                         &$safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
5131                         &$safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
5132                         &$safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
5133                         &$safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
5134                         &$safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
5135                         &$safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
5136                         &$safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
5137                         &$safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
5138                         &$safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
5139                         &$safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
5140                         &$safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max}) ||
5141                         &$safe_num_ne($drive->{bps_max_length}, $old_drive->{bps_max_length}) ||
5142                         &$safe_num_ne($drive->{bps_rd_max_length}, $old_drive->{bps_rd_max_length}) ||
5143                         &$safe_num_ne($drive->{bps_wr_max_length}, $old_drive->{bps_wr_max_length}) ||
5144                         &$safe_num_ne($drive->{iops_max_length}, $old_drive->{iops_max_length}) ||
5145                         &$safe_num_ne($drive->{iops_rd_max_length}, $old_drive->{iops_rd_max_length}) ||
5146                         &$safe_num_ne($drive->{iops_wr_max_length}, $old_drive->{iops_wr_max_length})) {
5147
5148                         qemu_block_set_io_throttle($vmid,"drive-$opt",
5149                                                    ($drive->{mbps} || 0)*1024*1024,
5150                                                    ($drive->{mbps_rd} || 0)*1024*1024,
5151                                                    ($drive->{mbps_wr} || 0)*1024*1024,
5152                                                    $drive->{iops} || 0,
5153                                                    $drive->{iops_rd} || 0,
5154                                                    $drive->{iops_wr} || 0,
5155                                                    ($drive->{mbps_max} || 0)*1024*1024,
5156                                                    ($drive->{mbps_rd_max} || 0)*1024*1024,
5157                                                    ($drive->{mbps_wr_max} || 0)*1024*1024,
5158                                                    $drive->{iops_max} || 0,
5159                                                    $drive->{iops_rd_max} || 0,
5160                                                    $drive->{iops_wr_max} || 0,
5161                                                    $drive->{bps_max_length} || 1,
5162                                                    $drive->{bps_rd_max_length} || 1,
5163                                                    $drive->{bps_wr_max_length} || 1,
5164                                                    $drive->{iops_max_length} || 1,
5165                                                    $drive->{iops_rd_max_length} || 1,
5166                                                    $drive->{iops_wr_max_length} || 1);
5167
5168                     }
5169
5170                     return 1;
5171                 }
5172
5173             } else { # cdrom
5174
5175                 if ($drive->{file} eq 'none') {
5176                     mon_cmd($vmid, "eject",force => JSON::true,device => "drive-$opt");
5177                     if (drive_is_cloudinit($old_drive)) {
5178                         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive);
5179                     }
5180                 } else {
5181                     my $path = get_iso_path($storecfg, $vmid, $drive->{file});
5182                     mon_cmd($vmid, "eject", force => JSON::true,device => "drive-$opt"); # force eject if locked
5183                     mon_cmd($vmid, "change", device => "drive-$opt",target => "$path") if $path;
5184                 }
5185
5186                 return 1;
5187             }
5188         }
5189     }
5190
5191     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
5192     # hotplug new disks
5193     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
5194     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive, $arch, $machine_type);
5195 }
5196
5197 sub vm_start {
5198     my ($storecfg, $vmid, $statefile, $skiplock, $migratedfrom, $paused,
5199         $forcemachine, $spice_ticket, $migration_network, $migration_type, $targetstorage) = @_;
5200
5201     PVE::QemuConfig->lock_config($vmid, sub {
5202         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
5203
5204         die "you can't start a vm if it's a template\n" if PVE::QemuConfig->is_template($conf);
5205
5206         my $is_suspended = PVE::QemuConfig->has_lock($conf, 'suspended');
5207
5208         PVE::QemuConfig->check_lock($conf)
5209             if !($skiplock || $is_suspended);
5210
5211         die "VM $vmid already running\n" if check_running($vmid, undef, $migratedfrom);
5212
5213         # clean up leftover reboot request files
5214         eval { clear_reboot_request($vmid); };
5215         warn $@ if $@;
5216
5217         if (!$statefile && scalar(keys %{$conf->{pending}})) {
5218             vmconfig_apply_pending($vmid, $conf, $storecfg);
5219             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5220         }
5221
5222         PVE::QemuServer::Cloudinit::generate_cloudinitconfig($conf, $vmid);
5223
5224         my $defaults = load_defaults();
5225
5226         # set environment variable useful inside network script
5227         $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
5228
5229         my $local_volumes = {};
5230
5231         if ($targetstorage) {
5232             foreach_drive($conf, sub {
5233                 my ($ds, $drive) = @_;
5234
5235                 return if drive_is_cdrom($drive);
5236
5237                 my $volid = $drive->{file};
5238
5239                 return if !$volid;
5240
5241                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
5242
5243                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5244                 return if $scfg->{shared};
5245                 $local_volumes->{$ds} = [$volid, $storeid, $volname];
5246             });
5247
5248             my $format = undef;
5249
5250             foreach my $opt (sort keys %$local_volumes) {
5251
5252                 my ($volid, $storeid, $volname) = @{$local_volumes->{$opt}};
5253                 my $drive = parse_drive($opt, $conf->{$opt});
5254
5255                 #if remote storage is specified, use default format
5256                 if ($targetstorage && $targetstorage ne "1") {
5257                     $storeid = $targetstorage;
5258                     my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
5259                     $format = $defFormat;
5260                 } else {
5261                     #else we use same format than original
5262                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5263                     $format = qemu_img_format($scfg, $volid);
5264                 }
5265
5266                 my $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $format, undef, ($drive->{size}/1024));
5267                 my $newdrive = $drive;
5268                 $newdrive->{format} = $format;
5269                 $newdrive->{file} = $newvolid;
5270                 my $drivestr = print_drive($newdrive);
5271                 $local_volumes->{$opt} = $drivestr;
5272                 #pass drive to conf for command line
5273                 $conf->{$opt} = $drivestr;
5274             }
5275         }
5276
5277         PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-start', 1);
5278
5279         if ($is_suspended) {
5280             # enforce machine type on suspended vm to ensure HW compatibility
5281             $forcemachine = $conf->{runningmachine};
5282             print "Resuming suspended VM\n";
5283         }
5284
5285         my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine);
5286
5287         my $migration_ip;
5288         my $get_migration_ip = sub {
5289             my ($cidr, $nodename) = @_;
5290
5291             return $migration_ip if defined($migration_ip);
5292
5293             if (!defined($cidr)) {
5294                 my $dc_conf = PVE::Cluster::cfs_read_file('datacenter.cfg');
5295                 $cidr = $dc_conf->{migration}->{network};
5296             }
5297
5298             if (defined($cidr)) {
5299                 my $ips = PVE::Network::get_local_ip_from_cidr($cidr);
5300
5301                 die "could not get IP: no address configured on local " .
5302                     "node for network '$cidr'\n" if scalar(@$ips) == 0;
5303
5304                 die "could not get IP: multiple addresses configured on local " .
5305                     "node for network '$cidr'\n" if scalar(@$ips) > 1;
5306
5307                 $migration_ip = @$ips[0];
5308             }
5309
5310             $migration_ip = PVE::Cluster::remote_node_ip($nodename, 1)
5311                 if !defined($migration_ip);
5312
5313             return $migration_ip;
5314         };
5315
5316         my $migrate_uri;
5317         if ($statefile) {
5318             if ($statefile eq 'tcp') {
5319                 my $localip = "localhost";
5320                 my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
5321                 my $nodename = PVE::INotify::nodename();
5322
5323                 if (!defined($migration_type)) {
5324                     if (defined($datacenterconf->{migration}->{type})) {
5325                         $migration_type = $datacenterconf->{migration}->{type};
5326                     } else {
5327                         $migration_type = 'secure';
5328                     }
5329                 }
5330
5331                 if ($migration_type eq 'insecure') {
5332                     $localip = $get_migration_ip->($migration_network, $nodename);
5333                     $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5334                 }
5335
5336                 my $pfamily = PVE::Tools::get_host_address_family($nodename);
5337                 my $migrate_port = PVE::Tools::next_migrate_port($pfamily);
5338                 $migrate_uri = "tcp:${localip}:${migrate_port}";
5339                 push @$cmd, '-incoming', $migrate_uri;
5340                 push @$cmd, '-S';
5341
5342             } elsif ($statefile eq 'unix') {
5343                 # should be default for secure migrations as a ssh TCP forward
5344                 # tunnel is not deterministic reliable ready and fails regurarly
5345                 # to set up in time, so use UNIX socket forwards
5346                 my $socket_addr = "/run/qemu-server/$vmid.migrate";
5347                 unlink $socket_addr;
5348
5349                 $migrate_uri = "unix:$socket_addr";
5350
5351                 push @$cmd, '-incoming', $migrate_uri;
5352                 push @$cmd, '-S';
5353
5354             } elsif (-e $statefile) {
5355                 push @$cmd, '-loadstate', $statefile;
5356             } else {
5357                 my $statepath = PVE::Storage::path($storecfg, $statefile);
5358                 push @$vollist, $statefile;
5359                 push @$cmd, '-loadstate', $statepath;
5360             }
5361         } elsif ($paused) {
5362             push @$cmd, '-S';
5363         }
5364
5365         # host pci devices
5366         for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
5367           my $d = parse_hostpci($conf->{"hostpci$i"});
5368           next if !$d;
5369           my $pcidevices = $d->{pciid};
5370           foreach my $pcidevice (@$pcidevices) {
5371                 my $pciid = $pcidevice->{id};
5372
5373                 my $info = PVE::SysFSTools::pci_device_info("$pciid");
5374                 die "IOMMU not present\n" if !PVE::SysFSTools::check_iommu_support();
5375                 die "no pci device info for device '$pciid'\n" if !$info;
5376
5377                 if ($d->{mdev}) {
5378                     my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $i);
5379                     PVE::SysFSTools::pci_create_mdev_device($pciid, $uuid, $d->{mdev});
5380                 } else {
5381                     die "can't unbind/bind pci group to vfio '$pciid'\n"
5382                         if !PVE::SysFSTools::pci_dev_group_bind_to_vfio($pciid);
5383                     die "can't reset pci device '$pciid'\n"
5384                         if $info->{has_fl_reset} and !PVE::SysFSTools::pci_dev_reset($info);
5385                 }
5386           }
5387         }
5388
5389         PVE::Storage::activate_volumes($storecfg, $vollist);
5390
5391         eval {
5392             run_command(['/bin/systemctl', 'stop', "$vmid.scope"],
5393                 outfunc => sub {}, errfunc => sub {});
5394         };
5395         # Issues with the above 'stop' not being fully completed are extremely rare, a very low
5396         # timeout should be more than enough here...
5397         PVE::Systemd::wait_for_unit_removed("$vmid.scope", 5);
5398
5399         my $cpuunits = defined($conf->{cpuunits}) ? $conf->{cpuunits}
5400                                                   : $defaults->{cpuunits};
5401
5402         my $start_timeout = ($conf->{hugepages} || $is_suspended) ? 300 : 30;
5403         my %run_params = (timeout => $statefile ? undef : $start_timeout, umask => 0077);
5404
5405         my %properties = (
5406             Slice => 'qemu.slice',
5407             KillMode => 'none',
5408             CPUShares => $cpuunits
5409         );
5410
5411         if (my $cpulimit = $conf->{cpulimit}) {
5412             $properties{CPUQuota} = int($cpulimit * 100);
5413         }
5414         $properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
5415
5416         my $run_qemu = sub {
5417             PVE::Tools::run_fork sub {
5418                 PVE::Systemd::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
5419                 run_command($cmd, %run_params);
5420             };
5421         };
5422
5423         if ($conf->{hugepages}) {
5424
5425             my $code = sub {
5426                 my $hugepages_topology = PVE::QemuServer::Memory::hugepages_topology($conf);
5427                 my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
5428
5429                 PVE::QemuServer::Memory::hugepages_mount();
5430                 PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
5431
5432                 eval { $run_qemu->() };
5433                 if (my $err = $@) {
5434                     PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology);
5435                     die $err;
5436                 }
5437
5438                 PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology);
5439             };
5440             eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
5441
5442         } else {
5443             eval { $run_qemu->() };
5444         }
5445
5446         if (my $err = $@) {
5447             # deactivate volumes if start fails
5448             eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
5449             die "start failed: $err";
5450         }
5451
5452         print "migration listens on $migrate_uri\n" if $migrate_uri;
5453
5454         if ($statefile && $statefile ne 'tcp' && $statefile ne 'unix')  {
5455             eval { mon_cmd($vmid, "cont"); };
5456             warn $@ if $@;
5457         }
5458
5459         #start nbd server for storage migration
5460         if ($targetstorage) {
5461             my $nodename = PVE::INotify::nodename();
5462             my $localip = $get_migration_ip->($migration_network, $nodename);
5463             my $pfamily = PVE::Tools::get_host_address_family($nodename);
5464             my $storage_migrate_port = PVE::Tools::next_migrate_port($pfamily);
5465
5466             mon_cmd($vmid, "nbd-server-start", addr => { type => 'inet', data => { host => "${localip}", port => "${storage_migrate_port}" } } );
5467
5468             $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5469
5470             foreach my $opt (sort keys %$local_volumes) {
5471                 my $volid = $local_volumes->{$opt};
5472                 mon_cmd($vmid, "nbd-server-add", device => "drive-$opt", writable => JSON::true );
5473                 my $migrate_storage_uri = "nbd:${localip}:${storage_migrate_port}:exportname=drive-$opt";
5474                 print "storage migration listens on $migrate_storage_uri volume:$volid\n";
5475             }
5476         }
5477
5478         if ($migratedfrom) {
5479             eval {
5480                 set_migration_caps($vmid);
5481             };
5482             warn $@ if $@;
5483
5484             if ($spice_port) {
5485                 print "spice listens on port $spice_port\n";
5486                 if ($spice_ticket) {
5487                     mon_cmd($vmid, "set_password", protocol => 'spice', password => $spice_ticket);
5488                     mon_cmd($vmid, "expire_password", protocol => 'spice', time => "+30");
5489                 }
5490             }
5491
5492         } else {
5493             mon_cmd($vmid, "balloon", value => $conf->{balloon}*1024*1024)
5494                 if !$statefile && $conf->{balloon};
5495
5496             foreach my $opt (keys %$conf) {
5497                 next if $opt !~  m/^net\d+$/;
5498                 my $nicconf = parse_net($conf->{$opt});
5499                 qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
5500             }
5501         }
5502
5503         mon_cmd($vmid, 'qom-set',
5504                     path => "machine/peripheral/balloon0",
5505                     property => "guest-stats-polling-interval",
5506                     value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
5507
5508         if ($is_suspended) {
5509             print "Resumed VM, removing state\n";
5510             if (my $vmstate = $conf->{vmstate}) {
5511                 PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
5512                 PVE::Storage::vdisk_free($storecfg, $vmstate);
5513             }
5514             delete $conf->@{qw(lock vmstate runningmachine)};
5515             PVE::QemuConfig->write_config($vmid, $conf);
5516         }
5517
5518         PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'post-start');
5519     });
5520 }
5521
5522 sub vm_commandline {
5523     my ($storecfg, $vmid, $snapname) = @_;
5524
5525     my $conf = PVE::QemuConfig->load_config($vmid);
5526     my $forcemachine;
5527
5528     if ($snapname) {
5529         my $snapshot = $conf->{snapshots}->{$snapname};
5530         die "snapshot '$snapname' does not exist\n" if !defined($snapshot);
5531
5532         # check for a 'runningmachine' in snapshot
5533         $forcemachine = $snapshot->{runningmachine} if $snapshot->{runningmachine};
5534
5535         $snapshot->{digest} = $conf->{digest}; # keep file digest for API
5536
5537         $conf = $snapshot;
5538     }
5539
5540     my $defaults = load_defaults();
5541
5542     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine);
5543
5544     return PVE::Tools::cmd2string($cmd);
5545 }
5546
5547 sub vm_reset {
5548     my ($vmid, $skiplock) = @_;
5549
5550     PVE::QemuConfig->lock_config($vmid, sub {
5551
5552         my $conf = PVE::QemuConfig->load_config($vmid);
5553
5554         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5555
5556         mon_cmd($vmid, "system_reset");
5557     });
5558 }
5559
5560 sub get_vm_volumes {
5561     my ($conf) = @_;
5562
5563     my $vollist = [];
5564     foreach_volid($conf, sub {
5565         my ($volid, $attr) = @_;
5566
5567         return if $volid =~ m|^/|;
5568
5569         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5570         return if !$sid;
5571
5572         push @$vollist, $volid;
5573     });
5574
5575     return $vollist;
5576 }
5577
5578 sub vm_stop_cleanup {
5579     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
5580
5581     eval {
5582
5583         if (!$keepActive) {
5584             my $vollist = get_vm_volumes($conf);
5585             PVE::Storage::deactivate_volumes($storecfg, $vollist);
5586         }
5587
5588         foreach my $ext (qw(mon qmp pid vnc qga)) {
5589             unlink "/var/run/qemu-server/${vmid}.$ext";
5590         }
5591
5592         if ($conf->{ivshmem}) {
5593             my $ivshmem = PVE::JSONSchema::parse_property_string($ivshmem_fmt, $conf->{ivshmem});
5594             # just delete it for now, VMs which have this already open do not
5595             # are affected, but new VMs will get a separated one. If this
5596             # becomes an issue we either add some sort of ref-counting or just
5597             # add a "don't delete on stop" flag to the ivshmem format.
5598             unlink '/dev/shm/pve-shm-' . ($ivshmem->{name} // $vmid);
5599         }
5600
5601         foreach my $key (keys %$conf) {
5602             next if $key !~ m/^hostpci(\d+)$/;
5603             my $hostpciindex = $1;
5604             my $d = parse_hostpci($conf->{$key});
5605             my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $hostpciindex);
5606
5607             foreach my $pci (@{$d->{pciid}}) {
5608                 my $pciid = $pci->{id};
5609                 PVE::SysFSTools::pci_cleanup_mdev_device($pciid, $uuid);
5610             }
5611         }
5612
5613         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
5614     };
5615     warn $@ if $@; # avoid errors - just warn
5616 }
5617
5618 # call only in locked context
5619 sub _do_vm_stop {
5620     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive) = @_;
5621
5622     my $pid = check_running($vmid, $nocheck);
5623     return if !$pid;
5624
5625     my $conf;
5626     if (!$nocheck) {
5627         $conf = PVE::QemuConfig->load_config($vmid);
5628         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5629         if (!defined($timeout) && $shutdown && $conf->{startup}) {
5630             my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
5631             $timeout = $opts->{down} if $opts->{down};
5632         }
5633         PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-stop');
5634     }
5635
5636     eval {
5637         if ($shutdown) {
5638             if (defined($conf) && parse_guest_agent($conf)->{enabled}) {
5639                 mon_cmd($vmid, "guest-shutdown", timeout => $timeout);
5640             } else {
5641                 mon_cmd($vmid, "system_powerdown");
5642             }
5643         } else {
5644             mon_cmd($vmid, "quit");
5645         }
5646     };
5647     my $err = $@;
5648
5649     if (!$err) {
5650         $timeout = 60 if !defined($timeout);
5651
5652         my $count = 0;
5653         while (($count < $timeout) && check_running($vmid, $nocheck)) {
5654             $count++;
5655             sleep 1;
5656         }
5657
5658         if ($count >= $timeout) {
5659             if ($force) {
5660                 warn "VM still running - terminating now with SIGTERM\n";
5661                 kill 15, $pid;
5662             } else {
5663                 die "VM quit/powerdown failed - got timeout\n";
5664             }
5665         } else {
5666             vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5667             return;
5668         }
5669     } else {
5670         if ($force) {
5671             warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
5672             kill 15, $pid;
5673         } else {
5674             die "VM quit/powerdown failed\n";
5675         }
5676     }
5677
5678     # wait again
5679     $timeout = 10;
5680
5681     my $count = 0;
5682     while (($count < $timeout) && check_running($vmid, $nocheck)) {
5683         $count++;
5684         sleep 1;
5685     }
5686
5687     if ($count >= $timeout) {
5688         warn "VM still running - terminating now with SIGKILL\n";
5689         kill 9, $pid;
5690         sleep 1;
5691     }
5692
5693     vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5694 }
5695
5696 # Note: use $nocheck to skip tests if VM configuration file exists.
5697 # We need that when migration VMs to other nodes (files already moved)
5698 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
5699 sub vm_stop {
5700     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
5701
5702     $force = 1 if !defined($force) && !$shutdown;
5703
5704     if ($migratedfrom){
5705         my $pid = check_running($vmid, $nocheck, $migratedfrom);
5706         kill 15, $pid if $pid;
5707         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
5708         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
5709         return;
5710     }
5711
5712     PVE::QemuConfig->lock_config($vmid, sub {
5713         _do_vm_stop($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive);
5714    });
5715 }
5716
5717 sub vm_reboot {
5718     my ($vmid, $timeout) = @_;
5719
5720     PVE::QemuConfig->lock_config($vmid, sub {
5721         eval {
5722
5723             # only reboot if running, as qmeventd starts it again on a stop event
5724             return if !check_running($vmid);
5725
5726             create_reboot_request($vmid);
5727
5728             my $storecfg = PVE::Storage::config();
5729             _do_vm_stop($storecfg, $vmid, undef, undef, $timeout, 1);
5730
5731         };
5732         if (my $err = $@) {
5733             # avoid that the next normal shutdown will be confused for a reboot
5734             clear_reboot_request($vmid);
5735             die $err;
5736         }
5737    });
5738 }
5739
5740 sub vm_suspend {
5741     my ($vmid, $skiplock, $includestate, $statestorage) = @_;
5742
5743     my $conf;
5744     my $path;
5745     my $storecfg;
5746     my $vmstate;
5747
5748     PVE::QemuConfig->lock_config($vmid, sub {
5749
5750         $conf = PVE::QemuConfig->load_config($vmid);
5751
5752         my $is_backing_up = PVE::QemuConfig->has_lock($conf, 'backup');
5753         PVE::QemuConfig->check_lock($conf)
5754             if !($skiplock || $is_backing_up);
5755
5756         die "cannot suspend to disk during backup\n"
5757             if $is_backing_up && $includestate;
5758
5759         if ($includestate) {
5760             $conf->{lock} = 'suspending';
5761             my $date = strftime("%Y-%m-%d", localtime(time()));
5762             $storecfg = PVE::Storage::config();
5763             $vmstate = PVE::QemuConfig->__snapshot_save_vmstate($vmid, $conf, "suspend-$date", $storecfg, $statestorage, 1);
5764             $path = PVE::Storage::path($storecfg, $vmstate);
5765             PVE::QemuConfig->write_config($vmid, $conf);
5766         } else {
5767             mon_cmd($vmid, "stop");
5768         }
5769     });
5770
5771     if ($includestate) {
5772         # save vm state
5773         PVE::Storage::activate_volumes($storecfg, [$vmstate]);
5774
5775         eval {
5776             mon_cmd($vmid, "savevm-start", statefile => $path);
5777             for(;;) {
5778                 my $state = mon_cmd($vmid, "query-savevm");
5779                 if (!$state->{status}) {
5780                     die "savevm not active\n";
5781                 } elsif ($state->{status} eq 'active') {
5782                     sleep(1);
5783                     next;
5784                 } elsif ($state->{status} eq 'completed') {
5785                     print "State saved, quitting\n";
5786                     last;
5787                 } elsif ($state->{status} eq 'failed' && $state->{error}) {
5788                     die "query-savevm failed with error '$state->{error}'\n"
5789                 } else {
5790                     die "query-savevm returned status '$state->{status}'\n";
5791                 }
5792             }
5793         };
5794         my $err = $@;
5795
5796         PVE::QemuConfig->lock_config($vmid, sub {
5797             $conf = PVE::QemuConfig->load_config($vmid);
5798             if ($err) {
5799                 # cleanup, but leave suspending lock, to indicate something went wrong
5800                 eval {
5801                     mon_cmd($vmid, "savevm-end");
5802                     PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
5803                     PVE::Storage::vdisk_free($storecfg, $vmstate);
5804                     delete $conf->@{qw(vmstate runningmachine)};
5805                     PVE::QemuConfig->write_config($vmid, $conf);
5806                 };
5807                 warn $@ if $@;
5808                 die $err;
5809             }
5810
5811             die "lock changed unexpectedly\n"
5812                 if !PVE::QemuConfig->has_lock($conf, 'suspending');
5813
5814             mon_cmd($vmid, "quit");
5815             $conf->{lock} = 'suspended';
5816             PVE::QemuConfig->write_config($vmid, $conf);
5817         });
5818     }
5819 }
5820
5821 sub vm_resume {
5822     my ($vmid, $skiplock, $nocheck) = @_;
5823
5824     PVE::QemuConfig->lock_config($vmid, sub {
5825         my $res = mon_cmd($vmid, 'query-status');
5826         my $resume_cmd = 'cont';
5827
5828         if ($res->{status} && $res->{status} eq 'suspended') {
5829             $resume_cmd = 'system_wakeup';
5830         }
5831
5832         if (!$nocheck) {
5833
5834             my $conf = PVE::QemuConfig->load_config($vmid);
5835
5836             PVE::QemuConfig->check_lock($conf)
5837                 if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
5838         }
5839
5840         mon_cmd($vmid, $resume_cmd);
5841     });
5842 }
5843
5844 sub vm_sendkey {
5845     my ($vmid, $skiplock, $key) = @_;
5846
5847     PVE::QemuConfig->lock_config($vmid, sub {
5848
5849         my $conf = PVE::QemuConfig->load_config($vmid);
5850
5851         # there is no qmp command, so we use the human monitor command
5852         my $res = PVE::QemuServer::Monitor::hmp_cmd($vmid, "sendkey $key");
5853         die $res if $res ne '';
5854     });
5855 }
5856
5857 # vzdump restore implementaion
5858
5859 sub tar_archive_read_firstfile {
5860     my $archive = shift;
5861
5862     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
5863
5864     # try to detect archive type first
5865     my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
5866         die "unable to open file '$archive'\n";
5867     my $firstfile = <$fh>;
5868     kill 15, $pid;
5869     close $fh;
5870
5871     die "ERROR: archive contaions no data\n" if !$firstfile;
5872     chomp $firstfile;
5873
5874     return $firstfile;
5875 }
5876
5877 sub tar_restore_cleanup {
5878     my ($storecfg, $statfile) = @_;
5879
5880     print STDERR "starting cleanup\n";
5881
5882     if (my $fd = IO::File->new($statfile, "r")) {
5883         while (defined(my $line = <$fd>)) {
5884             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5885                 my $volid = $2;
5886                 eval {
5887                     if ($volid =~ m|^/|) {
5888                         unlink $volid || die 'unlink failed\n';
5889                     } else {
5890                         PVE::Storage::vdisk_free($storecfg, $volid);
5891                     }
5892                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
5893                 };
5894                 print STDERR "unable to cleanup '$volid' - $@" if $@;
5895             } else {
5896                 print STDERR "unable to parse line in statfile - $line";
5897             }
5898         }
5899         $fd->close();
5900     }
5901 }
5902
5903 sub restore_archive {
5904     my ($archive, $vmid, $user, $opts) = @_;
5905
5906     my $format = $opts->{format};
5907     my $comp;
5908
5909     if ($archive =~ m/\.tgz$/ || $archive =~ m/\.tar\.gz$/) {
5910         $format = 'tar' if !$format;
5911         $comp = 'gzip';
5912     } elsif ($archive =~ m/\.tar$/) {
5913         $format = 'tar' if !$format;
5914     } elsif ($archive =~ m/.tar.lzo$/) {
5915         $format = 'tar' if !$format;
5916         $comp = 'lzop';
5917     } elsif ($archive =~ m/\.vma$/) {
5918         $format = 'vma' if !$format;
5919     } elsif ($archive =~ m/\.vma\.gz$/) {
5920         $format = 'vma' if !$format;
5921         $comp = 'gzip';
5922     } elsif ($archive =~ m/\.vma\.lzo$/) {
5923         $format = 'vma' if !$format;
5924         $comp = 'lzop';
5925     } else {
5926         $format = 'vma' if !$format; # default
5927     }
5928
5929     # try to detect archive format
5930     if ($format eq 'tar') {
5931         return restore_tar_archive($archive, $vmid, $user, $opts);
5932     } else {
5933         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
5934     }
5935 }
5936
5937 sub restore_update_config_line {
5938     my ($outfd, $cookie, $vmid, $map, $line, $unique) = @_;
5939
5940     return if $line =~ m/^\#qmdump\#/;
5941     return if $line =~ m/^\#vzdump\#/;
5942     return if $line =~ m/^lock:/;
5943     return if $line =~ m/^unused\d+:/;
5944     return if $line =~ m/^parent:/;
5945
5946     my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
5947     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
5948         # try to convert old 1.X settings
5949         my ($id, $ind, $ethcfg) = ($1, $2, $3);
5950         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
5951             my ($model, $macaddr) = split(/\=/, $devconfig);
5952             $macaddr = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if !$macaddr || $unique;
5953             my $net = {
5954                 model => $model,
5955                 bridge => "vmbr$ind",
5956                 macaddr => $macaddr,
5957             };
5958             my $netstr = print_net($net);
5959
5960             print $outfd "net$cookie->{netcount}: $netstr\n";
5961             $cookie->{netcount}++;
5962         }
5963     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
5964         my ($id, $netstr) = ($1, $2);
5965         my $net = parse_net($netstr);
5966         $net->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if $net->{macaddr};
5967         $netstr = print_net($net);
5968         print $outfd "$id: $netstr\n";
5969     } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk)\d+):\s*(\S+)\s*$/) {
5970         my $virtdev = $1;
5971         my $value = $3;
5972         my $di = parse_drive($virtdev, $value);
5973         if (defined($di->{backup}) && !$di->{backup}) {
5974             print $outfd "#$line";
5975         } elsif ($map->{$virtdev}) {
5976             delete $di->{format}; # format can change on restore
5977             $di->{file} = $map->{$virtdev};
5978             $value = print_drive($di);
5979             print $outfd "$virtdev: $value\n";
5980         } else {
5981             print $outfd $line;
5982         }
5983     } elsif (($line =~ m/^vmgenid: (.*)/)) {
5984         my $vmgenid = $1;
5985         if ($vmgenid ne '0') {
5986             # always generate a new vmgenid if there was a valid one setup
5987             $vmgenid = generate_uuid();
5988         }
5989         print $outfd "vmgenid: $vmgenid\n";
5990     } elsif (($line =~ m/^(smbios1: )(.*)/) && $unique) {
5991         my ($uuid, $uuid_str);
5992         UUID::generate($uuid);
5993         UUID::unparse($uuid, $uuid_str);
5994         my $smbios1 = parse_smbios1($2);
5995         $smbios1->{uuid} = $uuid_str;
5996         print $outfd $1.print_smbios1($smbios1)."\n";
5997     } else {
5998         print $outfd $line;
5999     }
6000 }
6001
6002 sub scan_volids {
6003     my ($cfg, $vmid) = @_;
6004
6005     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid);
6006
6007     my $volid_hash = {};
6008     foreach my $storeid (keys %$info) {
6009         foreach my $item (@{$info->{$storeid}}) {
6010             next if !($item->{volid} && $item->{size});
6011             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
6012             $volid_hash->{$item->{volid}} = $item;
6013         }
6014     }
6015
6016     return $volid_hash;
6017 }
6018
6019 sub is_volume_in_use {
6020     my ($storecfg, $conf, $skip_drive, $volid) = @_;
6021
6022     my $path = PVE::Storage::path($storecfg, $volid);
6023
6024     my $scan_config = sub {
6025         my ($cref, $snapname) = @_;
6026
6027         foreach my $key (keys %$cref) {
6028             my $value = $cref->{$key};
6029             if (is_valid_drivename($key)) {
6030                 next if $skip_drive && $key eq $skip_drive;
6031                 my $drive = parse_drive($key, $value);
6032                 next if !$drive || !$drive->{file} || drive_is_cdrom($drive);
6033                 return 1 if $volid eq $drive->{file};
6034                 if ($drive->{file} =~ m!^/!) {
6035                     return 1 if $drive->{file} eq $path;
6036                 } else {
6037                     my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file}, 1);
6038                     next if !$storeid;
6039                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid, 1);
6040                     next if !$scfg;
6041                     return 1 if $path eq PVE::Storage::path($storecfg, $drive->{file}, $snapname);
6042                 }
6043             }
6044         }
6045
6046         return 0;
6047     };
6048
6049     return 1 if &$scan_config($conf);
6050
6051     undef $skip_drive;
6052
6053     foreach my $snapname (keys %{$conf->{snapshots}}) {
6054         return 1 if &$scan_config($conf->{snapshots}->{$snapname}, $snapname);
6055     }
6056
6057     return 0;
6058 }
6059
6060 sub update_disksize {
6061     my ($vmid, $conf, $volid_hash) = @_;
6062
6063     my $changes;
6064     my $prefix = "VM $vmid:";
6065
6066     # used and unused disks
6067     my $referenced = {};
6068
6069     # Note: it is allowed to define multiple storages with same path (alias), so
6070     # we need to check both 'volid' and real 'path' (two different volid can point
6071     # to the same path).
6072
6073     my $referencedpath = {};
6074
6075     # update size info
6076     foreach my $opt (keys %$conf) {
6077         if (is_valid_drivename($opt)) {
6078             my $drive = parse_drive($opt, $conf->{$opt});
6079             my $volid = $drive->{file};
6080             next if !$volid;
6081
6082             $referenced->{$volid} = 1;
6083             if ($volid_hash->{$volid} &&
6084                 (my $path = $volid_hash->{$volid}->{path})) {
6085                 $referencedpath->{$path} = 1;
6086             }
6087
6088             next if drive_is_cdrom($drive);
6089             next if !$volid_hash->{$volid};
6090
6091             $drive->{size} = $volid_hash->{$volid}->{size};
6092             my $new = print_drive($drive);
6093             if ($new ne $conf->{$opt}) {
6094                 $changes = 1;
6095                 $conf->{$opt} = $new;
6096                 print "$prefix update disk '$opt' information.\n";
6097             }
6098         }
6099     }
6100
6101     # remove 'unusedX' entry if volume is used
6102     foreach my $opt (keys %$conf) {
6103         next if $opt !~ m/^unused\d+$/;
6104         my $volid = $conf->{$opt};
6105         my $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
6106         if ($referenced->{$volid} || ($path && $referencedpath->{$path})) {
6107             print "$prefix remove entry '$opt', its volume '$volid' is in use.\n";
6108             $changes = 1;
6109             delete $conf->{$opt};
6110         }
6111
6112         $referenced->{$volid} = 1;
6113         $referencedpath->{$path} = 1 if $path;
6114     }
6115
6116     foreach my $volid (sort keys %$volid_hash) {
6117         next if $volid =~ m/vm-$vmid-state-/;
6118         next if $referenced->{$volid};
6119         my $path = $volid_hash->{$volid}->{path};
6120         next if !$path; # just to be sure
6121         next if $referencedpath->{$path};
6122         $changes = 1;
6123         my $key = PVE::QemuConfig->add_unused_volume($conf, $volid);
6124         print "$prefix add unreferenced volume '$volid' as '$key' to config.\n";
6125         $referencedpath->{$path} = 1; # avoid to add more than once (aliases)
6126     }
6127
6128     return $changes;
6129 }
6130
6131 sub rescan {
6132     my ($vmid, $nolock, $dryrun) = @_;
6133
6134     my $cfg = PVE::Storage::config();
6135
6136     # FIXME: Remove once our RBD plugin can handle CT and VM on a single storage
6137     # see: https://pve.proxmox.com/pipermail/pve-devel/2018-July/032900.html
6138     foreach my $stor (keys %{$cfg->{ids}}) {
6139         delete($cfg->{ids}->{$stor}) if ! $cfg->{ids}->{$stor}->{content}->{images};
6140     }
6141
6142     print "rescan volumes...\n";
6143     my $volid_hash = scan_volids($cfg, $vmid);
6144
6145     my $updatefn =  sub {
6146         my ($vmid) = @_;
6147
6148         my $conf = PVE::QemuConfig->load_config($vmid);
6149
6150         PVE::QemuConfig->check_lock($conf);
6151
6152         my $vm_volids = {};
6153         foreach my $volid (keys %$volid_hash) {
6154             my $info = $volid_hash->{$volid};
6155             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
6156         }
6157
6158         my $changes = update_disksize($vmid, $conf, $vm_volids);
6159
6160         PVE::QemuConfig->write_config($vmid, $conf) if $changes && !$dryrun;
6161     };
6162
6163     if (defined($vmid)) {
6164         if ($nolock) {
6165             &$updatefn($vmid);
6166         } else {
6167             PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6168         }
6169     } else {
6170         my $vmlist = config_list();
6171         foreach my $vmid (keys %$vmlist) {
6172             if ($nolock) {
6173                 &$updatefn($vmid);
6174             } else {
6175                 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6176             }
6177         }
6178     }
6179 }
6180
6181 sub restore_vma_archive {
6182     my ($archive, $vmid, $user, $opts, $comp) = @_;
6183
6184     my $readfrom = $archive;
6185
6186     my $cfg = PVE::Storage::config();
6187     my $commands = [];
6188     my $bwlimit = $opts->{bwlimit};
6189
6190     my $dbg_cmdstring = '';
6191     my $add_pipe = sub {
6192         my ($cmd) = @_;
6193         push @$commands, $cmd;
6194         $dbg_cmdstring .= ' | ' if length($dbg_cmdstring);
6195         $dbg_cmdstring .= PVE::Tools::cmd2string($cmd);
6196         $readfrom = '-';
6197     };
6198
6199     my $input = undef;
6200     if ($archive eq '-') {
6201         $input = '<&STDIN';
6202     } else {
6203         # If we use a backup from a PVE defined storage we also consider that
6204         # storage's rate limit:
6205         my (undef, $volid) = PVE::Storage::path_to_volume_id($cfg, $archive);
6206         if (defined($volid)) {
6207             my ($sid, undef) = PVE::Storage::parse_volume_id($volid);
6208             my $readlimit = PVE::Storage::get_bandwidth_limit('restore', [$sid], $bwlimit);
6209             if ($readlimit) {
6210                 print STDERR "applying read rate limit: $readlimit\n";
6211                 my $cstream = ['cstream', '-t', $readlimit*1024, '--', $readfrom];
6212                 $add_pipe->($cstream);
6213             }
6214         }
6215     }
6216
6217     if ($comp) {
6218         my $cmd;
6219         if ($comp eq 'gzip') {
6220             $cmd = ['zcat', $readfrom];
6221         } elsif ($comp eq 'lzop') {
6222             $cmd = ['lzop', '-d', '-c', $readfrom];
6223         } else {
6224             die "unknown compression method '$comp'\n";
6225         }
6226         $add_pipe->($cmd);
6227     }
6228
6229     my $tmpdir = "/var/tmp/vzdumptmp$$";
6230     rmtree $tmpdir;
6231
6232     # disable interrupts (always do cleanups)
6233     local $SIG{INT} =
6234         local $SIG{TERM} =
6235         local $SIG{QUIT} =
6236         local $SIG{HUP} = sub { warn "got interrupt - ignored\n"; };
6237
6238     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
6239     POSIX::mkfifo($mapfifo, 0600);
6240     my $fifofh;
6241
6242     my $openfifo = sub {
6243         open($fifofh, '>', $mapfifo) || die $!;
6244     };
6245
6246     $add_pipe->(['vma', 'extract', '-v', '-r', $mapfifo, $readfrom, $tmpdir]);
6247
6248     my $oldtimeout;
6249     my $timeout = 5;
6250
6251     my $devinfo = {};
6252
6253     my $rpcenv = PVE::RPCEnvironment::get();
6254
6255     my $conffile = PVE::QemuConfig->config_file($vmid);
6256     my $tmpfn = "$conffile.$$.tmp";
6257
6258     # Note: $oldconf is undef if VM does not exists
6259     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
6260     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
6261
6262     my %storage_limits;
6263
6264     my $print_devmap = sub {
6265         my $virtdev_hash = {};
6266
6267         my $cfgfn = "$tmpdir/qemu-server.conf";
6268
6269         # we can read the config - that is already extracted
6270         my $fh = IO::File->new($cfgfn, "r") ||
6271             "unable to read qemu-server.conf - $!\n";
6272
6273         my $fwcfgfn = "$tmpdir/qemu-server.fw";
6274         if (-f $fwcfgfn) {
6275             my $pve_firewall_dir = '/etc/pve/firewall';
6276             mkdir $pve_firewall_dir; # make sure the dir exists
6277             PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
6278         }
6279
6280         while (defined(my $line = <$fh>)) {
6281             if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
6282                 my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
6283                 die "archive does not contain data for drive '$virtdev'\n"
6284                     if !$devinfo->{$devname};
6285                 if (defined($opts->{storage})) {
6286                     $storeid = $opts->{storage} || 'local';
6287                 } elsif (!$storeid) {
6288                     $storeid = 'local';
6289                 }
6290                 $format = 'raw' if !$format;
6291                 $devinfo->{$devname}->{devname} = $devname;
6292                 $devinfo->{$devname}->{virtdev} = $virtdev;
6293                 $devinfo->{$devname}->{format} = $format;
6294                 $devinfo->{$devname}->{storeid} = $storeid;
6295
6296                 # check permission on storage
6297                 my $pool = $opts->{pool}; # todo: do we need that?
6298                 if ($user ne 'root@pam') {
6299                     $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
6300                 }
6301
6302                 $storage_limits{$storeid} = $bwlimit;
6303
6304                 $virtdev_hash->{$virtdev} = $devinfo->{$devname};
6305             } elsif ($line =~ m/^((?:ide|sata|scsi)\d+):\s*(.*)\s*$/) {
6306                 my $virtdev = $1;
6307                 my $drive = parse_drive($virtdev, $2);
6308                 if (drive_is_cloudinit($drive)) {
6309                     my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6310                     my $scfg = PVE::Storage::storage_config($cfg, $storeid);
6311                     my $format = qemu_img_format($scfg, $volname); # has 'raw' fallback
6312
6313                     my $d = {
6314                         format => $format,
6315                         storeid => $opts->{storage} // $storeid,
6316                         size => PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE,
6317                         file => $drive->{file}, # to make drive_is_cloudinit check possible
6318                         name => "vm-$vmid-cloudinit",
6319                         is_cloudinit => 1,
6320                     };
6321                     $virtdev_hash->{$virtdev} = $d;
6322                 }
6323             }
6324         }
6325
6326         foreach my $key (keys %storage_limits) {
6327             my $limit = PVE::Storage::get_bandwidth_limit('restore', [$key], $bwlimit);
6328             next if !$limit;
6329             print STDERR "rate limit for storage $key: $limit KiB/s\n";
6330             $storage_limits{$key} = $limit * 1024;
6331         }
6332
6333         foreach my $devname (keys %$devinfo) {
6334             die "found no device mapping information for device '$devname'\n"
6335                 if !$devinfo->{$devname}->{virtdev};
6336         }
6337
6338         # create empty/temp config
6339         if ($oldconf) {
6340             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
6341             foreach_drive($oldconf, sub {
6342                 my ($ds, $drive) = @_;
6343
6344                 return if drive_is_cdrom($drive, 1);
6345
6346                 my $volid = $drive->{file};
6347                 return if !$volid || $volid =~ m|^/|;
6348
6349                 my ($path, $owner) = PVE::Storage::path($cfg, $volid);
6350                 return if !$path || !$owner || ($owner != $vmid);
6351
6352                 # Note: only delete disk we want to restore
6353                 # other volumes will become unused
6354                 if ($virtdev_hash->{$ds}) {
6355                     eval { PVE::Storage::vdisk_free($cfg, $volid); };
6356                     if (my $err = $@) {
6357                         warn $err;
6358                     }
6359                 }
6360             });
6361
6362             # delete vmstate files, after the restore we have no snapshots anymore
6363             foreach my $snapname (keys %{$oldconf->{snapshots}}) {
6364                 my $snap = $oldconf->{snapshots}->{$snapname};
6365                 if ($snap->{vmstate}) {
6366                     eval { PVE::Storage::vdisk_free($cfg, $snap->{vmstate}); };
6367                     if (my $err = $@) {
6368                         warn $err;
6369                     }
6370                 }
6371             }
6372         }
6373
6374         my $map = {};
6375         foreach my $virtdev (sort keys %$virtdev_hash) {
6376             my $d = $virtdev_hash->{$virtdev};
6377             my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
6378             my $storeid = $d->{storeid};
6379             my $scfg = PVE::Storage::storage_config($cfg, $storeid);
6380
6381             my $map_opts = '';
6382             if (my $limit = $storage_limits{$storeid}) {
6383                 $map_opts .= "throttling.bps=$limit:throttling.group=$storeid:";
6384             }
6385
6386             # test if requested format is supported
6387             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($cfg, $storeid);
6388             my $supported = grep { $_ eq $d->{format} } @$validFormats;
6389             $d->{format} = $defFormat if !$supported;
6390
6391             my $name;
6392             if ($d->{is_cloudinit}) {
6393                 $name = $d->{name};
6394                 $name .= ".$d->{format}" if $d->{format} ne 'raw';
6395             }
6396
6397             my $volid = PVE::Storage::vdisk_alloc($cfg, $storeid, $vmid, $d->{format}, $name, $alloc_size);
6398             print STDERR "new volume ID is '$volid'\n";
6399             $d->{volid} = $volid;
6400
6401             PVE::Storage::activate_volumes($cfg, [$volid]);
6402
6403             my $write_zeros = 1;
6404             if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
6405                 $write_zeros = 0;
6406             }
6407
6408             if (!$d->{is_cloudinit}) {
6409                 my $path = PVE::Storage::path($cfg, $volid);
6410
6411                 print $fifofh "${map_opts}format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
6412
6413                 print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
6414             }
6415             $map->{$virtdev} = $volid;
6416         }
6417
6418         $fh->seek(0, 0) || die "seek failed - $!\n";
6419
6420         my $outfd = new IO::File ($tmpfn, "w") ||
6421             die "unable to write config for VM $vmid\n";
6422
6423         my $cookie = { netcount => 0 };
6424         while (defined(my $line = <$fh>)) {
6425             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
6426         }
6427
6428         $fh->close();
6429         $outfd->close();
6430     };
6431
6432     eval {
6433         # enable interrupts
6434         local $SIG{INT} =
6435             local $SIG{TERM} =
6436             local $SIG{QUIT} =
6437             local $SIG{HUP} =
6438             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6439         local $SIG{ALRM} = sub { die "got timeout\n"; };
6440
6441         $oldtimeout = alarm($timeout);
6442
6443         my $parser = sub {
6444             my $line = shift;
6445
6446             print "$line\n";
6447
6448             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
6449                 my ($dev_id, $size, $devname) = ($1, $2, $3);
6450                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
6451             } elsif ($line =~ m/^CTIME: /) {
6452                 # we correctly received the vma config, so we can disable
6453                 # the timeout now for disk allocation (set to 10 minutes, so
6454                 # that we always timeout if something goes wrong)
6455                 alarm(600);
6456                 &$print_devmap();
6457                 print $fifofh "done\n";
6458                 my $tmp = $oldtimeout || 0;
6459                 $oldtimeout = undef;
6460                 alarm($tmp);
6461                 close($fifofh);
6462             }
6463         };
6464
6465         print "restore vma archive: $dbg_cmdstring\n";
6466         run_command($commands, input => $input, outfunc => $parser, afterfork => $openfifo);
6467     };
6468     my $err = $@;
6469
6470     alarm($oldtimeout) if $oldtimeout;
6471
6472     my $vollist = [];
6473     foreach my $devname (keys %$devinfo) {
6474         my $volid = $devinfo->{$devname}->{volid};
6475         push @$vollist, $volid if $volid;
6476     }
6477
6478     PVE::Storage::deactivate_volumes($cfg, $vollist);
6479
6480     unlink $mapfifo;
6481
6482     if ($err) {
6483         rmtree $tmpdir;
6484         unlink $tmpfn;
6485
6486         foreach my $devname (keys %$devinfo) {
6487             my $volid = $devinfo->{$devname}->{volid};
6488             next if !$volid;
6489             eval {
6490                 if ($volid =~ m|^/|) {
6491                     unlink $volid || die 'unlink failed\n';
6492                 } else {
6493                     PVE::Storage::vdisk_free($cfg, $volid);
6494                 }
6495                 print STDERR "temporary volume '$volid' sucessfuly removed\n";
6496             };
6497             print STDERR "unable to cleanup '$volid' - $@" if $@;
6498         }
6499         die $err;
6500     }
6501
6502     rmtree $tmpdir;
6503
6504     rename($tmpfn, $conffile) ||
6505         die "unable to commit configuration file '$conffile'\n";
6506
6507     PVE::Cluster::cfs_update(); # make sure we read new file
6508
6509     eval { rescan($vmid, 1); };
6510     warn $@ if $@;
6511 }
6512
6513 sub restore_tar_archive {
6514     my ($archive, $vmid, $user, $opts) = @_;
6515
6516     if ($archive ne '-') {
6517         my $firstfile = tar_archive_read_firstfile($archive);
6518         die "ERROR: file '$archive' dos not lock like a QemuServer vzdump backup\n"
6519             if $firstfile ne 'qemu-server.conf';
6520     }
6521
6522     my $storecfg = PVE::Storage::config();
6523
6524     # avoid zombie disks when restoring over an existing VM -> cleanup first
6525     # pass keep_empty_config=1 to keep the config (thus VMID) reserved for us
6526     # skiplock=1 because qmrestore has set the 'create' lock itself already
6527     my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
6528     destroy_vm($storecfg, $vmid, 1, { lock => 'restore' }) if -f $vmcfgfn;
6529
6530     my $tocmd = "/usr/lib/qemu-server/qmextract";
6531
6532     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
6533     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
6534     $tocmd .= ' --prealloc' if $opts->{prealloc};
6535     $tocmd .= ' --info' if $opts->{info};
6536
6537     # tar option "xf" does not autodetect compression when read from STDIN,
6538     # so we pipe to zcat
6539     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
6540         PVE::Tools::shellquote("--to-command=$tocmd");
6541
6542     my $tmpdir = "/var/tmp/vzdumptmp$$";
6543     mkpath $tmpdir;
6544
6545     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
6546     local $ENV{VZDUMP_VMID} = $vmid;
6547     local $ENV{VZDUMP_USER} = $user;
6548
6549     my $conffile = PVE::QemuConfig->config_file($vmid);
6550     my $tmpfn = "$conffile.$$.tmp";
6551
6552     # disable interrupts (always do cleanups)
6553     local $SIG{INT} =
6554         local $SIG{TERM} =
6555         local $SIG{QUIT} =
6556         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
6557
6558     eval {
6559         # enable interrupts
6560         local $SIG{INT} =
6561             local $SIG{TERM} =
6562             local $SIG{QUIT} =
6563             local $SIG{HUP} =
6564             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6565
6566         if ($archive eq '-') {
6567             print "extracting archive from STDIN\n";
6568             run_command($cmd, input => "<&STDIN");
6569         } else {
6570             print "extracting archive '$archive'\n";
6571             run_command($cmd);
6572         }
6573
6574         return if $opts->{info};
6575
6576         # read new mapping
6577         my $map = {};
6578         my $statfile = "$tmpdir/qmrestore.stat";
6579         if (my $fd = IO::File->new($statfile, "r")) {
6580             while (defined (my $line = <$fd>)) {
6581                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
6582                     $map->{$1} = $2 if $1;
6583                 } else {
6584                     print STDERR "unable to parse line in statfile - $line\n";
6585                 }
6586             }
6587             $fd->close();
6588         }
6589
6590         my $confsrc = "$tmpdir/qemu-server.conf";
6591
6592         my $srcfd = new IO::File($confsrc, "r") ||
6593             die "unable to open file '$confsrc'\n";
6594
6595         my $outfd = new IO::File ($tmpfn, "w") ||
6596             die "unable to write config for VM $vmid\n";
6597
6598         my $cookie = { netcount => 0 };
6599         while (defined (my $line = <$srcfd>)) {
6600             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
6601         }
6602
6603         $srcfd->close();
6604         $outfd->close();
6605     };
6606     if (my $err = $@) {
6607         unlink $tmpfn;
6608         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
6609         die $err;
6610     }
6611
6612     rmtree $tmpdir;
6613
6614     rename $tmpfn, $conffile ||
6615         die "unable to commit configuration file '$conffile'\n";
6616
6617     PVE::Cluster::cfs_update(); # make sure we read new file
6618
6619     eval { rescan($vmid, 1); };
6620     warn $@ if $@;
6621 };
6622
6623 sub foreach_storage_used_by_vm {
6624     my ($conf, $func) = @_;
6625
6626     my $sidhash = {};
6627
6628     foreach_drive($conf, sub {
6629         my ($ds, $drive) = @_;
6630         return if drive_is_cdrom($drive);
6631
6632         my $volid = $drive->{file};
6633
6634         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
6635         $sidhash->{$sid} = $sid if $sid;
6636     });
6637
6638     foreach my $sid (sort keys %$sidhash) {
6639         &$func($sid);
6640     }
6641 }
6642
6643 my $qemu_snap_storage = {
6644     rbd => 1,
6645 };
6646 sub do_snapshots_with_qemu {
6647     my ($storecfg, $volid) = @_;
6648
6649     my $storage_name = PVE::Storage::parse_volume_id($volid);
6650     my $scfg = $storecfg->{ids}->{$storage_name};
6651
6652     if ($qemu_snap_storage->{$scfg->{type}} && !$scfg->{krbd}){
6653         return 1;
6654     }
6655
6656     if ($volid =~ m/\.(qcow2|qed)$/){
6657         return 1;
6658     }
6659
6660     return undef;
6661 }
6662
6663 sub qga_check_running {
6664     my ($vmid, $nowarn) = @_;
6665
6666     eval { mon_cmd($vmid, "guest-ping", timeout => 3); };
6667     if ($@) {
6668         warn "Qemu Guest Agent is not running - $@" if !$nowarn;
6669         return 0;
6670     }
6671     return 1;
6672 }
6673
6674 sub template_create {
6675     my ($vmid, $conf, $disk) = @_;
6676
6677     my $storecfg = PVE::Storage::config();
6678
6679     foreach_drive($conf, sub {
6680         my ($ds, $drive) = @_;
6681
6682         return if drive_is_cdrom($drive);
6683         return if $disk && $ds ne $disk;
6684
6685         my $volid = $drive->{file};
6686         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
6687
6688         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
6689         $drive->{file} = $voliddst;
6690         $conf->{$ds} = print_drive($drive);
6691         PVE::QemuConfig->write_config($vmid, $conf);
6692     });
6693 }
6694
6695 sub convert_iscsi_path {
6696     my ($path) = @_;
6697
6698     if ($path =~ m|^iscsi://([^/]+)/([^/]+)/(.+)$|) {
6699         my $portal = $1;
6700         my $target = $2;
6701         my $lun = $3;
6702
6703         my $initiator_name = get_initiator_name();
6704
6705         return "file.driver=iscsi,file.transport=tcp,file.initiator-name=$initiator_name,".
6706                "file.portal=$portal,file.target=$target,file.lun=$lun,driver=raw";
6707     }
6708
6709     die "cannot convert iscsi path '$path', unkown format\n";
6710 }
6711
6712 sub qemu_img_convert {
6713     my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized) = @_;
6714
6715     my $storecfg = PVE::Storage::config();
6716     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
6717     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
6718
6719     die "destination '$dst_volid' is not a valid volid form qemu-img convert\n" if !$dst_storeid;
6720
6721     my $cachemode;
6722     my $src_path;
6723     my $src_is_iscsi = 0;
6724     my $src_format;
6725
6726     if ($src_storeid) {
6727         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
6728         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
6729         $src_format = qemu_img_format($src_scfg, $src_volname);
6730         $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
6731         $src_is_iscsi = ($src_path =~ m|^iscsi://|);
6732         $cachemode = 'none' if $src_scfg->{type} eq 'zfspool';
6733     } elsif (-f $src_volid) {
6734         $src_path = $src_volid;
6735         if ($src_path =~ m/\.($QEMU_FORMAT_RE)$/) {
6736             $src_format = $1;
6737         }
6738     }
6739
6740     die "source '$src_volid' is not a valid volid nor path for qemu-img convert\n" if !$src_path;
6741
6742     my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6743     my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
6744     my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6745     my $dst_is_iscsi = ($dst_path =~ m|^iscsi://|);
6746
6747     my $cmd = [];
6748     push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
6749     push @$cmd, '-l', "snapshot.name=$snapname"
6750         if $snapname && $src_format && $src_format eq "qcow2";
6751     push @$cmd, '-t', 'none' if $dst_scfg->{type} eq 'zfspool';
6752     push @$cmd, '-T', $cachemode if defined($cachemode);
6753
6754     if ($src_is_iscsi) {
6755         push @$cmd, '--image-opts';
6756         $src_path = convert_iscsi_path($src_path);
6757     } elsif ($src_format) {
6758         push @$cmd, '-f', $src_format;
6759     }
6760
6761     if ($dst_is_iscsi) {
6762         push @$cmd, '--target-image-opts';
6763         $dst_path = convert_iscsi_path($dst_path);
6764     } else {
6765         push @$cmd, '-O', $dst_format;
6766     }
6767
6768     push @$cmd, $src_path;
6769
6770     if (!$dst_is_iscsi && $is_zero_initialized) {
6771         push @$cmd, "zeroinit:$dst_path";
6772     } else {
6773         push @$cmd, $dst_path;
6774     }
6775
6776     my $parser = sub {
6777         my $line = shift;
6778         if($line =~ m/\((\S+)\/100\%\)/){
6779             my $percent = $1;
6780             my $transferred = int($size * $percent / 100);
6781             my $remaining = $size - $transferred;
6782
6783             print "transferred: $transferred bytes remaining: $remaining bytes total: $size bytes progression: $percent %\n";
6784         }
6785
6786     };
6787
6788     eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
6789     my $err = $@;
6790     die "copy failed: $err" if $err;
6791 }
6792
6793 sub qemu_img_format {
6794     my ($scfg, $volname) = @_;
6795
6796     if ($scfg->{path} && $volname =~ m/\.($QEMU_FORMAT_RE)$/) {
6797         return $1;
6798     } else {
6799         return "raw";
6800     }
6801 }
6802
6803 sub qemu_drive_mirror {
6804     my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $skipcomplete, $qga, $bwlimit) = @_;
6805
6806     $jobs = {} if !$jobs;
6807
6808     my $qemu_target;
6809     my $format;
6810     $jobs->{"drive-$drive"} = {};
6811
6812     if ($dst_volid =~ /^nbd:/) {
6813         $qemu_target = $dst_volid;
6814         $format = "nbd";
6815     } else {
6816         my $storecfg = PVE::Storage::config();
6817         my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
6818
6819         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6820
6821         $format = qemu_img_format($dst_scfg, $dst_volname);
6822
6823         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6824
6825         $qemu_target = $is_zero_initialized ? "zeroinit:$dst_path" : $dst_path;
6826     }
6827
6828     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $qemu_target };
6829     $opts->{format} = $format if $format;
6830
6831     if (defined($bwlimit)) {
6832         $opts->{speed} = $bwlimit * 1024;
6833         print "drive mirror is starting for drive-$drive with bandwidth limit: ${bwlimit} KB/s\n";
6834     } else {
6835         print "drive mirror is starting for drive-$drive\n";
6836     }
6837
6838     # if a job already runs for this device we get an error, catch it for cleanup
6839     eval { mon_cmd($vmid, "drive-mirror", %$opts); };
6840     if (my $err = $@) {
6841         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6842         warn "$@\n" if $@;
6843         die "mirroring error: $err\n";
6844     }
6845
6846     qemu_drive_mirror_monitor ($vmid, $vmiddst, $jobs, $skipcomplete, $qga);
6847 }
6848
6849 sub qemu_drive_mirror_monitor {
6850     my ($vmid, $vmiddst, $jobs, $skipcomplete, $qga) = @_;
6851
6852     eval {
6853         my $err_complete = 0;
6854
6855         while (1) {
6856             die "storage migration timed out\n" if $err_complete > 300;
6857
6858             my $stats = mon_cmd($vmid, "query-block-jobs");
6859
6860             my $running_mirror_jobs = {};
6861             foreach my $stat (@$stats) {
6862                 next if $stat->{type} ne 'mirror';
6863                 $running_mirror_jobs->{$stat->{device}} = $stat;
6864             }
6865
6866             my $readycounter = 0;
6867
6868             foreach my $job (keys %$jobs) {
6869
6870                 if(defined($jobs->{$job}->{complete}) && !defined($running_mirror_jobs->{$job})) {
6871                     print "$job : finished\n";
6872                     delete $jobs->{$job};
6873                     next;
6874                 }
6875
6876                 die "$job: mirroring has been cancelled\n" if !defined($running_mirror_jobs->{$job});
6877
6878                 my $busy = $running_mirror_jobs->{$job}->{busy};
6879                 my $ready = $running_mirror_jobs->{$job}->{ready};
6880                 if (my $total = $running_mirror_jobs->{$job}->{len}) {
6881                     my $transferred = $running_mirror_jobs->{$job}->{offset} || 0;
6882                     my $remaining = $total - $transferred;
6883                     my $percent = sprintf "%.2f", ($transferred * 100 / $total);
6884
6885                     print "$job: transferred: $transferred bytes remaining: $remaining bytes total: $total bytes progression: $percent % busy: $busy ready: $ready \n";
6886                 }
6887
6888                 $readycounter++ if $running_mirror_jobs->{$job}->{ready};
6889             }
6890
6891             last if scalar(keys %$jobs) == 0;
6892
6893             if ($readycounter == scalar(keys %$jobs)) {
6894                 print "all mirroring jobs are ready \n";
6895                 last if $skipcomplete; #do the complete later
6896
6897                 if ($vmiddst && $vmiddst != $vmid) {
6898                     my $agent_running = $qga && qga_check_running($vmid);
6899                     if ($agent_running) {
6900                         print "freeze filesystem\n";
6901                         eval { mon_cmd($vmid, "guest-fsfreeze-freeze"); };
6902                     } else {
6903                         print "suspend vm\n";
6904                         eval { PVE::QemuServer::vm_suspend($vmid, 1); };
6905                     }
6906
6907                     # if we clone a disk for a new target vm, we don't switch the disk
6908                     PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs);
6909
6910                     if ($agent_running) {
6911                         print "unfreeze filesystem\n";
6912                         eval { mon_cmd($vmid, "guest-fsfreeze-thaw"); };
6913                     } else {
6914                         print "resume vm\n";
6915                         eval {  PVE::QemuServer::vm_resume($vmid, 1, 1); };
6916                     }
6917
6918                     last;
6919                 } else {
6920
6921                     foreach my $job (keys %$jobs) {
6922                         # try to switch the disk if source and destination are on the same guest
6923                         print "$job: Completing block job...\n";
6924
6925                         eval { mon_cmd($vmid, "block-job-complete", device => $job) };
6926                         if ($@ =~ m/cannot be completed/) {
6927                             print "$job: Block job cannot be completed, try again.\n";
6928                             $err_complete++;
6929                         }else {
6930                             print "$job: Completed successfully.\n";
6931                             $jobs->{$job}->{complete} = 1;
6932                         }
6933                     }
6934                 }
6935             }
6936             sleep 1;
6937         }
6938     };
6939     my $err = $@;
6940
6941     if ($err) {
6942         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6943         die "mirroring error: $err";
6944     }
6945
6946 }
6947
6948 sub qemu_blockjobs_cancel {
6949     my ($vmid, $jobs) = @_;
6950
6951     foreach my $job (keys %$jobs) {
6952         print "$job: Cancelling block job\n";
6953         eval { mon_cmd($vmid, "block-job-cancel", device => $job); };
6954         $jobs->{$job}->{cancel} = 1;
6955     }
6956
6957     while (1) {
6958         my $stats = mon_cmd($vmid, "query-block-jobs");
6959
6960         my $running_jobs = {};
6961         foreach my $stat (@$stats) {
6962             $running_jobs->{$stat->{device}} = $stat;
6963         }
6964
6965         foreach my $job (keys %$jobs) {
6966
6967             if (defined($jobs->{$job}->{cancel}) && !defined($running_jobs->{$job})) {
6968                 print "$job: Done.\n";
6969                 delete $jobs->{$job};
6970             }
6971         }
6972
6973         last if scalar(keys %$jobs) == 0;
6974
6975         sleep 1;
6976     }
6977 }
6978
6979 sub clone_disk {
6980     my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
6981         $newvmid, $storage, $format, $full, $newvollist, $jobs, $skipcomplete, $qga, $bwlimit) = @_;
6982
6983     my $newvolid;
6984
6985     if (!$full) {
6986         print "create linked clone of drive $drivename ($drive->{file})\n";
6987         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
6988         push @$newvollist, $newvolid;
6989     } else {
6990
6991         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6992         $storeid = $storage if $storage;
6993
6994         my $dst_format = resolve_dst_disk_format($storecfg, $storeid, $volname, $format);
6995         my ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 3);
6996
6997         print "create full clone of drive $drivename ($drive->{file})\n";
6998         my $name = undef;
6999         if (drive_is_cloudinit($drive)) {
7000             $name = "vm-$newvmid-cloudinit";
7001             $name .= ".$dst_format" if $dst_format ne 'raw';
7002             $snapname = undef;
7003             $size = PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE;
7004         }
7005         $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $newvmid, $dst_format, $name, ($size/1024));
7006         push @$newvollist, $newvolid;
7007
7008         PVE::Storage::activate_volumes($storecfg, [$newvolid]);
7009
7010         if (drive_is_cloudinit($drive)) {
7011             goto no_data_clone;
7012         }
7013
7014         my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
7015         if (!$running || $snapname) {
7016             # TODO: handle bwlimits
7017             qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit);
7018         } else {
7019
7020             my $kvmver = get_running_qemu_version ($vmid);
7021             if (!min_version($kvmver, 2, 7)) {
7022                 die "drive-mirror with iothread requires qemu version 2.7 or higher\n"
7023                     if $drive->{iothread};
7024             }
7025
7026             qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid, $sparseinit, $jobs, $skipcomplete, $qga, $bwlimit);
7027         }
7028     }
7029
7030 no_data_clone:
7031     my ($size) = PVE::Storage::volume_size_info($storecfg, $newvolid, 3);
7032
7033     my $disk = $drive;
7034     $disk->{format} = undef;
7035     $disk->{file} = $newvolid;
7036     $disk->{size} = $size;
7037
7038     return $disk;
7039 }
7040
7041 sub get_running_qemu_version {
7042     my ($vmid) = @_;
7043     my $res = mon_cmd($vmid, "query-version");
7044     return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
7045 }
7046
7047 sub qemu_use_old_bios_files {
7048     my ($machine_type) = @_;
7049
7050     return if !$machine_type;
7051
7052     my $use_old_bios_files = undef;
7053
7054     if ($machine_type =~ m/^(\S+)\.pxe$/) {
7055         $machine_type = $1;
7056         $use_old_bios_files = 1;
7057     } else {
7058         my $version = PVE::QemuServer::Machine::extract_version($machine_type, kvm_user_version());
7059         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
7060         # load new efi bios files on migration. So this hack is required to allow
7061         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
7062         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
7063         $use_old_bios_files = !min_version($version, 2, 4);
7064     }
7065
7066     return ($use_old_bios_files, $machine_type);
7067 }
7068
7069 sub create_efidisk($$$$$) {
7070     my ($storecfg, $storeid, $vmid, $fmt, $arch) = @_;
7071
7072     my (undef, $ovmf_vars) = get_ovmf_files($arch);
7073     die "EFI vars default image not found\n" if ! -f $ovmf_vars;
7074
7075     my $vars_size_b = -s $ovmf_vars;
7076     my $vars_size = PVE::Tools::convert_size($vars_size_b, 'b' => 'kb');
7077     my $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $fmt, undef, $vars_size);
7078     PVE::Storage::activate_volumes($storecfg, [$volid]);
7079
7080     qemu_img_convert($ovmf_vars, $volid, $vars_size_b, undef, 0);
7081     my ($size) = PVE::Storage::volume_size_info($storecfg, $volid, 3);
7082
7083     return ($volid, $size/1024);
7084 }
7085
7086 sub vm_iothreads_list {
7087     my ($vmid) = @_;
7088
7089     my $res = mon_cmd($vmid, 'query-iothreads');
7090
7091     my $iothreads = {};
7092     foreach my $iothread (@$res) {
7093         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
7094     }
7095
7096     return $iothreads;
7097 }
7098
7099 sub scsihw_infos {
7100     my ($conf, $drive) = @_;
7101
7102     my $maxdev = 0;
7103
7104     if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)) {
7105         $maxdev = 7;
7106     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
7107         $maxdev = 1;
7108     } else {
7109         $maxdev = 256;
7110     }
7111
7112     my $controller = int($drive->{index} / $maxdev);
7113     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single') ? "virtioscsi" : "scsihw";
7114
7115     return ($maxdev, $controller, $controller_prefix);
7116 }
7117
7118 sub add_hyperv_enlightenments {
7119     my ($cpuFlags, $winversion, $machine_version, $bios, $gpu_passthrough, $hv_vendor_id) = @_;
7120
7121     return if $winversion < 6;
7122     return if $bios && $bios eq 'ovmf' && $winversion < 8;
7123
7124     if ($gpu_passthrough || defined($hv_vendor_id)) {
7125         $hv_vendor_id //= 'proxmox';
7126         push @$cpuFlags , "hv_vendor_id=$hv_vendor_id";
7127     }
7128
7129     if (min_version($machine_version, 2, 3)) {
7130         push @$cpuFlags , 'hv_spinlocks=0x1fff';
7131         push @$cpuFlags , 'hv_vapic';
7132         push @$cpuFlags , 'hv_time';
7133     } else {
7134         push @$cpuFlags , 'hv_spinlocks=0xffff';
7135     }
7136
7137     if (min_version($machine_version, 2, 6)) {
7138         push @$cpuFlags , 'hv_reset';
7139         push @$cpuFlags , 'hv_vpindex';
7140         push @$cpuFlags , 'hv_runtime';
7141     }
7142
7143     if ($winversion >= 7) {
7144         push @$cpuFlags , 'hv_relaxed';
7145
7146         if (min_version($machine_version, 2, 12)) {
7147             push @$cpuFlags , 'hv_synic';
7148             push @$cpuFlags , 'hv_stimer';
7149         }
7150
7151         if (min_version($machine_version, 3, 1)) {
7152             push @$cpuFlags , 'hv_ipi';
7153         }
7154     }
7155 }
7156
7157 sub windows_version {
7158     my ($ostype) = @_;
7159
7160     return 0 if !$ostype;
7161
7162     my $winversion = 0;
7163
7164     if($ostype eq 'wxp' || $ostype eq 'w2k3' || $ostype eq 'w2k') {
7165         $winversion = 5;
7166     } elsif($ostype eq 'w2k8' || $ostype eq 'wvista') {
7167         $winversion = 6;
7168     } elsif ($ostype =~ m/^win(\d+)$/) {
7169         $winversion = $1;
7170     }
7171
7172     return $winversion;
7173 }
7174
7175 sub resolve_dst_disk_format {
7176         my ($storecfg, $storeid, $src_volname, $format) = @_;
7177         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
7178
7179         if (!$format) {
7180             # if no target format is specified, use the source disk format as hint
7181             if ($src_volname) {
7182                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
7183                 $format = qemu_img_format($scfg, $src_volname);
7184             } else {
7185                 return $defFormat;
7186             }
7187         }
7188
7189         # test if requested format is supported - else use default
7190         my $supported = grep { $_ eq $format } @$validFormats;
7191         $format = $defFormat if !$supported;
7192         return $format;
7193 }
7194
7195 sub resolve_first_disk {
7196     my $conf = shift;
7197     my @disks = PVE::QemuServer::valid_drive_names();
7198     my $firstdisk;
7199     foreach my $ds (reverse @disks) {
7200         next if !$conf->{$ds};
7201         my $disk = PVE::QemuServer::parse_drive($ds, $conf->{$ds});
7202         next if PVE::QemuServer::drive_is_cdrom($disk);
7203         $firstdisk = $ds;
7204     }
7205     return $firstdisk;
7206 }
7207
7208 sub generate_uuid {
7209     my ($uuid, $uuid_str);
7210     UUID::generate($uuid);
7211     UUID::unparse($uuid, $uuid_str);
7212     return $uuid_str;
7213 }
7214
7215 sub generate_smbios1_uuid {
7216     return "uuid=".generate_uuid();
7217 }
7218
7219 sub nbd_stop {
7220     my ($vmid) = @_;
7221
7222     mon_cmd($vmid, 'nbd-server-stop');
7223 }
7224
7225 sub create_reboot_request {
7226     my ($vmid) = @_;
7227     open(my $fh, '>', "/run/qemu-server/$vmid.reboot")
7228         or die "failed to create reboot trigger file: $!\n";
7229     close($fh);
7230 }
7231
7232 sub clear_reboot_request {
7233     my ($vmid) = @_;
7234     my $path = "/run/qemu-server/$vmid.reboot";
7235     my $res = 0;
7236
7237     $res = unlink($path);
7238     die "could not remove reboot request for $vmid: $!"
7239         if !$res && $! != POSIX::ENOENT;
7240
7241     return $res;
7242 }
7243
7244 # bash completion helper
7245
7246 sub complete_backup_archives {
7247     my ($cmdname, $pname, $cvalue) = @_;
7248
7249     my $cfg = PVE::Storage::config();
7250
7251     my $storeid;
7252
7253     if ($cvalue =~ m/^([^:]+):/) {
7254         $storeid = $1;
7255     }
7256
7257     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
7258
7259     my $res = [];
7260     foreach my $id (keys %$data) {
7261         foreach my $item (@{$data->{$id}}) {
7262             next if $item->{format} !~ m/^vma\.(gz|lzo)$/;
7263             push @$res, $item->{volid} if defined($item->{volid});
7264         }
7265     }
7266
7267     return $res;
7268 }
7269
7270 my $complete_vmid_full = sub {
7271     my ($running) = @_;
7272
7273     my $idlist = vmstatus();
7274
7275     my $res = [];
7276
7277     foreach my $id (keys %$idlist) {
7278         my $d = $idlist->{$id};
7279         if (defined($running)) {
7280             next if $d->{template};
7281             next if $running && $d->{status} ne 'running';
7282             next if !$running && $d->{status} eq 'running';
7283         }
7284         push @$res, $id;
7285
7286     }
7287     return $res;
7288 };
7289
7290 sub complete_vmid {
7291     return &$complete_vmid_full();
7292 }
7293
7294 sub complete_vmid_stopped {
7295     return &$complete_vmid_full(0);
7296 }
7297
7298 sub complete_vmid_running {
7299     return &$complete_vmid_full(1);
7300 }
7301
7302 sub complete_storage {
7303
7304     my $cfg = PVE::Storage::config();
7305     my $ids = $cfg->{ids};
7306
7307     my $res = [];
7308     foreach my $sid (keys %$ids) {
7309         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
7310         next if !$ids->{$sid}->{content}->{images};
7311         push @$res, $sid;
7312     }
7313
7314     return $res;
7315 }
7316
7317 sub complete_migration_storage {
7318     my ($cmd, $param, $current_value, $all_args) = @_;
7319
7320     my $targetnode = @$all_args[1];
7321
7322     my $cfg = PVE::Storage::config();
7323     my $ids = $cfg->{ids};
7324
7325     my $res = [];
7326     foreach my $sid (keys %$ids) {
7327         next if !PVE::Storage::storage_check_enabled($cfg, $sid, $targetnode, 1);
7328         next if !$ids->{$sid}->{content}->{images};
7329         push @$res, $sid;
7330     }
7331
7332     return $res;
7333 }
7334
7335 1;