PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5
   6 use Cwd 'abs_path';
   7 use Digest::SHA;
   8 use Fcntl ':flock';
   9 use Fcntl;
  10 use File::Basename;
  11 use File::Copy qw(copy);
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use IO::Dir;
  16 use IO::File;
  17 use IO::Handle;
  18 use IO::Select;
  19 use IO::Socket::UNIX;
  20 use IPC::Open3;
  21 use JSON;
  22 use MIME::Base64;
  23 use POSIX;
  24 use Storable qw(dclone);
  25 use Time::HiRes qw(gettimeofday);
  26 use URI::Escape;
  27 use UUID;
  28
  29 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file cfs_lock_file);
  30 use PVE::DataCenterConfig;
  31 use PVE::Exception qw(raise raise_param_exc);
  32 use PVE::GuestHelpers;
  33 use PVE::INotify;
  34 use PVE::JSONSchema qw(get_standard_option);
  35 use PVE::ProcFSTools;
  36 use PVE::RPCEnvironment;
  37 use PVE::Storage;
  38 use PVE::SysFSTools;
  39 use PVE::Systemd;
  40 use PVE::Tools qw(run_command lock_file lock_file_full file_read_firstline dir_glob_foreach get_host_arch $IPV6RE);
  41
  42 use PVE::QMPClient;
  43 use PVE::QemuConfig;
  44 use PVE::QemuServer::Helpers qw(min_version);
  45 use PVE::QemuServer::Cloudinit;
  46 use PVE::QemuServer::Machine;
  47 use PVE::QemuServer::Memory;
  48 use PVE::QemuServer::Monitor qw(mon_cmd);
  49 use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr print_pcie_root_port);
  50 use PVE::QemuServer::USB qw(parse_usb_device);
  51
  52 my $EDK2_FW_BASE = '/usr/share/pve-edk2-firmware/';
  53 my $OVMF = {
  54     x86_64 => [
  55         "$EDK2_FW_BASE/OVMF_CODE.fd",
  56         "$EDK2_FW_BASE/OVMF_VARS.fd"
  57     ],
  58     aarch64 => [
  59         "$EDK2_FW_BASE/AAVMF_CODE.fd",
  60         "$EDK2_FW_BASE/AAVMF_VARS.fd"
  61     ],
  62 };
  63
  64 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  65
  66 my $QEMU_FORMAT_RE = qr/raw|cow|qcow|qcow2|qed|vmdk|cloop/;
  67
  68 # Note about locking: we use flock on the config file protect
  69 # against concurent actions.
  70 # Aditionaly, we have a 'lock' setting in the config file. This
  71 # can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
  72 # allowed when such lock is set. But you can ignore this kind of
  73 # lock with the --skiplock flag.
  74
  75 cfs_register_file('/qemu-server/',
  76                   \&parse_vm_config,
  77                   \&write_vm_config);
  78
  79 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
  80     description => "Some command save/restore state from this location.",
  81     type => 'string',
  82     maxLength => 128,
  83     optional => 1,
  84 });
  85
  86 PVE::JSONSchema::register_standard_option('pve-qm-image-format', {
  87     type => 'string',
  88     enum => [qw(raw cow qcow qed qcow2 vmdk cloop)],
  89     description => "The drive's backing file's data format.",
  90     optional => 1,
  91 });
  92
  93 PVE::JSONSchema::register_standard_option('pve-qemu-machine', {
  94         description => "Specifies the Qemu machine type.",
  95         type => 'string',
  96         pattern => '(pc|pc(-i440fx)?-\d+(\.\d+)+(\.pxe)?|q35|pc-q35-\d+(\.\d+)+(\.pxe)?|virt(?:-\d+(\.\d+)+)?)',
  97         maxLength => 40,
  98         optional => 1,
  99 });
 100
 101 #no warnings 'redefine';
 102
 103 sub cgroups_write {
 104    my ($controller, $vmid, $option, $value) = @_;
 105
 106    my $path = "/sys/fs/cgroup/$controller/qemu.slice/$vmid.scope/$option";
 107    PVE::ProcFSTools::write_proc_entry($path, $value);
 108
 109 }
 110
 111 my $nodename = PVE::INotify::nodename();
 112
 113 my $cpu_vendor_list = {
 114     # Intel CPUs
 115     486 => 'GenuineIntel',
 116     pentium => 'GenuineIntel',
 117     pentium2  => 'GenuineIntel',
 118     pentium3  => 'GenuineIntel',
 119     coreduo => 'GenuineIntel',
 120     core2duo => 'GenuineIntel',
 121     Conroe  => 'GenuineIntel',
 122     Penryn  => 'GenuineIntel',
 123     Nehalem  => 'GenuineIntel',
 124     'Nehalem-IBRS'  => 'GenuineIntel',
 125     Westmere => 'GenuineIntel',
 126     'Westmere-IBRS' => 'GenuineIntel',
 127     SandyBridge => 'GenuineIntel',
 128     'SandyBridge-IBRS' => 'GenuineIntel',
 129     IvyBridge => 'GenuineIntel',
 130     'IvyBridge-IBRS' => 'GenuineIntel',
 131     Haswell => 'GenuineIntel',
 132     'Haswell-IBRS' => 'GenuineIntel',
 133     'Haswell-noTSX' => 'GenuineIntel',
 134     'Haswell-noTSX-IBRS' => 'GenuineIntel',
 135     Broadwell => 'GenuineIntel',
 136     'Broadwell-IBRS' => 'GenuineIntel',
 137     'Broadwell-noTSX' => 'GenuineIntel',
 138     'Broadwell-noTSX-IBRS' => 'GenuineIntel',
 139     'Skylake-Client' => 'GenuineIntel',
 140     'Skylake-Client-IBRS' => 'GenuineIntel',
 141     'Skylake-Server' => 'GenuineIntel',
 142     'Skylake-Server-IBRS' => 'GenuineIntel',
 143     'Cascadelake-Server' => 'GenuineIntel',
 144     KnightsMill => 'GenuineIntel',
 145
 146
 147     # AMD CPUs
 148     athlon => 'AuthenticAMD',
 149     phenom  => 'AuthenticAMD',
 150     Opteron_G1  => 'AuthenticAMD',
 151     Opteron_G2  => 'AuthenticAMD',
 152     Opteron_G3  => 'AuthenticAMD',
 153     Opteron_G4  => 'AuthenticAMD',
 154     Opteron_G5  => 'AuthenticAMD',
 155     EPYC => 'AuthenticAMD',
 156     'EPYC-IBPB' => 'AuthenticAMD',
 157
 158     # generic types, use vendor from host node
 159     host => 'default',
 160     kvm32 => 'default',
 161     kvm64 => 'default',
 162     qemu32 => 'default',
 163     qemu64 => 'default',
 164     max => 'default',
 165 };
 166
 167 my @supported_cpu_flags = (
 168     'pcid',
 169     'spec-ctrl',
 170     'ibpb',
 171     'ssbd',
 172     'virt-ssbd',
 173     'amd-ssbd',
 174     'amd-no-ssb',
 175     'pdpe1gb',
 176     'md-clear',
 177     'hv-tlbflush',
 178     'hv-evmcs',
 179     'aes'
 180 );
 181 my $cpu_flag = qr/[+-](@{[join('|', @supported_cpu_flags)]})/;
 182
 183 my $cpu_fmt = {
 184     cputype => {
 185         description => "Emulated CPU type.",
 186         type => 'string',
 187         enum => [ sort { "\L$a" cmp "\L$b" } keys %$cpu_vendor_list ],
 188         default => 'kvm64',
 189         default_key => 1,
 190     },
 191     hidden => {
 192         description => "Do not identify as a KVM virtual machine.",
 193         type => 'boolean',
 194         optional => 1,
 195         default => 0
 196     },
 197     'hv-vendor-id' => {
 198         type => 'string',
 199         pattern => qr/[a-zA-Z0-9]{1,12}/,
 200         format_description => 'vendor-id',
 201         description => 'The Hyper-V vendor ID. Some drivers or programs inside Windows guests need a specific ID.',
 202         optional => 1,
 203     },
 204     flags => {
 205         description => "List of additional CPU flags separated by ';'."
 206                      . " Use '+FLAG' to enable, '-FLAG' to disable a flag."
 207                      . " Currently supported flags: @{[join(', ', @supported_cpu_flags)]}.",
 208         format_description => '+FLAG[;-FLAG...]',
 209         type => 'string',
 210         pattern => qr/$cpu_flag(;$cpu_flag)*/,
 211         optional => 1,
 212     },
 213 };
 214
 215 my $watchdog_fmt = {
 216     model => {
 217         default_key => 1,
 218         type => 'string',
 219         enum => [qw(i6300esb ib700)],
 220         description => "Watchdog type to emulate.",
 221         default => 'i6300esb',
 222         optional => 1,
 223     },
 224     action => {
 225         type => 'string',
 226         enum => [qw(reset shutdown poweroff pause debug none)],
 227         description => "The action to perform if after activation the guest fails to poll the watchdog in time.",
 228         optional => 1,
 229     },
 230 };
 231 PVE::JSONSchema::register_format('pve-qm-watchdog', $watchdog_fmt);
 232
 233 my $agent_fmt = {
 234     enabled => {
 235         description => "Enable/disable Qemu GuestAgent.",
 236         type => 'boolean',
 237         default => 0,
 238         default_key => 1,
 239     },
 240     fstrim_cloned_disks => {
 241         description => "Run fstrim after cloning/moving a disk.",
 242         type => 'boolean',
 243         optional => 1,
 244         default => 0
 245     },
 246     type => {
 247         description => "Select the agent type",
 248         type => 'string',
 249         default => 'virtio',
 250         optional => 1,
 251         enum => [qw(virtio isa)],
 252     },
 253 };
 254
 255 my $vga_fmt = {
 256     type => {
 257         description => "Select the VGA type.",
 258         type => 'string',
 259         default => 'std',
 260         optional => 1,
 261         default_key => 1,
 262         enum => [qw(cirrus qxl qxl2 qxl3 qxl4 none serial0 serial1 serial2 serial3 std virtio vmware)],
 263     },
 264     memory => {
 265         description => "Sets the VGA memory (in MiB). Has no effect with serial display.",
 266         type => 'integer',
 267         optional => 1,
 268         minimum => 4,
 269         maximum => 512,
 270     },
 271 };
 272
 273 my $ivshmem_fmt = {
 274     size => {
 275         type => 'integer',
 276         minimum => 1,
 277         description => "The size of the file in MB.",
 278     },
 279     name => {
 280         type => 'string',
 281         pattern => '[a-zA-Z0-9\-]+',
 282         optional => 1,
 283         format_description => 'string',
 284         description => "The name of the file. Will be prefixed with 'pve-shm-'. Default is the VMID. Will be deleted when the VM is stopped.",
 285     },
 286 };
 287
 288 my $audio_fmt = {
 289     device => {
 290         type => 'string',
 291         enum => [qw(ich9-intel-hda intel-hda AC97)],
 292         description =>  "Configure an audio device."
 293     },
 294     driver =>  {
 295         type => 'string',
 296         enum => ['spice'],
 297         default => 'spice',
 298         optional => 1,
 299         description => "Driver backend for the audio device."
 300     },
 301 };
 302
 303 my $spice_enhancements_fmt = {
 304     foldersharing => {
 305         type => 'boolean',
 306         optional => 1,
 307         default => '0',
 308         description =>  "Enable folder sharing via SPICE. Needs Spice-WebDAV daemon installed in the VM."
 309     },
 310     videostreaming =>  {
 311         type => 'string',
 312         enum => ['off', 'all', 'filter'],
 313         default => 'off',
 314         optional => 1,
 315         description => "Enable video streaming. Uses compression for detected video streams."
 316     },
 317 };
 318
 319 my $confdesc = {
 320     onboot => {
 321         optional => 1,
 322         type => 'boolean',
 323         description => "Specifies whether a VM will be started during system bootup.",
 324         default => 0,
 325     },
 326     autostart => {
 327         optional => 1,
 328         type => 'boolean',
 329         description => "Automatic restart after crash (currently ignored).",
 330         default => 0,
 331     },
 332     hotplug => {
 333         optional => 1,
 334         type => 'string', format => 'pve-hotplug-features',
 335         description => "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
 336         default => 'network,disk,usb',
 337     },
 338     reboot => {
 339         optional => 1,
 340         type => 'boolean',
 341         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 342         default => 1,
 343     },
 344     lock => {
 345         optional => 1,
 346         type => 'string',
 347         description => "Lock/unlock the VM.",
 348         enum => [qw(backup clone create migrate rollback snapshot snapshot-delete suspending suspended)],
 349     },
 350     cpulimit => {
 351         optional => 1,
 352         type => 'number',
 353         description => "Limit of CPU usage.",
 354         verbose_description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
 355         minimum => 0,
 356         maximum => 128,
 357         default => 0,
 358     },
 359     cpuunits => {
 360         optional => 1,
 361         type => 'integer',
 362         description => "CPU weight for a VM.",
 363         verbose_description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.",
 364         minimum => 2,
 365         maximum => 262144,
 366         default => 1024,
 367     },
 368     memory => {
 369         optional => 1,
 370         type => 'integer',
 371         description => "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
 372         minimum => 16,
 373         default => 512,
 374     },
 375     balloon => {
 376         optional => 1,
 377         type => 'integer',
 378         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 379         minimum => 0,
 380     },
 381     shares => {
 382         optional => 1,
 383         type => 'integer',
 384         description => "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning. Auto-ballooning is done by pvestatd.",
 385         minimum => 0,
 386         maximum => 50000,
 387         default => 1000,
 388     },
 389     keyboard => {
 390         optional => 1,
 391         type => 'string',
 392         description => "Keybord layout for vnc server. Default is read from the '/etc/pve/datacenter.cfg' configuration file.".
 393                        "It should not be necessary to set it.",
 394         enum => PVE::Tools::kvmkeymaplist(),
 395         default => undef,
 396     },
 397     name => {
 398         optional => 1,
 399         type => 'string', format => 'dns-name',
 400         description => "Set a name for the VM. Only used on the configuration web interface.",
 401     },
 402     scsihw => {
 403         optional => 1,
 404         type => 'string',
 405         description => "SCSI controller model",
 406         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 407         default => 'lsi',
 408     },
 409     description => {
 410         optional => 1,
 411         type => 'string',
 412         description => "Description for the VM. Only used on the configuration web interface. This is saved as comment inside the configuration file.",
 413     },
 414     ostype => {
 415         optional => 1,
 416         type => 'string',
 417         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 l24 l26 solaris)],
 418         description => "Specify guest operating system.",
 419         verbose_description => <<EODESC,
 420 Specify guest operating system. This is used to enable special
 421 optimization/features for specific operating systems:
 422
 423 [horizontal]
 424 other;; unspecified OS
 425 wxp;; Microsoft Windows XP
 426 w2k;; Microsoft Windows 2000
 427 w2k3;; Microsoft Windows 2003
 428 w2k8;; Microsoft Windows 2008
 429 wvista;; Microsoft Windows Vista
 430 win7;; Microsoft Windows 7
 431 win8;; Microsoft Windows 8/2012/2012r2
 432 win10;; Microsoft Windows 10/2016
 433 l24;; Linux 2.4 Kernel
 434 l26;; Linux 2.6 - 5.X Kernel
 435 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
 436 EODESC
 437     },
 438     boot => {
 439         optional => 1,
 440         type => 'string',
 441         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n).",
 442         pattern => '[acdn]{1,4}',
 443         default => 'cdn',
 444     },
 445     bootdisk => {
 446         optional => 1,
 447         type => 'string', format => 'pve-qm-bootdisk',
 448         description => "Enable booting from specified disk.",
 449         pattern => '(ide|sata|scsi|virtio)\d+',
 450     },
 451     smp => {
 452         optional => 1,
 453         type => 'integer',
 454         description => "The number of CPUs. Please use option -sockets instead.",
 455         minimum => 1,
 456         default => 1,
 457     },
 458     sockets => {
 459         optional => 1,
 460         type => 'integer',
 461         description => "The number of CPU sockets.",
 462         minimum => 1,
 463         default => 1,
 464     },
 465     cores => {
 466         optional => 1,
 467         type => 'integer',
 468         description => "The number of cores per socket.",
 469         minimum => 1,
 470         default => 1,
 471     },
 472     numa => {
 473         optional => 1,
 474         type => 'boolean',
 475         description => "Enable/disable NUMA.",
 476         default => 0,
 477     },
 478     hugepages => {
 479         optional => 1,
 480         type => 'string',
 481         description => "Enable/disable hugepages memory.",
 482         enum => [qw(any 2 1024)],
 483     },
 484     vcpus => {
 485         optional => 1,
 486         type => 'integer',
 487         description => "Number of hotplugged vcpus.",
 488         minimum => 1,
 489         default => 0,
 490     },
 491     acpi => {
 492         optional => 1,
 493         type => 'boolean',
 494         description => "Enable/disable ACPI.",
 495         default => 1,
 496     },
 497     agent => {
 498         optional => 1,
 499         description => "Enable/disable Qemu GuestAgent and its properties.",
 500         type => 'string',
 501         format => $agent_fmt,
 502     },
 503     kvm => {
 504         optional => 1,
 505         type => 'boolean',
 506         description => "Enable/disable KVM hardware virtualization.",
 507         default => 1,
 508     },
 509     tdf => {
 510         optional => 1,
 511         type => 'boolean',
 512         description => "Enable/disable time drift fix.",
 513         default => 0,
 514     },
 515     localtime => {
 516         optional => 1,
 517         type => 'boolean',
 518         description => "Set the real time clock to local time. This is enabled by default if ostype indicates a Microsoft OS.",
 519     },
 520     freeze => {
 521         optional => 1,
 522         type => 'boolean',
 523         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 524     },
 525     vga => {
 526         optional => 1,
 527         type => 'string', format => $vga_fmt,
 528         description => "Configure the VGA hardware.",
 529         verbose_description => "Configure the VGA Hardware. If you want to use ".
 530             "high resolution modes (>= 1280x1024x16) you may need to increase " .
 531             "the vga memory option. Since QEMU 2.9 the default VGA display type " .
 532             "is 'std' for all OS types besides some Windows versions (XP and " .
 533             "older) which use 'cirrus'. The 'qxl' option enables the SPICE " .
 534             "display server. For win* OS you can select how many independent " .
 535             "displays you want, Linux guests can add displays them self.\n".
 536             "You can also run without any graphic card, using a serial device as terminal.",
 537     },
 538     watchdog => {
 539         optional => 1,
 540         type => 'string', format => 'pve-qm-watchdog',
 541         description => "Create a virtual hardware watchdog device.",
 542         verbose_description => "Create a virtual hardware watchdog device. Once enabled" .
 543             " (by a guest action), the watchdog must be periodically polled " .
 544             "by an agent inside the guest or else the watchdog will reset " .
 545             "the guest (or execute the respective action specified)",
 546     },
 547     startdate => {
 548         optional => 1,
 549         type => 'string',
 550         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 551         description => "Set the initial date of the real time clock. Valid format for date are: 'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 552         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 553         default => 'now',
 554     },
 555     startup =>  get_standard_option('pve-startup-order'),
 556     template => {
 557         optional => 1,
 558         type => 'boolean',
 559         description => "Enable/disable Template.",
 560         default => 0,
 561     },
 562     args => {
 563         optional => 1,
 564         type => 'string',
 565         description => "Arbitrary arguments passed to kvm.",
 566         verbose_description => <<EODESCR,
 567 Arbitrary arguments passed to kvm, for example:
 568
 569 args: -no-reboot -no-hpet
 570
 571 NOTE: this option is for experts only.
 572 EODESCR
 573     },
 574     tablet => {
 575         optional => 1,
 576         type => 'boolean',
 577         default => 1,
 578         description => "Enable/disable the USB tablet device.",
 579         verbose_description => "Enable/disable the USB tablet device. This device is " .
 580             "usually needed to allow absolute mouse positioning with VNC. " .
 581             "Else the mouse runs out of sync with normal VNC clients. " .
 582             "If you're running lots of console-only guests on one host, " .
 583             "you may consider disabling this to save some context switches. " .
 584             "This is turned off by default if you use spice (-vga=qxl).",
 585     },
 586     migrate_speed => {
 587         optional => 1,
 588         type => 'integer',
 589         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 590         minimum => 0,
 591         default => 0,
 592     },
 593     migrate_downtime => {
 594         optional => 1,
 595         type => 'number',
 596         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 597         minimum => 0,
 598         default => 0.1,
 599     },
 600     cdrom => {
 601         optional => 1,
 602         type => 'string', format => 'pve-qm-ide',
 603         typetext => '<volume>',
 604         description => "This is an alias for option -ide2",
 605     },
 606     cpu => {
 607         optional => 1,
 608         description => "Emulated CPU type.",
 609         type => 'string',
 610         format => $cpu_fmt,
 611     },
 612     parent => get_standard_option('pve-snapshot-name', {
 613         optional => 1,
 614         description => "Parent snapshot name. This is used internally, and should not be modified.",
 615     }),
 616     snaptime => {
 617         optional => 1,
 618         description => "Timestamp for snapshots.",
 619         type => 'integer',
 620         minimum => 0,
 621     },
 622     vmstate => {
 623         optional => 1,
 624         type => 'string', format => 'pve-volume-id',
 625         description => "Reference to a volume which stores the VM state. This is used internally for snapshots.",
 626     },
 627     vmstatestorage => get_standard_option('pve-storage-id', {
 628         description => "Default storage for VM state volumes/files.",
 629         optional => 1,
 630     }),
 631     runningmachine => get_standard_option('pve-qemu-machine', {
 632         description => "Specifies the Qemu machine type of the running vm. This is used internally for snapshots.",
 633     }),
 634     machine => get_standard_option('pve-qemu-machine'),
 635     arch => {
 636         description => "Virtual processor architecture. Defaults to the host.",
 637         optional => 1,
 638         type => 'string',
 639         enum => [qw(x86_64 aarch64)],
 640     },
 641     smbios1 => {
 642         description => "Specify SMBIOS type 1 fields.",
 643         type => 'string', format => 'pve-qm-smbios1',
 644         maxLength => 512,
 645         optional => 1,
 646     },
 647     protection => {
 648         optional => 1,
 649         type => 'boolean',
 650         description => "Sets the protection flag of the VM. This will disable the remove VM and remove disk operations.",
 651         default => 0,
 652     },
 653     bios => {
 654         optional => 1,
 655         type => 'string',
 656         enum => [ qw(seabios ovmf) ],
 657         description => "Select BIOS implementation.",
 658         default => 'seabios',
 659     },
 660     vmgenid => {
 661         type => 'string',
 662         pattern => '(?:[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}|[01])',
 663         format_description => 'UUID',
 664         description => "Set VM Generation ID. Use '1' to autogenerate on create or update, pass '0' to disable explicitly.",
 665         verbose_description => "The VM generation ID (vmgenid) device exposes a".
 666             " 128-bit integer value identifier to the guest OS. This allows to".
 667             " notify the guest operating system when the virtual machine is".
 668             " executed with a different configuration (e.g. snapshot execution".
 669             " or creation from a template). The guest operating system notices".
 670             " the change, and is then able to react as appropriate by marking".
 671             " its copies of distributed databases as dirty, re-initializing its".
 672             " random number generator, etc.\n".
 673             "Note that auto-creation only works when done throug API/CLI create".
 674             " or update methods, but not when manually editing the config file.",
 675         default => "1 (autogenerated)",
 676         optional => 1,
 677     },
 678     hookscript => {
 679         type => 'string',
 680         format => 'pve-volume-id',
 681         optional => 1,
 682         description => "Script that will be executed during various steps in the vms lifetime.",
 683     },
 684     ivshmem => {
 685         type => 'string',
 686         format => $ivshmem_fmt,
 687         description => "Inter-VM shared memory. Useful for direct communication between VMs, or to the host.",
 688         optional => 1,
 689     },
 690     audio0 => {
 691         type => 'string',
 692         format => $audio_fmt,
 693         description => "Configure a audio device, useful in combination with QXL/Spice.",
 694         optional => 1
 695     },
 696     spice_enhancements => {
 697         type => 'string',
 698         format => $spice_enhancements_fmt,
 699         description => "Configure additional enhancements for SPICE.",
 700         optional => 1
 701     },
 702     tags => {
 703         type => 'string', format => 'pve-tag-list',
 704         description => 'Tags of the VM. This is only meta information.',
 705         optional => 1,
 706     },
 707 };
 708
 709 my $cicustom_fmt = {
 710     meta => {
 711         type => 'string',
 712         optional => 1,
 713         description => 'Specify a custom file containing all meta data passed to the VM via cloud-init. This is provider specific meaning configdrive2 and nocloud differ.',
 714         format => 'pve-volume-id',
 715         format_description => 'volume',
 716     },
 717     network => {
 718         type => 'string',
 719         optional => 1,
 720         description => 'Specify a custom file containing all network data passed to the VM via cloud-init.',
 721         format => 'pve-volume-id',
 722         format_description => 'volume',
 723     },
 724     user => {
 725         type => 'string',
 726         optional => 1,
 727         description => 'Specify a custom file containing all user data passed to the VM via cloud-init.',
 728         format => 'pve-volume-id',
 729         format_description => 'volume',
 730     },
 731 };
 732 PVE::JSONSchema::register_format('pve-qm-cicustom', $cicustom_fmt);
 733
 734 my $confdesc_cloudinit = {
 735     citype => {
 736         optional => 1,
 737         type => 'string',
 738         description => 'Specifies the cloud-init configuration format. The default depends on the configured operating system type (`ostype`. We use the `nocloud` format for Linux, and `configdrive2` for windows.',
 739         enum => ['configdrive2', 'nocloud'],
 740     },
 741     ciuser => {
 742         optional => 1,
 743         type => 'string',
 744         description => "cloud-init: User name to change ssh keys and password for instead of the image's configured default user.",
 745     },
 746     cipassword => {
 747         optional => 1,
 748         type => 'string',
 749         description => 'cloud-init: Password to assign the user. Using this is generally not recommended. Use ssh keys instead. Also note that older cloud-init versions do not support hashed passwords.',
 750     },
 751     cicustom => {
 752         optional => 1,
 753         type => 'string',
 754         description => 'cloud-init: Specify custom files to replace the automatically generated ones at start.',
 755         format => 'pve-qm-cicustom',
 756     },
 757     searchdomain => {
 758         optional => 1,
 759         type => 'string',
 760         description => "cloud-init: Sets DNS search domains for a container. Create will automatically use the setting from the host if neither searchdomain nor nameserver are set.",
 761     },
 762     nameserver => {
 763         optional => 1,
 764         type => 'string', format => 'address-list',
 765         description => "cloud-init: Sets DNS server IP address for a container. Create will automatically use the setting from the host if neither searchdomain nor nameserver are set.",
 766     },
 767     sshkeys => {
 768         optional => 1,
 769         type => 'string',
 770         format => 'urlencoded',
 771         description => "cloud-init: Setup public SSH keys (one key per line, OpenSSH format).",
 772     },
 773 };
 774
 775 # what about other qemu settings ?
 776 #cpu => 'string',
 777 #machine => 'string',
 778 #fda => 'file',
 779 #fdb => 'file',
 780 #mtdblock => 'file',
 781 #sd => 'file',
 782 #pflash => 'file',
 783 #snapshot => 'bool',
 784 #bootp => 'file',
 785 ##tftp => 'dir',
 786 ##smb => 'dir',
 787 #kernel => 'file',
 788 #append => 'string',
 789 #initrd => 'file',
 790 ##soundhw => 'string',
 791
 792 while (my ($k, $v) = each %$confdesc) {
 793     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 794 }
 795
 796 my $MAX_IDE_DISKS = 4;
 797 my $MAX_SCSI_DISKS = 14;
 798 my $MAX_VIRTIO_DISKS = 16;
 799 my $MAX_SATA_DISKS = 6;
 800 my $MAX_USB_DEVICES = 5;
 801 my $MAX_NETS = 32;
 802 my $MAX_UNUSED_DISKS = 256;
 803 my $MAX_HOSTPCI_DEVICES = 16;
 804 my $MAX_SERIAL_PORTS = 4;
 805 my $MAX_PARALLEL_PORTS = 3;
 806 my $MAX_NUMA = 8;
 807
 808 my $numa_fmt = {
 809     cpus => {
 810         type => "string",
 811         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 812         description => "CPUs accessing this NUMA node.",
 813         format_description => "id[-id];...",
 814     },
 815     memory => {
 816         type => "number",
 817         description => "Amount of memory this NUMA node provides.",
 818         optional => 1,
 819     },
 820     hostnodes => {
 821         type => "string",
 822         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 823         description => "Host NUMA nodes to use.",
 824         format_description => "id[-id];...",
 825         optional => 1,
 826     },
 827     policy => {
 828         type => 'string',
 829         enum => [qw(preferred bind interleave)],
 830         description => "NUMA allocation policy.",
 831         optional => 1,
 832     },
 833 };
 834 PVE::JSONSchema::register_format('pve-qm-numanode', $numa_fmt);
 835 my $numadesc = {
 836     optional => 1,
 837     type => 'string', format => $numa_fmt,
 838     description => "NUMA topology.",
 839 };
 840 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 841
 842 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 843     $confdesc->{"numa$i"} = $numadesc;
 844 }
 845
 846 my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000',  'pcnet',  'virtio',
 847                       'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3',
 848                       'e1000-82540em', 'e1000-82544gc', 'e1000-82545em'];
 849 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 850
 851 my $net_fmt_bridge_descr = <<__EOD__;
 852 Bridge to attach the network device to. The Proxmox VE standard bridge
 853 is called 'vmbr0'.
 854
 855 If you do not specify a bridge, we create a kvm user (NATed) network
 856 device, which provides DHCP and DNS services. The following addresses
 857 are used:
 858
 859  10.0.2.2   Gateway
 860  10.0.2.3   DNS Server
 861  10.0.2.4   SMB Server
 862
 863 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 864 __EOD__
 865
 866 my $net_fmt = {
 867     macaddr  => get_standard_option('mac-addr', {
 868         description => "MAC address. That address must be unique withing your network. This is automatically generated if not specified.",
 869     }),
 870     model => {
 871         type => 'string',
 872         description => "Network Card Model. The 'virtio' model provides the best performance with very low CPU overhead. If your guest does not support this driver, it is usually best to use 'e1000'.",
 873         enum => $nic_model_list,
 874         default_key => 1,
 875     },
 876     (map { $_ => { keyAlias => 'model', alias => 'macaddr' }} @$nic_model_list),
 877     bridge => {
 878         type => 'string',
 879         description => $net_fmt_bridge_descr,
 880         format_description => 'bridge',
 881         optional => 1,
 882     },
 883     queues => {
 884         type => 'integer',
 885         minimum => 0, maximum => 16,
 886         description => 'Number of packet queues to be used on the device.',
 887         optional => 1,
 888     },
 889     rate => {
 890         type => 'number',
 891         minimum => 0,
 892         description => "Rate limit in mbps (megabytes per second) as floating point number.",
 893         optional => 1,
 894     },
 895     tag => {
 896         type => 'integer',
 897         minimum => 1, maximum => 4094,
 898         description => 'VLAN tag to apply to packets on this interface.',
 899         optional => 1,
 900     },
 901     trunks => {
 902         type => 'string',
 903         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 904         description => 'VLAN trunks to pass through this interface.',
 905         format_description => 'vlanid[;vlanid...]',
 906         optional => 1,
 907     },
 908     firewall => {
 909         type => 'boolean',
 910         description => 'Whether this interface should be protected by the firewall.',
 911         optional => 1,
 912     },
 913     link_down => {
 914         type => 'boolean',
 915         description => 'Whether this interface should be disconnected (like pulling the plug).',
 916         optional => 1,
 917     },
 918 };
 919
 920 my $netdesc = {
 921     optional => 1,
 922     type => 'string', format => $net_fmt,
 923     description => "Specify network devices.",
 924 };
 925
 926 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 927
 928 my $ipconfig_fmt = {
 929     ip => {
 930         type => 'string',
 931         format => 'pve-ipv4-config',
 932         format_description => 'IPv4Format/CIDR',
 933         description => 'IPv4 address in CIDR format.',
 934         optional => 1,
 935         default => 'dhcp',
 936     },
 937     gw => {
 938         type => 'string',
 939         format => 'ipv4',
 940         format_description => 'GatewayIPv4',
 941         description => 'Default gateway for IPv4 traffic.',
 942         optional => 1,
 943         requires => 'ip',
 944     },
 945     ip6 => {
 946         type => 'string',
 947         format => 'pve-ipv6-config',
 948         format_description => 'IPv6Format/CIDR',
 949         description => 'IPv6 address in CIDR format.',
 950         optional => 1,
 951         default => 'dhcp',
 952     },
 953     gw6 => {
 954         type => 'string',
 955         format => 'ipv6',
 956         format_description => 'GatewayIPv6',
 957         description => 'Default gateway for IPv6 traffic.',
 958         optional => 1,
 959         requires => 'ip6',
 960     },
 961 };
 962 PVE::JSONSchema::register_format('pve-qm-ipconfig', $ipconfig_fmt);
 963 my $ipconfigdesc = {
 964     optional => 1,
 965     type => 'string', format => 'pve-qm-ipconfig',
 966     description => <<'EODESCR',
 967 cloud-init: Specify IP addresses and gateways for the corresponding interface.
 968
 969 IP addresses use CIDR notation, gateways are optional but need an IP of the same type specified.
 970
 971 The special string 'dhcp' can be used for IP addresses to use DHCP, in which case no explicit gateway should be provided.
 972 For IPv6 the special string 'auto' can be used to use stateless autoconfiguration.
 973
 974 If cloud-init is enabled and neither an IPv4 nor an IPv6 address is specified, it defaults to using dhcp on IPv4.
 975 EODESCR
 976 };
 977 PVE::JSONSchema::register_standard_option("pve-qm-ipconfig", $netdesc);
 978
 979 for (my $i = 0; $i < $MAX_NETS; $i++)  {
 980     $confdesc->{"net$i"} = $netdesc;
 981     $confdesc_cloudinit->{"ipconfig$i"} = $ipconfigdesc;
 982 }
 983
 984 foreach my $key (keys %$confdesc_cloudinit) {
 985     $confdesc->{$key} = $confdesc_cloudinit->{$key};
 986 }
 987
 988 PVE::JSONSchema::register_format('pve-volume-id-or-qm-path', \&verify_volume_id_or_qm_path);
 989 sub verify_volume_id_or_qm_path {
 990     my ($volid, $noerr) = @_;
 991
 992     if ($volid eq 'none' || $volid eq 'cdrom' || $volid =~ m|^/|) {
 993         return $volid;
 994     }
 995
 996     # if its neither 'none' nor 'cdrom' nor a path, check if its a volume-id
 997     $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') };
 998     if ($@) {
 999         return undef if $noerr;
1000         die $@;
1001     }
1002     return $volid;
1003 }
1004
1005 my $drivename_hash;
1006
1007 my %drivedesc_base = (
1008     volume => { alias => 'file' },
1009     file => {
1010         type => 'string',
1011         format => 'pve-volume-id-or-qm-path',
1012         default_key => 1,
1013         format_description => 'volume',
1014         description => "The drive's backing volume.",
1015     },
1016     media => {
1017         type => 'string',
1018         enum => [qw(cdrom disk)],
1019         description => "The drive's media type.",
1020         default => 'disk',
1021         optional => 1
1022     },
1023     cyls => {
1024         type => 'integer',
1025         description => "Force the drive's physical geometry to have a specific cylinder count.",
1026         optional => 1
1027     },
1028     heads => {
1029         type => 'integer',
1030         description => "Force the drive's physical geometry to have a specific head count.",
1031         optional => 1
1032     },
1033     secs => {
1034         type => 'integer',
1035         description => "Force the drive's physical geometry to have a specific sector count.",
1036         optional => 1
1037     },
1038     trans => {
1039         type => 'string',
1040         enum => [qw(none lba auto)],
1041         description => "Force disk geometry bios translation mode.",
1042         optional => 1,
1043     },
1044     snapshot => {
1045         type => 'boolean',
1046         description => "Controls qemu's snapshot mode feature."
1047             . " If activated, changes made to the disk are temporary and will"
1048             . " be discarded when the VM is shutdown.",
1049         optional => 1,
1050     },
1051     cache => {
1052         type => 'string',
1053         enum => [qw(none writethrough writeback unsafe directsync)],
1054         description => "The drive's cache mode",
1055         optional => 1,
1056     },
1057     format => get_standard_option('pve-qm-image-format'),
1058     size => {
1059         type => 'string',
1060         format => 'disk-size',
1061         format_description => 'DiskSize',
1062         description => "Disk size. This is purely informational and has no effect.",
1063         optional => 1,
1064     },
1065     backup => {
1066         type => 'boolean',
1067         description => "Whether the drive should be included when making backups.",
1068         optional => 1,
1069     },
1070     replicate => {
1071         type => 'boolean',
1072         description => 'Whether the drive should considered for replication jobs.',
1073         optional => 1,
1074         default => 1,
1075     },
1076     rerror => {
1077         type => 'string',
1078         enum => [qw(ignore report stop)],
1079         description => 'Read error action.',
1080         optional => 1,
1081     },
1082     werror => {
1083         type => 'string',
1084         enum => [qw(enospc ignore report stop)],
1085         description => 'Write error action.',
1086         optional => 1,
1087     },
1088     aio => {
1089         type => 'string',
1090         enum => [qw(native threads)],
1091         description => 'AIO type to use.',
1092         optional => 1,
1093     },
1094     discard => {
1095         type => 'string',
1096         enum => [qw(ignore on)],
1097         description => 'Controls whether to pass discard/trim requests to the underlying storage.',
1098         optional => 1,
1099     },
1100     detect_zeroes => {
1101         type => 'boolean',
1102         description => 'Controls whether to detect and try to optimize writes of zeroes.',
1103         optional => 1,
1104     },
1105     serial => {
1106         type => 'string',
1107         format => 'urlencoded',
1108         format_description => 'serial',
1109         maxLength => 20*3, # *3 since it's %xx url enoded
1110         description => "The drive's reported serial number, url-encoded, up to 20 bytes long.",
1111         optional => 1,
1112     },
1113     shared => {
1114         type => 'boolean',
1115         description => 'Mark this locally-managed volume as available on all nodes',
1116         verbose_description => "Mark this locally-managed volume as available on all nodes.\n\nWARNING: This option does not share the volume automatically, it assumes it is shared already!",
1117         optional => 1,
1118         default => 0,
1119     }
1120 );
1121
1122 my %iothread_fmt = ( iothread => {
1123         type => 'boolean',
1124         description => "Whether to use iothreads for this drive",
1125         optional => 1,
1126 });
1127
1128 my %model_fmt = (
1129     model => {
1130         type => 'string',
1131         format => 'urlencoded',
1132         format_description => 'model',
1133         maxLength => 40*3, # *3 since it's %xx url enoded
1134         description => "The drive's reported model name, url-encoded, up to 40 bytes long.",
1135         optional => 1,
1136     },
1137 );
1138
1139 my %queues_fmt = (
1140     queues => {
1141         type => 'integer',
1142         description => "Number of queues.",
1143         minimum => 2,
1144         optional => 1
1145     }
1146 );
1147
1148 my %scsiblock_fmt = (
1149     scsiblock => {
1150         type => 'boolean',
1151         description => "whether to use scsi-block for full passthrough of host block device\n\nWARNING: can lead to I/O errors in combination with low memory or high memory fragmentation on host",
1152         optional => 1,
1153         default => 0,
1154     },
1155 );
1156
1157 my %ssd_fmt = (
1158     ssd => {
1159         type => 'boolean',
1160         description => "Whether to expose this drive as an SSD, rather than a rotational hard disk.",
1161         optional => 1,
1162     },
1163 );
1164
1165 my %wwn_fmt = (
1166     wwn => {
1167         type => 'string',
1168         pattern => qr/^(0x)[0-9a-fA-F]{16}/,
1169         format_description => 'wwn',
1170         description => "The drive's worldwide name, encoded as 16 bytes hex string, prefixed by '0x'.",
1171         optional => 1,
1172     },
1173 );
1174
1175 my $add_throttle_desc = sub {
1176     my ($key, $type, $what, $unit, $longunit, $minimum) = @_;
1177     my $d = {
1178         type => $type,
1179         format_description => $unit,
1180         description => "Maximum $what in $longunit.",
1181         optional => 1,
1182     };
1183     $d->{minimum} = $minimum if defined($minimum);
1184     $drivedesc_base{$key} = $d;
1185 };
1186 # throughput: (leaky bucket)
1187 $add_throttle_desc->('bps',     'integer', 'r/w speed',   'bps',  'bytes per second');
1188 $add_throttle_desc->('bps_rd',  'integer', 'read speed',  'bps',  'bytes per second');
1189 $add_throttle_desc->('bps_wr',  'integer', 'write speed', 'bps',  'bytes per second');
1190 $add_throttle_desc->('mbps',    'number',  'r/w speed',   'mbps', 'megabytes per second');
1191 $add_throttle_desc->('mbps_rd', 'number',  'read speed',  'mbps', 'megabytes per second');
1192 $add_throttle_desc->('mbps_wr', 'number',  'write speed', 'mbps', 'megabytes per second');
1193 $add_throttle_desc->('iops',    'integer', 'r/w I/O',     'iops', 'operations per second');
1194 $add_throttle_desc->('iops_rd', 'integer', 'read I/O',    'iops', 'operations per second');
1195 $add_throttle_desc->('iops_wr', 'integer', 'write I/O',   'iops', 'operations per second');
1196
1197 # pools: (pool of IO before throttling starts taking effect)
1198 $add_throttle_desc->('mbps_max',    'number',  'unthrottled r/w pool',       'mbps', 'megabytes per second');
1199 $add_throttle_desc->('mbps_rd_max', 'number',  'unthrottled read pool',      'mbps', 'megabytes per second');
1200 $add_throttle_desc->('mbps_wr_max', 'number',  'unthrottled write pool',     'mbps', 'megabytes per second');
1201 $add_throttle_desc->('iops_max',    'integer', 'unthrottled r/w I/O pool',   'iops', 'operations per second');
1202 $add_throttle_desc->('iops_rd_max', 'integer', 'unthrottled read I/O pool',  'iops', 'operations per second');
1203 $add_throttle_desc->('iops_wr_max', 'integer', 'unthrottled write I/O pool', 'iops', 'operations per second');
1204
1205 # burst lengths
1206 $add_throttle_desc->('bps_max_length',     'integer', 'length of I/O bursts',       'seconds', 'seconds', 1);
1207 $add_throttle_desc->('bps_rd_max_length',  'integer', 'length of read I/O bursts',  'seconds', 'seconds', 1);
1208 $add_throttle_desc->('bps_wr_max_length',  'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
1209 $add_throttle_desc->('iops_max_length',    'integer', 'length of I/O bursts',       'seconds', 'seconds', 1);
1210 $add_throttle_desc->('iops_rd_max_length', 'integer', 'length of read I/O bursts',  'seconds', 'seconds', 1);
1211 $add_throttle_desc->('iops_wr_max_length', 'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
1212
1213 # legacy support
1214 $drivedesc_base{'bps_rd_length'} = { alias => 'bps_rd_max_length' };
1215 $drivedesc_base{'bps_wr_length'} = { alias => 'bps_wr_max_length' };
1216 $drivedesc_base{'iops_rd_length'} = { alias => 'iops_rd_max_length' };
1217 $drivedesc_base{'iops_wr_length'} = { alias => 'iops_wr_max_length' };
1218
1219 my $ide_fmt = {
1220     %drivedesc_base,
1221     %model_fmt,
1222     %ssd_fmt,
1223     %wwn_fmt,
1224 };
1225 PVE::JSONSchema::register_format("pve-qm-ide", $ide_fmt);
1226
1227 my $idedesc = {
1228     optional => 1,
1229     type => 'string', format => $ide_fmt,
1230     description => "Use volume as IDE hard disk or CD-ROM (n is 0 to " .($MAX_IDE_DISKS -1) . ").",
1231 };
1232 PVE::JSONSchema::register_standard_option("pve-qm-ide", $idedesc);
1233
1234 my $scsi_fmt = {
1235     %drivedesc_base,
1236     %iothread_fmt,
1237     %queues_fmt,
1238     %scsiblock_fmt,
1239     %ssd_fmt,
1240     %wwn_fmt,
1241 };
1242 my $scsidesc = {
1243     optional => 1,
1244     type => 'string', format => $scsi_fmt,
1245     description => "Use volume as SCSI hard disk or CD-ROM (n is 0 to " . ($MAX_SCSI_DISKS - 1) . ").",
1246 };
1247 PVE::JSONSchema::register_standard_option("pve-qm-scsi", $scsidesc);
1248
1249 my $sata_fmt = {
1250     %drivedesc_base,
1251     %ssd_fmt,
1252     %wwn_fmt,
1253 };
1254 my $satadesc = {
1255     optional => 1,
1256     type => 'string', format => $sata_fmt,
1257     description => "Use volume as SATA hard disk or CD-ROM (n is 0 to " . ($MAX_SATA_DISKS - 1). ").",
1258 };
1259 PVE::JSONSchema::register_standard_option("pve-qm-sata", $satadesc);
1260
1261 my $virtio_fmt = {
1262     %drivedesc_base,
1263     %iothread_fmt,
1264 };
1265 my $virtiodesc = {
1266     optional => 1,
1267     type => 'string', format => $virtio_fmt,
1268     description => "Use volume as VIRTIO hard disk (n is 0 to " . ($MAX_VIRTIO_DISKS - 1) . ").",
1269 };
1270 PVE::JSONSchema::register_standard_option("pve-qm-virtio", $virtiodesc);
1271
1272 my $alldrive_fmt = {
1273     %drivedesc_base,
1274     %iothread_fmt,
1275     %model_fmt,
1276     %queues_fmt,
1277     %scsiblock_fmt,
1278     %ssd_fmt,
1279     %wwn_fmt,
1280 };
1281
1282 my $efidisk_fmt = {
1283     volume => { alias => 'file' },
1284     file => {
1285         type => 'string',
1286         format => 'pve-volume-id-or-qm-path',
1287         default_key => 1,
1288         format_description => 'volume',
1289         description => "The drive's backing volume.",
1290     },
1291     format => get_standard_option('pve-qm-image-format'),
1292     size => {
1293         type => 'string',
1294         format => 'disk-size',
1295         format_description => 'DiskSize',
1296         description => "Disk size. This is purely informational and has no effect.",
1297         optional => 1,
1298     },
1299 };
1300
1301 my $efidisk_desc = {
1302     optional => 1,
1303     type => 'string', format => $efidisk_fmt,
1304     description => "Configure a Disk for storing EFI vars",
1305 };
1306
1307 PVE::JSONSchema::register_standard_option("pve-qm-efidisk", $efidisk_desc);
1308
1309 my $usb_fmt = {
1310     host => {
1311         default_key => 1,
1312         type => 'string', format => 'pve-qm-usb-device',
1313         format_description => 'HOSTUSBDEVICE|spice',
1314         description => <<EODESCR,
1315 The Host USB device or port or the value 'spice'. HOSTUSBDEVICE syntax is:
1316
1317  'bus-port(.port)*' (decimal numbers) or
1318  'vendor_id:product_id' (hexadeciaml numbers) or
1319  'spice'
1320
1321 You can use the 'lsusb -t' command to list existing usb devices.
1322
1323 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1324
1325 The value 'spice' can be used to add a usb redirection devices for spice.
1326 EODESCR
1327     },
1328     usb3 => {
1329         optional => 1,
1330         type => 'boolean',
1331         description => "Specifies whether if given host option is a USB3 device or port.",
1332         default => 0,
1333     },
1334 };
1335
1336 my $usbdesc = {
1337     optional => 1,
1338     type => 'string', format => $usb_fmt,
1339     description => "Configure an USB device (n is 0 to 4).",
1340 };
1341 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
1342
1343 my $PCIRE = qr/([a-f0-9]{4}:)?[a-f0-9]{2}:[a-f0-9]{2}(?:\.[a-f0-9])?/;
1344 my $hostpci_fmt = {
1345     host => {
1346         default_key => 1,
1347         type => 'string',
1348         pattern => qr/$PCIRE(;$PCIRE)*/,
1349         format_description => 'HOSTPCIID[;HOSTPCIID2...]',
1350         description => <<EODESCR,
1351 Host PCI device pass through. The PCI ID of a host's PCI device or a list
1352 of PCI virtual functions of the host. HOSTPCIID syntax is:
1353
1354 'bus:dev.func' (hexadecimal numbers)
1355
1356 You can us the 'lspci' command to list existing PCI devices.
1357 EODESCR
1358     },
1359     rombar => {
1360         type => 'boolean',
1361         description =>  "Specify whether or not the device's ROM will be visible in the guest's memory map.",
1362         optional => 1,
1363         default => 1,
1364     },
1365     romfile => {
1366         type => 'string',
1367         pattern => '[^,;]+',
1368         format_description => 'string',
1369         description => "Custom pci device rom filename (must be located in /usr/share/kvm/).",
1370         optional => 1,
1371     },
1372     pcie => {
1373         type => 'boolean',
1374         description =>  "Choose the PCI-express bus (needs the 'q35' machine model).",
1375         optional => 1,
1376         default => 0,
1377     },
1378     'x-vga' => {
1379         type => 'boolean',
1380         description =>  "Enable vfio-vga device support.",
1381         optional => 1,
1382         default => 0,
1383     },
1384     'mdev' => {
1385         type => 'string',
1386         format_description => 'string',
1387         pattern => '[^/\.:]+',
1388         optional => 1,
1389         description => <<EODESCR
1390 The type of mediated device to use.
1391 An instance of this type will be created on startup of the VM and
1392 will be cleaned up when the VM stops.
1393 EODESCR
1394     }
1395 };
1396 PVE::JSONSchema::register_format('pve-qm-hostpci', $hostpci_fmt);
1397
1398 my $hostpcidesc = {
1399         optional => 1,
1400         type => 'string', format => 'pve-qm-hostpci',
1401         description => "Map host PCI devices into guest.",
1402         verbose_description =>  <<EODESCR,
1403 Map host PCI devices into guest.
1404
1405 NOTE: This option allows direct access to host hardware. So it is no longer
1406 possible to migrate such machines - use with special care.
1407
1408 CAUTION: Experimental! User reported problems with this option.
1409 EODESCR
1410 };
1411 PVE::JSONSchema::register_standard_option("pve-qm-hostpci", $hostpcidesc);
1412
1413 my $serialdesc = {
1414         optional => 1,
1415         type => 'string',
1416         pattern => '(/dev/.+|socket)',
1417         description =>  "Create a serial device inside the VM (n is 0 to 3)",
1418         verbose_description =>  <<EODESCR,
1419 Create a serial device inside the VM (n is 0 to 3), and pass through a
1420 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1421 host side (use 'qm terminal' to open a terminal connection).
1422
1423 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines - use with special care.
1424
1425 CAUTION: Experimental! User reported problems with this option.
1426 EODESCR
1427 };
1428
1429 my $paralleldesc= {
1430         optional => 1,
1431         type => 'string',
1432         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
1433         description =>  "Map host parallel devices (n is 0 to 2).",
1434         verbose_description =>  <<EODESCR,
1435 Map host parallel devices (n is 0 to 2).
1436
1437 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1438
1439 CAUTION: Experimental! User reported problems with this option.
1440 EODESCR
1441 };
1442
1443 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
1444     $confdesc->{"parallel$i"} = $paralleldesc;
1445 }
1446
1447 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
1448     $confdesc->{"serial$i"} = $serialdesc;
1449 }
1450
1451 for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
1452     $confdesc->{"hostpci$i"} = $hostpcidesc;
1453 }
1454
1455 for (my $i = 0; $i < $MAX_IDE_DISKS; $i++)  {
1456     $drivename_hash->{"ide$i"} = 1;
1457     $confdesc->{"ide$i"} = $idedesc;
1458 }
1459
1460 for (my $i = 0; $i < $MAX_SATA_DISKS; $i++)  {
1461     $drivename_hash->{"sata$i"} = 1;
1462     $confdesc->{"sata$i"} = $satadesc;
1463 }
1464
1465 for (my $i = 0; $i < $MAX_SCSI_DISKS; $i++)  {
1466     $drivename_hash->{"scsi$i"} = 1;
1467     $confdesc->{"scsi$i"} = $scsidesc ;
1468 }
1469
1470 for (my $i = 0; $i < $MAX_VIRTIO_DISKS; $i++)  {
1471     $drivename_hash->{"virtio$i"} = 1;
1472     $confdesc->{"virtio$i"} = $virtiodesc;
1473 }
1474
1475 $drivename_hash->{efidisk0} = 1;
1476 $confdesc->{efidisk0} = $efidisk_desc;
1477
1478 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
1479     $confdesc->{"usb$i"} = $usbdesc;
1480 }
1481
1482 my $unuseddesc = {
1483     optional => 1,
1484     type => 'string', format => 'pve-volume-id',
1485     description => "Reference to unused volumes. This is used internally, and should not be modified manually.",
1486 };
1487
1488 for (my $i = 0; $i < $MAX_UNUSED_DISKS; $i++)  {
1489     $confdesc->{"unused$i"} = $unuseddesc;
1490 }
1491
1492 my $kvm_api_version = 0;
1493
1494 sub kvm_version {
1495     return $kvm_api_version if $kvm_api_version;
1496
1497     open my $fh, '<', '/dev/kvm'
1498         or return undef;
1499
1500     # 0xae00 => KVM_GET_API_VERSION
1501     $kvm_api_version = ioctl($fh, 0xae00, 0);
1502
1503     return $kvm_api_version;
1504 }
1505
1506 my $kvm_user_version = {};
1507 my $kvm_mtime = {};
1508
1509 sub kvm_user_version {
1510     my ($binary) = @_;
1511
1512     $binary //= get_command_for_arch(get_host_arch()); # get the native arch by default
1513     my $st = stat($binary);
1514
1515     my $cachedmtime = $kvm_mtime->{$binary} // -1;
1516     return $kvm_user_version->{$binary} if $kvm_user_version->{$binary} &&
1517         $cachedmtime == $st->mtime;
1518
1519     $kvm_user_version->{$binary} = 'unknown';
1520     $kvm_mtime->{$binary} = $st->mtime;
1521
1522     my $code = sub {
1523         my $line = shift;
1524         if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1525             $kvm_user_version->{$binary} = $2;
1526         }
1527     };
1528
1529     eval { run_command([$binary, '--version'], outfunc => $code); };
1530     warn $@ if $@;
1531
1532     return $kvm_user_version->{$binary};
1533
1534 }
1535
1536 sub kernel_has_vhost_net {
1537     return -c '/dev/vhost-net';
1538 }
1539
1540 sub valid_drive_names {
1541     # order is important - used to autoselect boot disk
1542     return ((map { "ide$_" } (0 .. ($MAX_IDE_DISKS - 1))),
1543             (map { "scsi$_" } (0 .. ($MAX_SCSI_DISKS - 1))),
1544             (map { "virtio$_" } (0 .. ($MAX_VIRTIO_DISKS - 1))),
1545             (map { "sata$_" } (0 .. ($MAX_SATA_DISKS - 1))),
1546             'efidisk0');
1547 }
1548
1549 sub is_valid_drivename {
1550     my $dev = shift;
1551
1552     return defined($drivename_hash->{$dev});
1553 }
1554
1555 sub option_exists {
1556     my $key = shift;
1557     return defined($confdesc->{$key});
1558 }
1559
1560 my $cdrom_path;
1561 sub get_cdrom_path {
1562
1563     return  $cdrom_path if $cdrom_path;
1564
1565     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
1566     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
1567     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
1568 }
1569
1570 sub get_iso_path {
1571     my ($storecfg, $vmid, $cdrom) = @_;
1572
1573     if ($cdrom eq 'cdrom') {
1574         return get_cdrom_path();
1575     } elsif ($cdrom eq 'none') {
1576         return '';
1577     } elsif ($cdrom =~ m|^/|) {
1578         return $cdrom;
1579     } else {
1580         return PVE::Storage::path($storecfg, $cdrom);
1581     }
1582 }
1583
1584 # try to convert old style file names to volume IDs
1585 sub filename_to_volume_id {
1586     my ($vmid, $file, $media) = @_;
1587
1588      if (!($file eq 'none' || $file eq 'cdrom' ||
1589           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1590
1591         return undef if $file =~ m|/|;
1592
1593         if ($media && $media eq 'cdrom') {
1594             $file = "local:iso/$file";
1595         } else {
1596             $file = "local:$vmid/$file";
1597         }
1598     }
1599
1600     return $file;
1601 }
1602
1603 sub verify_media_type {
1604     my ($opt, $vtype, $media) = @_;
1605
1606     return if !$media;
1607
1608     my $etype;
1609     if ($media eq 'disk') {
1610         $etype = 'images';
1611     } elsif ($media eq 'cdrom') {
1612         $etype = 'iso';
1613     } else {
1614         die "internal error";
1615     }
1616
1617     return if ($vtype eq $etype);
1618
1619     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1620 }
1621
1622 sub cleanup_drive_path {
1623     my ($opt, $storecfg, $drive) = @_;
1624
1625     # try to convert filesystem paths to volume IDs
1626
1627     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1628         ($drive->{file} !~ m|^/dev/.+|) &&
1629         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1630         ($drive->{file} !~ m/^\d+$/)) {
1631         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1632         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"}) if !$vtype;
1633         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1634         verify_media_type($opt, $vtype, $drive->{media});
1635         $drive->{file} = $volid;
1636     }
1637
1638     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1639 }
1640
1641 sub parse_hotplug_features {
1642     my ($data) = @_;
1643
1644     my $res = {};
1645
1646     return $res if $data eq '0';
1647
1648     $data = $confdesc->{hotplug}->{default} if $data eq '1';
1649
1650     foreach my $feature (PVE::Tools::split_list($data)) {
1651         if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
1652             $res->{$1} = 1;
1653         } else {
1654             die "invalid hotplug feature '$feature'\n";
1655         }
1656     }
1657     return $res;
1658 }
1659
1660 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1661 sub pve_verify_hotplug_features {
1662     my ($value, $noerr) = @_;
1663
1664     return $value if parse_hotplug_features($value);
1665
1666     return undef if $noerr;
1667
1668     die "unable to parse hotplug option\n";
1669 }
1670
1671 # ideX = [volume=]volume-id[,media=d][,cyls=c,heads=h,secs=s[,trans=t]]
1672 #        [,snapshot=on|off][,cache=on|off][,format=f][,backup=yes|no]
1673 #        [,rerror=ignore|report|stop][,werror=enospc|ignore|report|stop]
1674 #        [,aio=native|threads][,discard=ignore|on][,detect_zeroes=on|off]
1675 #        [,iothread=on][,serial=serial][,model=model]
1676
1677 sub parse_drive {
1678     my ($key, $data) = @_;
1679
1680     my ($interface, $index);
1681
1682     if ($key =~ m/^([^\d]+)(\d+)$/) {
1683         $interface = $1;
1684         $index = $2;
1685     } else {
1686         return undef;
1687     }
1688
1689     my $desc = $key =~ /^unused\d+$/ ? $alldrive_fmt
1690                                      : $confdesc->{$key}->{format};
1691     if (!$desc) {
1692         warn "invalid drive key: $key\n";
1693         return undef;
1694     }
1695     my $res = eval { PVE::JSONSchema::parse_property_string($desc, $data) };
1696     return undef if !$res;
1697     $res->{interface} = $interface;
1698     $res->{index} = $index;
1699
1700     my $error = 0;
1701     foreach my $opt (qw(bps bps_rd bps_wr)) {
1702         if (my $bps = defined(delete $res->{$opt})) {
1703             if (defined($res->{"m$opt"})) {
1704                 warn "both $opt and m$opt specified\n";
1705                 ++$error;
1706                 next;
1707             }
1708             $res->{"m$opt"} = sprintf("%.3f", $bps / (1024*1024.0));
1709         }
1710     }
1711
1712     # can't use the schema's 'requires' because of the mbps* => bps* "transforming aliases"
1713     for my $requirement (
1714         [mbps_max => 'mbps'],
1715         [mbps_rd_max => 'mbps_rd'],
1716         [mbps_wr_max => 'mbps_wr'],
1717         [miops_max => 'miops'],
1718         [miops_rd_max => 'miops_rd'],
1719         [miops_wr_max => 'miops_wr'],
1720         [bps_max_length => 'mbps_max'],
1721         [bps_rd_max_length => 'mbps_rd_max'],
1722         [bps_wr_max_length => 'mbps_wr_max'],
1723         [iops_max_length => 'iops_max'],
1724         [iops_rd_max_length => 'iops_rd_max'],
1725         [iops_wr_max_length => 'iops_wr_max']) {
1726         my ($option, $requires) = @$requirement;
1727         if ($res->{$option} && !$res->{$requires}) {
1728             warn "$option requires $requires\n";
1729             ++$error;
1730         }
1731     }
1732
1733     return undef if $error;
1734
1735     return undef if $res->{mbps_rd} && $res->{mbps};
1736     return undef if $res->{mbps_wr} && $res->{mbps};
1737     return undef if $res->{iops_rd} && $res->{iops};
1738     return undef if $res->{iops_wr} && $res->{iops};
1739
1740     if ($res->{media} && ($res->{media} eq 'cdrom')) {
1741         return undef if $res->{snapshot} || $res->{trans} || $res->{format};
1742         return undef if $res->{heads} || $res->{secs} || $res->{cyls};
1743         return undef if $res->{interface} eq 'virtio';
1744     }
1745
1746     if (my $size = $res->{size}) {
1747         return undef if !defined($res->{size} = PVE::JSONSchema::parse_size($size));
1748     }
1749
1750     return $res;
1751 }
1752
1753 sub print_drive {
1754     my ($vmid, $drive) = @_;
1755     my $data = { %$drive };
1756     delete $data->{$_} for qw(index interface);
1757     return PVE::JSONSchema::print_property_string($data, $alldrive_fmt);
1758 }
1759
1760 sub scsi_inquiry {
1761     my($fh, $noerr) = @_;
1762
1763     my $SG_IO = 0x2285;
1764     my $SG_GET_VERSION_NUM = 0x2282;
1765
1766     my $versionbuf = "\x00" x 8;
1767     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1768     if (!$ret) {
1769         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1770         return undef;
1771     }
1772     my $version = unpack("I", $versionbuf);
1773     if ($version < 30000) {
1774         die "scsi generic interface too old\n"  if !$noerr;
1775         return undef;
1776     }
1777
1778     my $buf = "\x00" x 36;
1779     my $sensebuf = "\x00" x 8;
1780     my $cmd = pack("C x3 C x1", 0x12, 36);
1781
1782     # see /usr/include/scsi/sg.h
1783     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1784
1785     my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1786                       length($sensebuf), 0, length($buf), $buf,
1787                       $cmd, $sensebuf, 6000);
1788
1789     $ret = ioctl($fh, $SG_IO, $packet);
1790     if (!$ret) {
1791         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1792         return undef;
1793     }
1794
1795     my @res = unpack($sg_io_hdr_t, $packet);
1796     if ($res[17] || $res[18]) {
1797         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1798         return undef;
1799     }
1800
1801     my $res = {};
1802     (my $byte0, my $byte1, $res->{vendor},
1803      $res->{product}, $res->{revision}) = unpack("C C x6 A8 A16 A4", $buf);
1804
1805     $res->{removable} = $byte1 & 128 ? 1 : 0;
1806     $res->{type} = $byte0 & 31;
1807
1808     return $res;
1809 }
1810
1811 sub path_is_scsi {
1812     my ($path) = @_;
1813
1814     my $fh = IO::File->new("+<$path") || return undef;
1815     my $res = scsi_inquiry($fh, 1);
1816     close($fh);
1817
1818     return $res;
1819 }
1820
1821 sub print_tabletdevice_full {
1822     my ($conf, $arch) = @_;
1823
1824     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
1825
1826     # we use uhci for old VMs because tablet driver was buggy in older qemu
1827     my $usbbus;
1828     if (PVE::QemuServer::Machine::machine_type_is_q35($conf) || $arch eq 'aarch64') {
1829         $usbbus = 'ehci';
1830     } else {
1831         $usbbus = 'uhci';
1832     }
1833
1834     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1835 }
1836
1837 sub print_keyboarddevice_full {
1838     my ($conf, $arch, $machine) = @_;
1839
1840     return undef if $arch ne 'aarch64';
1841
1842     return "usb-kbd,id=keyboard,bus=ehci.0,port=2";
1843 }
1844
1845 sub print_drivedevice_full {
1846     my ($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type) = @_;
1847
1848     my $device = '';
1849     my $maxdev = 0;
1850
1851     if ($drive->{interface} eq 'virtio') {
1852         my $pciaddr = print_pci_addr("$drive->{interface}$drive->{index}", $bridges, $arch, $machine_type);
1853         $device = "virtio-blk-pci,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}$pciaddr";
1854         $device .= ",iothread=iothread-$drive->{interface}$drive->{index}" if $drive->{iothread};
1855     } elsif ($drive->{interface} eq 'scsi') {
1856
1857         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1858         my $unit = $drive->{index} % $maxdev;
1859         my $devicetype = 'hd';
1860         my $path = '';
1861         if (drive_is_cdrom($drive)) {
1862             $devicetype = 'cd';
1863         } else {
1864             if ($drive->{file} =~ m|^/|) {
1865                 $path = $drive->{file};
1866                 if (my $info = path_is_scsi($path)) {
1867                     if ($info->{type} == 0 && $drive->{scsiblock}) {
1868                         $devicetype = 'block';
1869                     } elsif ($info->{type} == 1) { # tape
1870                         $devicetype = 'generic';
1871                     }
1872                 }
1873             } else {
1874                  $path = PVE::Storage::path($storecfg, $drive->{file});
1875             }
1876
1877             # for compatibility only, we prefer scsi-hd (#2408, #2355, #2380)
1878             my $version = PVE::QemuServer::Machine::extract_version($machine_type) // kvm_user_version();
1879             if ($path =~ m/^iscsi\:\/\// &&
1880                !min_version($version, 4, 1)) {
1881                 $devicetype = 'generic';
1882             }
1883         }
1884
1885         if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)){
1886             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1887         } else {
1888             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0,lun=$drive->{index},drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1889         }
1890
1891         if ($drive->{ssd} && ($devicetype eq 'block' || $devicetype eq 'hd')) {
1892             $device .= ",rotation_rate=1";
1893         }
1894         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1895
1896     } elsif ($drive->{interface} eq 'ide' || $drive->{interface} eq 'sata') {
1897         my $maxdev = ($drive->{interface} eq 'sata') ? $MAX_SATA_DISKS : 2;
1898         my $controller = int($drive->{index} / $maxdev);
1899         my $unit = $drive->{index} % $maxdev;
1900         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1901
1902         $device = "ide-$devicetype";
1903         if ($drive->{interface} eq 'ide') {
1904             $device .= ",bus=ide.$controller,unit=$unit";
1905         } else {
1906             $device .= ",bus=ahci$controller.$unit";
1907         }
1908         $device .= ",drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1909
1910         if ($devicetype eq 'hd') {
1911             if (my $model = $drive->{model}) {
1912                 $model = URI::Escape::uri_unescape($model);
1913                 $device .= ",model=$model";
1914             }
1915             if ($drive->{ssd}) {
1916                 $device .= ",rotation_rate=1";
1917             }
1918         }
1919         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1920     } elsif ($drive->{interface} eq 'usb') {
1921         die "implement me";
1922         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1923     } else {
1924         die "unsupported interface type";
1925     }
1926
1927     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1928
1929     if (my $serial = $drive->{serial}) {
1930         $serial = URI::Escape::uri_unescape($serial);
1931         $device .= ",serial=$serial";
1932     }
1933
1934
1935     return $device;
1936 }
1937
1938 sub get_initiator_name {
1939     my $initiator;
1940
1941     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return undef;
1942     while (defined(my $line = <$fh>)) {
1943         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1944         $initiator = $1;
1945         last;
1946     }
1947     $fh->close();
1948
1949     return $initiator;
1950 }
1951
1952 sub print_drive_full {
1953     my ($storecfg, $vmid, $drive) = @_;
1954
1955     my $path;
1956     my $volid = $drive->{file};
1957     my $format;
1958
1959     if (drive_is_cdrom($drive)) {
1960         $path = get_iso_path($storecfg, $vmid, $volid);
1961     } else {
1962         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1963         if ($storeid) {
1964             $path = PVE::Storage::path($storecfg, $volid);
1965             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
1966             $format = qemu_img_format($scfg, $volname);
1967         } else {
1968             $path = $volid;
1969             $format = "raw";
1970         }
1971    }
1972
1973     my $opts = '';
1974     my @qemu_drive_options = qw(heads secs cyls trans media format cache rerror werror aio discard);
1975     foreach my $o (@qemu_drive_options) {
1976         $opts .= ",$o=$drive->{$o}" if defined($drive->{$o});
1977     }
1978
1979     # snapshot only accepts on|off
1980     if (defined($drive->{snapshot})) {
1981         my $v = $drive->{snapshot} ? 'on' : 'off';
1982         $opts .= ",snapshot=$v";
1983     }
1984
1985     foreach my $type (['', '-total'], [_rd => '-read'], [_wr => '-write']) {
1986         my ($dir, $qmpname) = @$type;
1987         if (my $v = $drive->{"mbps$dir"}) {
1988             $opts .= ",throttling.bps$qmpname=".int($v*1024*1024);
1989         }
1990         if (my $v = $drive->{"mbps${dir}_max"}) {
1991             $opts .= ",throttling.bps$qmpname-max=".int($v*1024*1024);
1992         }
1993         if (my $v = $drive->{"bps${dir}_max_length"}) {
1994             $opts .= ",throttling.bps$qmpname-max-length=$v";
1995         }
1996         if (my $v = $drive->{"iops${dir}"}) {
1997             $opts .= ",throttling.iops$qmpname=$v";
1998         }
1999         if (my $v = $drive->{"iops${dir}_max"}) {
2000             $opts .= ",throttling.iops$qmpname-max=$v";
2001         }
2002         if (my $v = $drive->{"iops${dir}_max_length"}) {
2003             $opts .= ",throttling.iops$qmpname-max-length=$v";
2004         }
2005     }
2006
2007     $opts .= ",format=$format" if $format && !$drive->{format};
2008
2009     my $cache_direct = 0;
2010
2011     if (my $cache = $drive->{cache}) {
2012         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
2013     } elsif (!drive_is_cdrom($drive)) {
2014         $opts .= ",cache=none";
2015         $cache_direct = 1;
2016     }
2017
2018     # aio native works only with O_DIRECT
2019     if (!$drive->{aio}) {
2020         if($cache_direct) {
2021             $opts .= ",aio=native";
2022         } else {
2023             $opts .= ",aio=threads";
2024         }
2025     }
2026
2027     if (!drive_is_cdrom($drive)) {
2028         my $detectzeroes;
2029         if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
2030             $detectzeroes = 'off';
2031         } elsif ($drive->{discard}) {
2032             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
2033         } else {
2034             # This used to be our default with discard not being specified:
2035             $detectzeroes = 'on';
2036         }
2037         $opts .= ",detect-zeroes=$detectzeroes" if $detectzeroes;
2038     }
2039
2040     my $pathinfo = $path ? "file=$path," : '';
2041
2042     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
2043 }
2044
2045 sub print_netdevice_full {
2046     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files, $arch, $machine_type) = @_;
2047
2048     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
2049
2050     my $device = $net->{model};
2051     if ($net->{model} eq 'virtio') {
2052          $device = 'virtio-net-pci';
2053      };
2054
2055     my $pciaddr = print_pci_addr("$netid", $bridges, $arch, $machine_type);
2056     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
2057     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
2058         #Consider we have N queues, the number of vectors needed is 2*N + 2 (plus one config interrupt and control vq)
2059         my $vectors = $net->{queues} * 2 + 2;
2060         $tmpstr .= ",vectors=$vectors,mq=on";
2061     }
2062     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
2063
2064     if ($use_old_bios_files) {
2065         my $romfile;
2066         if ($device eq 'virtio-net-pci') {
2067             $romfile = 'pxe-virtio.rom';
2068         } elsif ($device eq 'e1000') {
2069             $romfile = 'pxe-e1000.rom';
2070         } elsif ($device eq 'ne2k') {
2071             $romfile = 'pxe-ne2k_pci.rom';
2072         } elsif ($device eq 'pcnet') {
2073             $romfile = 'pxe-pcnet.rom';
2074         } elsif ($device eq 'rtl8139') {
2075             $romfile = 'pxe-rtl8139.rom';
2076         }
2077         $tmpstr .= ",romfile=$romfile" if $romfile;
2078     }
2079
2080     return $tmpstr;
2081 }
2082
2083 sub print_netdev_full {
2084     my ($vmid, $conf, $arch, $net, $netid, $hotplug) = @_;
2085
2086     my $i = '';
2087     if ($netid =~ m/^net(\d+)$/) {
2088         $i = int($1);
2089     }
2090
2091     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
2092
2093     my $ifname = "tap${vmid}i$i";
2094
2095     # kvm uses TUNSETIFF ioctl, and that limits ifname length
2096     die "interface name '$ifname' is too long (max 15 character)\n"
2097         if length($ifname) >= 16;
2098
2099     my $vhostparam = '';
2100     if (is_native($arch)) {
2101         $vhostparam = ',vhost=on' if kernel_has_vhost_net() && $net->{model} eq 'virtio';
2102     }
2103
2104     my $vmname = $conf->{name} || "vm$vmid";
2105
2106     my $netdev = "";
2107     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
2108
2109     if ($net->{bridge}) {
2110         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script,downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
2111     } else {
2112         $netdev = "type=user,id=$netid,hostname=$vmname";
2113     }
2114
2115     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
2116
2117     return $netdev;
2118 }
2119
2120
2121 sub print_cpu_device {
2122     my ($conf, $id) = @_;
2123
2124     my $kvm = $conf->{kvm} // 1;
2125     my $cpu = $kvm ? "kvm64" : "qemu64";
2126     if (my $cputype = $conf->{cpu}) {
2127         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
2128             or die "Cannot parse cpu description: $cputype\n";
2129         $cpu = $cpuconf->{cputype};
2130     }
2131
2132     my $cores = $conf->{cores} || 1;
2133
2134     my $current_core = ($id - 1) % $cores;
2135     my $current_socket = int(($id - 1 - $current_core)/$cores);
2136
2137     return "$cpu-x86_64-cpu,id=cpu$id,socket-id=$current_socket,core-id=$current_core,thread-id=0";
2138 }
2139
2140 my $vga_map = {
2141     'cirrus' => 'cirrus-vga',
2142     'std' => 'VGA',
2143     'vmware' => 'vmware-svga',
2144     'virtio' => 'virtio-vga',
2145 };
2146
2147 sub print_vga_device {
2148     my ($conf, $vga, $arch, $machine_version, $machine, $id, $qxlnum, $bridges) = @_;
2149
2150     my $type = $vga_map->{$vga->{type}};
2151     if ($arch eq 'aarch64' && defined($type) && $type eq 'virtio-vga') {
2152         $type = 'virtio-gpu';
2153     }
2154     my $vgamem_mb = $vga->{memory};
2155
2156     my $max_outputs = '';
2157     if ($qxlnum) {
2158         $type = $id ? 'qxl' : 'qxl-vga';
2159
2160         if (!$conf->{ostype} || $conf->{ostype} =~ m/^(?:l\d\d)|(?:other)$/) {
2161             # set max outputs so linux can have up to 4 qxl displays with one device
2162             if (min_version($machine_version, 4, 1)) {
2163                 $max_outputs = ",max_outputs=4";
2164             }
2165         }
2166     }
2167
2168     die "no devicetype for $vga->{type}\n" if !$type;
2169
2170     my $memory = "";
2171     if ($vgamem_mb) {
2172         if ($vga->{type} eq 'virtio') {
2173             my $bytes = PVE::Tools::convert_size($vgamem_mb, "mb" => "b");
2174             $memory = ",max_hostmem=$bytes";
2175         } elsif ($qxlnum) {
2176             # from https://www.spice-space.org/multiple-monitors.html
2177             $memory = ",vgamem_mb=$vga->{memory}";
2178             my $ram = $vgamem_mb * 4;
2179             my $vram = $vgamem_mb * 2;
2180             $memory .= ",ram_size_mb=$ram,vram_size_mb=$vram";
2181         } else {
2182             $memory = ",vgamem_mb=$vga->{memory}";
2183         }
2184     } elsif ($qxlnum && $id) {
2185         $memory = ",ram_size=67108864,vram_size=33554432";
2186     }
2187
2188     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
2189     my $vgaid = "vga" . ($id // '');
2190     my $pciaddr;
2191
2192     if ($q35 && $vgaid eq 'vga') {
2193         # the first display uses pcie.0 bus on q35 machines
2194         $pciaddr = print_pcie_addr($vgaid, $bridges, $arch, $machine);
2195     } else {
2196         $pciaddr = print_pci_addr($vgaid, $bridges, $arch, $machine);
2197     }
2198
2199     return "$type,id=${vgaid}${memory}${max_outputs}${pciaddr}";
2200 }
2201
2202 sub drive_is_cloudinit {
2203     my ($drive) = @_;
2204     return $drive->{file} =~ m@[:/]vm-\d+-cloudinit(?:\.$QEMU_FORMAT_RE)?$@;
2205 }
2206
2207 sub drive_is_cdrom {
2208     my ($drive, $exclude_cloudinit) = @_;
2209
2210     return 0 if $exclude_cloudinit && drive_is_cloudinit($drive);
2211
2212     return $drive && $drive->{media} && ($drive->{media} eq 'cdrom');
2213
2214 }
2215
2216 sub parse_number_sets {
2217     my ($set) = @_;
2218     my $res = [];
2219     foreach my $part (split(/;/, $set)) {
2220         if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
2221             die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
2222             push @$res, [ $1, $2 ];
2223         } else {
2224             die "invalid range: $part\n";
2225         }
2226     }
2227     return $res;
2228 }
2229
2230 sub parse_numa {
2231     my ($data) = @_;
2232
2233     my $res = PVE::JSONSchema::parse_property_string($numa_fmt, $data);
2234     $res->{cpus} = parse_number_sets($res->{cpus}) if defined($res->{cpus});
2235     $res->{hostnodes} = parse_number_sets($res->{hostnodes}) if defined($res->{hostnodes});
2236     return $res;
2237 }
2238
2239 sub parse_hostpci {
2240     my ($value) = @_;
2241
2242     return undef if !$value;
2243
2244     my $res = PVE::JSONSchema::parse_property_string($hostpci_fmt, $value);
2245
2246     my @idlist = split(/;/, $res->{host});
2247     delete $res->{host};
2248     foreach my $id (@idlist) {
2249         if ($id =~ m/\./) { # full id 00:00.1
2250             push @{$res->{pciid}}, {
2251                 id => $id,
2252             };
2253         } else { # partial id 00:00
2254             $res->{pciid} = PVE::SysFSTools::lspci($id);
2255         }
2256     }
2257     return $res;
2258 }
2259
2260 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
2261 sub parse_net {
2262     my ($data) = @_;
2263
2264     my $res = eval { PVE::JSONSchema::parse_property_string($net_fmt, $data) };
2265     if ($@) {
2266         warn $@;
2267         return undef;
2268     }
2269     if (!defined($res->{macaddr})) {
2270         my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
2271         $res->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix});
2272     }
2273     return $res;
2274 }
2275
2276 # ipconfigX ip=cidr,gw=ip,ip6=cidr,gw6=ip
2277 sub parse_ipconfig {
2278     my ($data) = @_;
2279
2280     my $res = eval { PVE::JSONSchema::parse_property_string($ipconfig_fmt, $data) };
2281     if ($@) {
2282         warn $@;
2283         return undef;
2284     }
2285
2286     if ($res->{gw} && !$res->{ip}) {
2287         warn 'gateway specified without specifying an IP address';
2288         return undef;
2289     }
2290     if ($res->{gw6} && !$res->{ip6}) {
2291         warn 'IPv6 gateway specified without specifying an IPv6 address';
2292         return undef;
2293     }
2294     if ($res->{gw} && $res->{ip} eq 'dhcp') {
2295         warn 'gateway specified together with DHCP';
2296         return undef;
2297     }
2298     if ($res->{gw6} && $res->{ip6} !~ /^$IPV6RE/) {
2299         # gw6 + auto/dhcp
2300         warn "IPv6 gateway specified together with $res->{ip6} address";
2301         return undef;
2302     }
2303
2304     if (!$res->{ip} && !$res->{ip6}) {
2305         return { ip => 'dhcp', ip6 => 'dhcp' };
2306     }
2307
2308     return $res;
2309 }
2310
2311 sub print_net {
2312     my $net = shift;
2313
2314     return PVE::JSONSchema::print_property_string($net, $net_fmt);
2315 }
2316
2317 sub add_random_macs {
2318     my ($settings) = @_;
2319
2320     foreach my $opt (keys %$settings) {
2321         next if $opt !~ m/^net(\d+)$/;
2322         my $net = parse_net($settings->{$opt});
2323         next if !$net;
2324         $settings->{$opt} = print_net($net);
2325     }
2326 }
2327
2328 sub vm_is_volid_owner {
2329     my ($storecfg, $vmid, $volid) = @_;
2330
2331     if ($volid !~  m|^/|) {
2332         my ($path, $owner);
2333         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
2334         if ($owner && ($owner == $vmid)) {
2335             return 1;
2336         }
2337     }
2338
2339     return undef;
2340 }
2341
2342 sub vmconfig_register_unused_drive {
2343     my ($storecfg, $vmid, $conf, $drive) = @_;
2344
2345     if (drive_is_cloudinit($drive)) {
2346         eval { PVE::Storage::vdisk_free($storecfg, $drive->{file}) };
2347         warn $@ if $@;
2348     } elsif (!drive_is_cdrom($drive)) {
2349         my $volid = $drive->{file};
2350         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
2351             PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
2352         }
2353     }
2354 }
2355
2356 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str][,base64=bool]
2357 my $smbios1_fmt = {
2358     uuid => {
2359         type => 'string',
2360         pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
2361         format_description => 'UUID',
2362         description => "Set SMBIOS1 UUID.",
2363         optional => 1,
2364     },
2365     version => {
2366         type => 'string',
2367         pattern => '[A-Za-z0-9+\/]+={0,2}',
2368         format_description => 'Base64 encoded string',
2369         description => "Set SMBIOS1 version.",
2370         optional => 1,
2371     },
2372     serial => {
2373         type => 'string',
2374         pattern => '[A-Za-z0-9+\/]+={0,2}',
2375         format_description => 'Base64 encoded string',
2376         description => "Set SMBIOS1 serial number.",
2377         optional => 1,
2378     },
2379     manufacturer => {
2380         type => 'string',
2381         pattern => '[A-Za-z0-9+\/]+={0,2}',
2382         format_description => 'Base64 encoded string',
2383         description => "Set SMBIOS1 manufacturer.",
2384         optional => 1,
2385     },
2386     product => {
2387         type => 'string',
2388         pattern => '[A-Za-z0-9+\/]+={0,2}',
2389         format_description => 'Base64 encoded string',
2390         description => "Set SMBIOS1 product ID.",
2391         optional => 1,
2392     },
2393     sku => {
2394         type => 'string',
2395         pattern => '[A-Za-z0-9+\/]+={0,2}',
2396         format_description => 'Base64 encoded string',
2397         description => "Set SMBIOS1 SKU string.",
2398         optional => 1,
2399     },
2400     family => {
2401         type => 'string',
2402         pattern => '[A-Za-z0-9+\/]+={0,2}',
2403         format_description => 'Base64 encoded string',
2404         description => "Set SMBIOS1 family string.",
2405         optional => 1,
2406     },
2407     base64 => {
2408         type => 'boolean',
2409         description => 'Flag to indicate that the SMBIOS values are base64 encoded',
2410         optional => 1,
2411     },
2412 };
2413
2414 sub parse_smbios1 {
2415     my ($data) = @_;
2416
2417     my $res = eval { PVE::JSONSchema::parse_property_string($smbios1_fmt, $data) };
2418     warn $@ if $@;
2419     return $res;
2420 }
2421
2422 sub print_smbios1 {
2423     my ($smbios1) = @_;
2424     return PVE::JSONSchema::print_property_string($smbios1, $smbios1_fmt);
2425 }
2426
2427 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_fmt);
2428
2429 PVE::JSONSchema::register_format('pve-qm-bootdisk', \&verify_bootdisk);
2430 sub verify_bootdisk {
2431     my ($value, $noerr) = @_;
2432
2433     return $value if is_valid_drivename($value);
2434
2435     return undef if $noerr;
2436
2437     die "invalid boot disk '$value'\n";
2438 }
2439
2440 sub parse_watchdog {
2441     my ($value) = @_;
2442
2443     return undef if !$value;
2444
2445     my $res = eval { PVE::JSONSchema::parse_property_string($watchdog_fmt, $value) };
2446     warn $@ if $@;
2447     return $res;
2448 }
2449
2450 sub parse_guest_agent {
2451     my ($value) = @_;
2452
2453     return {} if !defined($value->{agent});
2454
2455     my $res = eval { PVE::JSONSchema::parse_property_string($agent_fmt, $value->{agent}) };
2456     warn $@ if $@;
2457
2458     # if the agent is disabled ignore the other potentially set properties
2459     return {} if !$res->{enabled};
2460     return $res;
2461 }
2462
2463 sub parse_vga {
2464     my ($value) = @_;
2465
2466     return {} if !$value;
2467     my $res = eval { PVE::JSONSchema::parse_property_string($vga_fmt, $value) };
2468     warn $@ if $@;
2469     return $res;
2470 }
2471
2472 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
2473 sub verify_usb_device {
2474     my ($value, $noerr) = @_;
2475
2476     return $value if parse_usb_device($value);
2477
2478     return undef if $noerr;
2479
2480     die "unable to parse usb device\n";
2481 }
2482
2483 # add JSON properties for create and set function
2484 sub json_config_properties {
2485     my $prop = shift;
2486
2487     foreach my $opt (keys %$confdesc) {
2488         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate' || $opt eq 'runningmachine';
2489         $prop->{$opt} = $confdesc->{$opt};
2490     }
2491
2492     return $prop;
2493 }
2494
2495 # return copy of $confdesc_cloudinit to generate documentation
2496 sub cloudinit_config_properties {
2497
2498     return dclone($confdesc_cloudinit);
2499 }
2500
2501 sub check_type {
2502     my ($key, $value) = @_;
2503
2504     die "unknown setting '$key'\n" if !$confdesc->{$key};
2505
2506     my $type = $confdesc->{$key}->{type};
2507
2508     if (!defined($value)) {
2509         die "got undefined value\n";
2510     }
2511
2512     if ($value =~ m/[\n\r]/) {
2513         die "property contains a line feed\n";
2514     }
2515
2516     if ($type eq 'boolean') {
2517         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
2518         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
2519         die "type check ('boolean') failed - got '$value'\n";
2520     } elsif ($type eq 'integer') {
2521         return int($1) if $value =~ m/^(\d+)$/;
2522         die "type check ('integer') failed - got '$value'\n";
2523     } elsif ($type eq 'number') {
2524         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
2525         die "type check ('number') failed - got '$value'\n";
2526     } elsif ($type eq 'string') {
2527         if (my $fmt = $confdesc->{$key}->{format}) {
2528             PVE::JSONSchema::check_format($fmt, $value);
2529             return $value;
2530         }
2531         $value =~ s/^\"(.*)\"$/$1/;
2532         return $value;
2533     } else {
2534         die "internal error"
2535     }
2536 }
2537
2538 sub destroy_vm {
2539     my ($storecfg, $vmid, $skiplock, $replacement_conf) = @_;
2540
2541     my $conf = PVE::QemuConfig->load_config($vmid);
2542
2543     PVE::QemuConfig->check_lock($conf) if !$skiplock;
2544
2545     if ($conf->{template}) {
2546         # check if any base image is still used by a linked clone
2547         foreach_drive($conf, sub {
2548                 my ($ds, $drive) = @_;
2549                 return if drive_is_cdrom($drive);
2550
2551                 my $volid = $drive->{file};
2552                 return if !$volid || $volid =~ m|^/|;
2553
2554                 die "base volume '$volid' is still in use by linked cloned\n"
2555                     if PVE::Storage::volume_is_base_and_used($storecfg, $volid);
2556
2557         });
2558     }
2559
2560     # only remove disks owned by this VM
2561     foreach_drive($conf, sub {
2562         my ($ds, $drive) = @_;
2563         return if drive_is_cdrom($drive, 1);
2564
2565         my $volid = $drive->{file};
2566         return if !$volid || $volid =~ m|^/|;
2567
2568         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
2569         return if !$path || !$owner || ($owner != $vmid);
2570
2571         eval { PVE::Storage::vdisk_free($storecfg, $volid) };
2572         warn "Could not remove disk '$volid', check manually: $@" if $@;
2573     });
2574
2575     # also remove unused disk
2576     my $vmdisks = PVE::Storage::vdisk_list($storecfg, undef, $vmid);
2577     PVE::Storage::foreach_volid($vmdisks, sub {
2578         my ($volid, $sid, $volname, $d) = @_;
2579         eval { PVE::Storage::vdisk_free($storecfg, $volid) };
2580         warn $@ if $@;
2581     });
2582
2583     if (defined $replacement_conf) {
2584         PVE::QemuConfig->write_config($vmid, $replacement_conf);
2585     } else {
2586         PVE::QemuConfig->destroy_config($vmid);
2587     }
2588 }
2589
2590 sub parse_vm_config {
2591     my ($filename, $raw) = @_;
2592
2593     return undef if !defined($raw);
2594
2595     my $res = {
2596         digest => Digest::SHA::sha1_hex($raw),
2597         snapshots => {},
2598         pending => {},
2599     };
2600
2601     $filename =~ m|/qemu-server/(\d+)\.conf$|
2602         || die "got strange filename '$filename'";
2603
2604     my $vmid = $1;
2605
2606     my $conf = $res;
2607     my $descr;
2608     my $section = '';
2609
2610     my @lines = split(/\n/, $raw);
2611     foreach my $line (@lines) {
2612         next if $line =~ m/^\s*$/;
2613
2614         if ($line =~ m/^\[PENDING\]\s*$/i) {
2615             $section = 'pending';
2616             if (defined($descr)) {
2617                 $descr =~ s/\s+$//;
2618                 $conf->{description} = $descr;
2619             }
2620             $descr = undef;
2621             $conf = $res->{$section} = {};
2622             next;
2623
2624         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2625             $section = $1;
2626             if (defined($descr)) {
2627                 $descr =~ s/\s+$//;
2628                 $conf->{description} = $descr;
2629             }
2630             $descr = undef;
2631             $conf = $res->{snapshots}->{$section} = {};
2632             next;
2633         }
2634
2635         if ($line =~ m/^\#(.*)\s*$/) {
2636             $descr = '' if !defined($descr);
2637             $descr .= PVE::Tools::decode_text($1) . "\n";
2638             next;
2639         }
2640
2641         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2642             $descr = '' if !defined($descr);
2643             $descr .= PVE::Tools::decode_text($2);
2644         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2645             $conf->{snapstate} = $1;
2646         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2647             my $key = $1;
2648             my $value = $2;
2649             $conf->{$key} = $value;
2650         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2651             my $value = $1;
2652             if ($section eq 'pending') {
2653                 $conf->{delete} = $value; # we parse this later
2654             } else {
2655                 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
2656             }
2657         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(.+?)\s*$/) {
2658             my $key = $1;
2659             my $value = $2;
2660             eval { $value = check_type($key, $value); };
2661             if ($@) {
2662                 warn "vm $vmid - unable to parse value of '$key' - $@";
2663             } else {
2664                 $key = 'ide2' if $key eq 'cdrom';
2665                 my $fmt = $confdesc->{$key}->{format};
2666                 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2667                     my $v = parse_drive($key, $value);
2668                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2669                         $v->{file} = $volid;
2670                         $value = print_drive($vmid, $v);
2671                     } else {
2672                         warn "vm $vmid - unable to parse value of '$key'\n";
2673                         next;
2674                     }
2675                 }
2676
2677                 $conf->{$key} = $value;
2678             }
2679         }
2680     }
2681
2682     if (defined($descr)) {
2683         $descr =~ s/\s+$//;
2684         $conf->{description} = $descr;
2685     }
2686     delete $res->{snapstate}; # just to be sure
2687
2688     return $res;
2689 }
2690
2691 sub write_vm_config {
2692     my ($filename, $conf) = @_;
2693
2694     delete $conf->{snapstate}; # just to be sure
2695
2696     if ($conf->{cdrom}) {
2697         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2698         $conf->{ide2} = $conf->{cdrom};
2699         delete $conf->{cdrom};
2700     }
2701
2702     # we do not use 'smp' any longer
2703     if ($conf->{sockets}) {
2704         delete $conf->{smp};
2705     } elsif ($conf->{smp}) {
2706         $conf->{sockets} = $conf->{smp};
2707         delete $conf->{cores};
2708         delete $conf->{smp};
2709     }
2710
2711     my $used_volids = {};
2712
2713     my $cleanup_config = sub {
2714         my ($cref, $pending, $snapname) = @_;
2715
2716         foreach my $key (keys %$cref) {
2717             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2718                 $key eq 'snapstate' || $key eq 'pending';
2719             my $value = $cref->{$key};
2720             if ($key eq 'delete') {
2721                 die "propertry 'delete' is only allowed in [PENDING]\n"
2722                     if !$pending;
2723                 # fixme: check syntax?
2724                 next;
2725             }
2726             eval { $value = check_type($key, $value); };
2727             die "unable to parse value of '$key' - $@" if $@;
2728
2729             $cref->{$key} = $value;
2730
2731             if (!$snapname && is_valid_drivename($key)) {
2732                 my $drive = parse_drive($key, $value);
2733                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2734             }
2735         }
2736     };
2737
2738     &$cleanup_config($conf);
2739
2740     &$cleanup_config($conf->{pending}, 1);
2741
2742     foreach my $snapname (keys %{$conf->{snapshots}}) {
2743         die "internal error" if $snapname eq 'pending';
2744         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2745     }
2746
2747     # remove 'unusedX' settings if we re-add a volume
2748     foreach my $key (keys %$conf) {
2749         my $value = $conf->{$key};
2750         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2751             delete $conf->{$key};
2752         }
2753     }
2754
2755     my $generate_raw_config = sub {
2756         my ($conf, $pending) = @_;
2757
2758         my $raw = '';
2759
2760         # add description as comment to top of file
2761         if (defined(my $descr = $conf->{description})) {
2762             if ($descr) {
2763                 foreach my $cl (split(/\n/, $descr)) {
2764                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2765                 }
2766             } else {
2767                 $raw .= "#\n" if $pending;
2768             }
2769         }
2770
2771         foreach my $key (sort keys %$conf) {
2772             next if $key eq 'digest' || $key eq 'description' || $key eq 'pending' || $key eq 'snapshots';
2773             $raw .= "$key: $conf->{$key}\n";
2774         }
2775         return $raw;
2776     };
2777
2778     my $raw = &$generate_raw_config($conf);
2779
2780     if (scalar(keys %{$conf->{pending}})){
2781         $raw .= "\n[PENDING]\n";
2782         $raw .= &$generate_raw_config($conf->{pending}, 1);
2783     }
2784
2785     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2786         $raw .= "\n[$snapname]\n";
2787         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2788     }
2789
2790     return $raw;
2791 }
2792
2793 sub load_defaults {
2794
2795     my $res = {};
2796
2797     # we use static defaults from our JSON schema configuration
2798     foreach my $key (keys %$confdesc) {
2799         if (defined(my $default = $confdesc->{$key}->{default})) {
2800             $res->{$key} = $default;
2801         }
2802     }
2803
2804     return $res;
2805 }
2806
2807 sub config_list {
2808     my $vmlist = PVE::Cluster::get_vmlist();
2809     my $res = {};
2810     return $res if !$vmlist || !$vmlist->{ids};
2811     my $ids = $vmlist->{ids};
2812
2813     foreach my $vmid (keys %$ids) {
2814         my $d = $ids->{$vmid};
2815         next if !$d->{node} || $d->{node} ne $nodename;
2816         next if !$d->{type} || $d->{type} ne 'qemu';
2817         $res->{$vmid}->{exists} = 1;
2818     }
2819     return $res;
2820 }
2821
2822 # test if VM uses local resources (to prevent migration)
2823 sub check_local_resources {
2824     my ($conf, $noerr) = @_;
2825
2826     my @loc_res = ();
2827
2828     push @loc_res, "hostusb" if $conf->{hostusb}; # old syntax
2829     push @loc_res, "hostpci" if $conf->{hostpci}; # old syntax
2830
2831     push @loc_res, "ivshmem" if $conf->{ivshmem};
2832
2833     foreach my $k (keys %$conf) {
2834         next if $k =~ m/^usb/ && ($conf->{$k} =~ m/^spice(?![^,])/);
2835         # sockets are safe: they will recreated be on the target side post-migrate
2836         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2837         push @loc_res, $k if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2838     }
2839
2840     die "VM uses local resources\n" if scalar @loc_res && !$noerr;
2841
2842     return \@loc_res;
2843 }
2844
2845 # check if used storages are available on all nodes (use by migrate)
2846 sub check_storage_availability {
2847     my ($storecfg, $conf, $node) = @_;
2848
2849     foreach_drive($conf, sub {
2850         my ($ds, $drive) = @_;
2851
2852         my $volid = $drive->{file};
2853         return if !$volid;
2854
2855         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2856         return if !$sid;
2857
2858         # check if storage is available on both nodes
2859         my $scfg = PVE::Storage::storage_check_node($storecfg, $sid);
2860         PVE::Storage::storage_check_node($storecfg, $sid, $node);
2861    });
2862 }
2863
2864 # list nodes where all VM images are available (used by has_feature API)
2865 sub shared_nodes {
2866     my ($conf, $storecfg) = @_;
2867
2868     my $nodelist = PVE::Cluster::get_nodelist();
2869     my $nodehash = { map { $_ => 1 } @$nodelist };
2870     my $nodename = PVE::INotify::nodename();
2871
2872     foreach_drive($conf, sub {
2873         my ($ds, $drive) = @_;
2874
2875         my $volid = $drive->{file};
2876         return if !$volid;
2877
2878         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2879         if ($storeid) {
2880             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2881             if ($scfg->{disable}) {
2882                 $nodehash = {};
2883             } elsif (my $avail = $scfg->{nodes}) {
2884                 foreach my $node (keys %$nodehash) {
2885                     delete $nodehash->{$node} if !$avail->{$node};
2886                 }
2887             } elsif (!$scfg->{shared}) {
2888                 foreach my $node (keys %$nodehash) {
2889                     delete $nodehash->{$node} if $node ne $nodename
2890                 }
2891             }
2892         }
2893     });
2894
2895     return $nodehash
2896 }
2897
2898 sub check_local_storage_availability {
2899     my ($conf, $storecfg) = @_;
2900
2901     my $nodelist = PVE::Cluster::get_nodelist();
2902     my $nodehash = { map { $_ => {} } @$nodelist };
2903
2904     foreach_drive($conf, sub {
2905         my ($ds, $drive) = @_;
2906
2907         my $volid = $drive->{file};
2908         return if !$volid;
2909
2910         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2911         if ($storeid) {
2912             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2913
2914             if ($scfg->{disable}) {
2915                 foreach my $node (keys %$nodehash) {
2916                     $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2917                 }
2918             } elsif (my $avail = $scfg->{nodes}) {
2919                 foreach my $node (keys %$nodehash) {
2920                     if (!$avail->{$node}) {
2921                         $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2922                     }
2923                 }
2924             }
2925         }
2926     });
2927
2928     foreach my $node (values %$nodehash) {
2929         if (my $unavail = $node->{unavailable_storages}) {
2930             $node->{unavailable_storages} = [ sort keys %$unavail ];
2931         }
2932     }
2933
2934     return $nodehash
2935 }
2936
2937 # Compat only, use assert_config_exists_on_node and vm_running_locally where possible
2938 sub check_running {
2939     my ($vmid, $nocheck, $node) = @_;
2940
2941     PVE::QemuConfig::assert_config_exists_on_node($vmid, $node) if !$nocheck;
2942     return PVE::QemuServer::Helpers::vm_running_locally($vmid);
2943 }
2944
2945 sub vzlist {
2946
2947     my $vzlist = config_list();
2948
2949     my $fd = IO::Dir->new($PVE::QemuServer::Helpers::var_run_tmpdir) || return $vzlist;
2950
2951     while (defined(my $de = $fd->read)) {
2952         next if $de !~ m/^(\d+)\.pid$/;
2953         my $vmid = $1;
2954         next if !defined($vzlist->{$vmid});
2955         if (my $pid = check_running($vmid)) {
2956             $vzlist->{$vmid}->{pid} = $pid;
2957         }
2958     }
2959
2960     return $vzlist;
2961 }
2962
2963 sub disksize {
2964     my ($storecfg, $conf) = @_;
2965
2966     my $bootdisk = $conf->{bootdisk};
2967     return undef if !$bootdisk;
2968     return undef if !is_valid_drivename($bootdisk);
2969
2970     return undef if !$conf->{$bootdisk};
2971
2972     my $drive = parse_drive($bootdisk, $conf->{$bootdisk});
2973     return undef if !defined($drive);
2974
2975     return undef if drive_is_cdrom($drive);
2976
2977     my $volid = $drive->{file};
2978     return undef if !$volid;
2979
2980     return $drive->{size};
2981 }
2982
2983 our $vmstatus_return_properties = {
2984     vmid => get_standard_option('pve-vmid'),
2985     status => {
2986         description => "Qemu process status.",
2987         type => 'string',
2988         enum => ['stopped', 'running'],
2989     },
2990     maxmem => {
2991         description => "Maximum memory in bytes.",
2992         type => 'integer',
2993         optional => 1,
2994         renderer => 'bytes',
2995     },
2996     maxdisk => {
2997         description => "Root disk size in bytes.",
2998         type => 'integer',
2999         optional => 1,
3000         renderer => 'bytes',
3001     },
3002     name => {
3003         description => "VM name.",
3004         type => 'string',
3005         optional => 1,
3006     },
3007     qmpstatus => {
3008         description => "Qemu QMP agent status.",
3009         type => 'string',
3010         optional => 1,
3011     },
3012     pid => {
3013         description => "PID of running qemu process.",
3014         type => 'integer',
3015         optional => 1,
3016     },
3017     uptime => {
3018         description => "Uptime.",
3019         type => 'integer',
3020         optional => 1,
3021         renderer => 'duration',
3022     },
3023     cpus => {
3024         description => "Maximum usable CPUs.",
3025         type => 'number',
3026         optional => 1,
3027     },
3028     lock => {
3029         description => "The current config lock, if any.",
3030         type => 'string',
3031         optional => 1,
3032     },
3033     tags => {
3034         description  => "The current configured tags, if any",
3035         type => 'string',
3036         optional => 1,
3037     },
3038 };
3039
3040 my $last_proc_pid_stat;
3041
3042 # get VM status information
3043 # This must be fast and should not block ($full == false)
3044 # We only query KVM using QMP if $full == true (this can be slow)
3045 sub vmstatus {
3046     my ($opt_vmid, $full) = @_;
3047
3048     my $res = {};
3049
3050     my $storecfg = PVE::Storage::config();
3051
3052     my $list = vzlist();
3053     my $defaults = load_defaults();
3054
3055     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
3056
3057     my $cpucount = $cpuinfo->{cpus} || 1;
3058
3059     foreach my $vmid (keys %$list) {
3060         next if $opt_vmid && ($vmid ne $opt_vmid);
3061
3062         my $conf = PVE::QemuConfig->load_config($vmid);
3063
3064         my $d = { vmid => $vmid };
3065         $d->{pid} = $list->{$vmid}->{pid};
3066
3067         # fixme: better status?
3068         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
3069
3070         my $size = disksize($storecfg, $conf);
3071         if (defined($size)) {
3072             $d->{disk} = 0; # no info available
3073             $d->{maxdisk} = $size;
3074         } else {
3075             $d->{disk} = 0;
3076             $d->{maxdisk} = 0;
3077         }
3078
3079         $d->{cpus} = ($conf->{sockets} || $defaults->{sockets})
3080             * ($conf->{cores} || $defaults->{cores});
3081         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
3082         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
3083
3084         $d->{name} = $conf->{name} || "VM $vmid";
3085         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024)
3086             : $defaults->{memory}*(1024*1024);
3087
3088         if ($conf->{balloon}) {
3089             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
3090             $d->{shares} = defined($conf->{shares}) ? $conf->{shares}
3091                 : $defaults->{shares};
3092         }
3093
3094         $d->{uptime} = 0;
3095         $d->{cpu} = 0;
3096         $d->{mem} = 0;
3097
3098         $d->{netout} = 0;
3099         $d->{netin} = 0;
3100
3101         $d->{diskread} = 0;
3102         $d->{diskwrite} = 0;
3103
3104         $d->{template} = PVE::QemuConfig->is_template($conf);
3105
3106         $d->{serial} = 1 if conf_has_serial($conf);
3107         $d->{lock} = $conf->{lock} if $conf->{lock};
3108         $d->{tags} = $conf->{tags} if defined($conf->{tags});
3109
3110         $res->{$vmid} = $d;
3111     }
3112
3113     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
3114     foreach my $dev (keys %$netdev) {
3115         next if $dev !~ m/^tap([1-9]\d*)i/;
3116         my $vmid = $1;
3117         my $d = $res->{$vmid};
3118         next if !$d;
3119
3120         $d->{netout} += $netdev->{$dev}->{receive};
3121         $d->{netin} += $netdev->{$dev}->{transmit};
3122
3123         if ($full) {
3124             $d->{nics}->{$dev}->{netout} = $netdev->{$dev}->{receive};
3125             $d->{nics}->{$dev}->{netin} = $netdev->{$dev}->{transmit};
3126         }
3127
3128     }
3129
3130     my $ctime = gettimeofday;
3131
3132     foreach my $vmid (keys %$list) {
3133
3134         my $d = $res->{$vmid};
3135         my $pid = $d->{pid};
3136         next if !$pid;
3137
3138         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
3139         next if !$pstat; # not running
3140
3141         my $used = $pstat->{utime} + $pstat->{stime};
3142
3143         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
3144
3145         if ($pstat->{vsize}) {
3146             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
3147         }
3148
3149         my $old = $last_proc_pid_stat->{$pid};
3150         if (!$old) {
3151             $last_proc_pid_stat->{$pid} = {
3152                 time => $ctime,
3153                 used => $used,
3154                 cpu => 0,
3155             };
3156             next;
3157         }
3158
3159         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
3160
3161         if ($dtime > 1000) {
3162             my $dutime = $used -  $old->{used};
3163
3164             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
3165             $last_proc_pid_stat->{$pid} = {
3166                 time => $ctime,
3167                 used => $used,
3168                 cpu => $d->{cpu},
3169             };
3170         } else {
3171             $d->{cpu} = $old->{cpu};
3172         }
3173     }
3174
3175     return $res if !$full;
3176
3177     my $qmpclient = PVE::QMPClient->new();
3178
3179     my $ballooncb = sub {
3180         my ($vmid, $resp) = @_;
3181
3182         my $info = $resp->{'return'};
3183         return if !$info->{max_mem};
3184
3185         my $d = $res->{$vmid};
3186
3187         # use memory assigned to VM
3188         $d->{maxmem} = $info->{max_mem};
3189         $d->{balloon} = $info->{actual};
3190
3191         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
3192             $d->{mem} = $info->{total_mem} - $info->{free_mem};
3193             $d->{freemem} = $info->{free_mem};
3194         }
3195
3196         $d->{ballooninfo} = $info;
3197     };
3198
3199     my $blockstatscb = sub {
3200         my ($vmid, $resp) = @_;
3201         my $data = $resp->{'return'} || [];
3202         my $totalrdbytes = 0;
3203         my $totalwrbytes = 0;
3204
3205         for my $blockstat (@$data) {
3206             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
3207             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
3208
3209             $blockstat->{device} =~ s/drive-//;
3210             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
3211         }
3212         $res->{$vmid}->{diskread} = $totalrdbytes;
3213         $res->{$vmid}->{diskwrite} = $totalwrbytes;
3214     };
3215
3216     my $statuscb = sub {
3217         my ($vmid, $resp) = @_;
3218
3219         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
3220         # this fails if ballon driver is not loaded, so this must be
3221         # the last commnand (following command are aborted if this fails).
3222         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
3223
3224         my $status = 'unknown';
3225         if (!defined($status = $resp->{'return'}->{status})) {
3226             warn "unable to get VM status\n";
3227             return;
3228         }
3229
3230         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
3231     };
3232
3233     foreach my $vmid (keys %$list) {
3234         next if $opt_vmid && ($vmid ne $opt_vmid);
3235         next if !$res->{$vmid}->{pid}; # not running
3236         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
3237     }
3238
3239     $qmpclient->queue_execute(undef, 2);
3240
3241     foreach my $vmid (keys %$list) {
3242         next if $opt_vmid && ($vmid ne $opt_vmid);
3243         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
3244     }
3245
3246     return $res;
3247 }
3248
3249 sub foreach_drive {
3250     my ($conf, $func, @param) = @_;
3251
3252     foreach my $ds (valid_drive_names()) {
3253         next if !defined($conf->{$ds});
3254
3255         my $drive = parse_drive($ds, $conf->{$ds});
3256         next if !$drive;
3257
3258         &$func($ds, $drive, @param);
3259     }
3260 }
3261
3262 sub foreach_volid {
3263     my ($conf, $func, @param) = @_;
3264
3265     my $volhash = {};
3266
3267     my $test_volid = sub {
3268         my ($volid, $is_cdrom, $replicate, $shared, $snapname, $size) = @_;
3269
3270         return if !$volid;
3271
3272         $volhash->{$volid}->{cdrom} //= 1;
3273         $volhash->{$volid}->{cdrom} = 0 if !$is_cdrom;
3274
3275         $volhash->{$volid}->{replicate} //= 0;
3276         $volhash->{$volid}->{replicate} = 1 if $replicate;
3277
3278         $volhash->{$volid}->{shared} //= 0;
3279         $volhash->{$volid}->{shared} = 1 if $shared;
3280
3281         $volhash->{$volid}->{referenced_in_config} //= 0;
3282         $volhash->{$volid}->{referenced_in_config} = 1 if !defined($snapname);
3283
3284         $volhash->{$volid}->{referenced_in_snapshot}->{$snapname} = 1
3285             if defined($snapname);
3286         $volhash->{$volid}->{size} = $size if $size;
3287     };
3288
3289     foreach_drive($conf, sub {
3290         my ($ds, $drive) = @_;
3291         $test_volid->($drive->{file}, drive_is_cdrom($drive), $drive->{replicate} // 1, $drive->{shared}, undef, $drive->{size});
3292     });
3293
3294     foreach my $snapname (keys %{$conf->{snapshots}}) {
3295         my $snap = $conf->{snapshots}->{$snapname};
3296         $test_volid->($snap->{vmstate}, 0, 1, $snapname);
3297         foreach_drive($snap, sub {
3298             my ($ds, $drive) = @_;
3299             $test_volid->($drive->{file}, drive_is_cdrom($drive), $drive->{replicate} // 1, $drive->{shared}, $snapname);
3300         });
3301     }
3302
3303     foreach my $volid (keys %$volhash) {
3304         &$func($volid, $volhash->{$volid}, @param);
3305     }
3306 }
3307
3308 sub conf_has_serial {
3309     my ($conf) = @_;
3310
3311     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3312         if ($conf->{"serial$i"}) {
3313             return 1;
3314         }
3315     }
3316
3317     return 0;
3318 }
3319
3320 sub conf_has_audio {
3321     my ($conf, $id) = @_;
3322
3323     $id //= 0;
3324     my $audio = $conf->{"audio$id"};
3325     return undef if !defined($audio);
3326
3327     my $audioproperties = PVE::JSONSchema::parse_property_string($audio_fmt, $audio);
3328     my $audiodriver = $audioproperties->{driver} // 'spice';
3329
3330     return {
3331         dev => $audioproperties->{device},
3332         dev_id => "audiodev$id",
3333         backend => $audiodriver,
3334         backend_id => "$audiodriver-backend${id}",
3335     };
3336 }
3337
3338 sub vga_conf_has_spice {
3339     my ($vga) = @_;
3340
3341     my $vgaconf = parse_vga($vga);
3342     my $vgatype = $vgaconf->{type};
3343     return 0 if !$vgatype || $vgatype !~ m/^qxl([234])?$/;
3344
3345     return $1 || 1;
3346 }
3347
3348 sub is_native($) {
3349     my ($arch) = @_;
3350     return get_host_arch() eq $arch;
3351 }
3352
3353 sub get_vm_arch {
3354     my ($conf) = @_;
3355     return $conf->{arch} // get_host_arch();
3356 }
3357
3358 my $default_machines = {
3359     x86_64 => 'pc',
3360     aarch64 => 'virt',
3361 };
3362
3363 sub get_vm_machine {
3364     my ($conf, $forcemachine, $arch) = @_;
3365
3366     my $machine = $forcemachine || $conf->{machine};
3367
3368     if (!$machine) {
3369         $arch //= 'x86_64';
3370         $machine ||= $default_machines->{$arch};
3371     }
3372
3373     return $machine;
3374 }
3375
3376 sub get_ovmf_files($) {
3377     my ($arch) = @_;
3378
3379     my $ovmf = $OVMF->{$arch}
3380         or die "no OVMF images known for architecture '$arch'\n";
3381
3382     return @$ovmf;
3383 }
3384
3385 my $Arch2Qemu = {
3386     aarch64 => '/usr/bin/qemu-system-aarch64',
3387     x86_64 => '/usr/bin/qemu-system-x86_64',
3388 };
3389 sub get_command_for_arch($) {
3390     my ($arch) = @_;
3391     return '/usr/bin/kvm' if is_native($arch);
3392
3393     my $cmd = $Arch2Qemu->{$arch}
3394         or die "don't know how to emulate architecture '$arch'\n";
3395     return $cmd;
3396 }
3397
3398 sub get_cpu_options {
3399     my ($conf, $arch, $kvm, $kvm_off, $machine_version, $winversion, $gpu_passthrough) = @_;
3400
3401     my $cpuFlags = [];
3402     my $ostype = $conf->{ostype};
3403
3404     my $cpu = $kvm ? "kvm64" : "qemu64";
3405     if ($arch eq 'aarch64') {
3406         $cpu = 'cortex-a57';
3407     }
3408     my $hv_vendor_id;
3409     if (my $cputype = $conf->{cpu}) {
3410         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
3411             or die "Cannot parse cpu description: $cputype\n";
3412         $cpu = $cpuconf->{cputype};
3413         $kvm_off = 1 if $cpuconf->{hidden};
3414         $hv_vendor_id = $cpuconf->{'hv-vendor-id'};
3415
3416         if (defined(my $flags = $cpuconf->{flags})) {
3417             push @$cpuFlags, split(";", $flags);
3418         }
3419     }
3420
3421     push @$cpuFlags , '+lahf_lm' if $cpu eq 'kvm64' && $arch eq 'x86_64';
3422
3423     push @$cpuFlags , '-x2apic' if $ostype && $ostype eq 'solaris';
3424
3425     push @$cpuFlags, '+sep' if $cpu eq 'kvm64' || $cpu eq 'kvm32';
3426
3427     push @$cpuFlags, '-rdtscp' if $cpu =~ m/^Opteron/;
3428
3429     if (min_version($machine_version, 2, 3) && $arch eq 'x86_64') {
3430
3431         push @$cpuFlags , '+kvm_pv_unhalt' if $kvm;
3432         push @$cpuFlags , '+kvm_pv_eoi' if $kvm;
3433     }
3434
3435     add_hyperv_enlightenments($cpuFlags, $winversion, $machine_version, $conf->{bios}, $gpu_passthrough, $hv_vendor_id) if $kvm;
3436
3437     push @$cpuFlags, 'enforce' if $cpu ne 'host' && $kvm && $arch eq 'x86_64';
3438
3439     push @$cpuFlags, 'kvm=off' if $kvm_off;
3440
3441     if (my $cpu_vendor = $cpu_vendor_list->{$cpu}) {
3442         push @$cpuFlags, "vendor=${cpu_vendor}"
3443             if $cpu_vendor ne 'default';
3444     } elsif ($arch ne 'aarch64') {
3445         die "internal error"; # should not happen
3446     }
3447
3448     $cpu .= "," . join(',', @$cpuFlags) if scalar(@$cpuFlags);
3449
3450     return ('-cpu', $cpu);
3451 }
3452
3453 sub config_to_command {
3454     my ($storecfg, $vmid, $conf, $defaults, $forcemachine) = @_;
3455
3456     my $cmd = [];
3457     my $globalFlags = [];
3458     my $machineFlags = [];
3459     my $rtcFlags = [];
3460     my $devices = [];
3461     my $pciaddr = '';
3462     my $bridges = {};
3463     my $vernum = 0; # unknown
3464     my $ostype = $conf->{ostype};
3465     my $winversion = windows_version($ostype);
3466     my $kvm = $conf->{kvm};
3467
3468     my $arch = get_vm_arch($conf);
3469     my $kvm_binary = get_command_for_arch($arch);
3470     my $kvmver = kvm_user_version($kvm_binary);
3471
3472     my $machine_type = get_vm_machine($conf, $forcemachine, $arch);
3473     my $machine_version = PVE::QemuServer::Machine::extract_version($machine_type) // $kvmver;
3474     $kvm //= 1 if is_native($arch);
3475
3476     if ($kvm) {
3477         die "KVM virtualisation configured, but not available. Either disable in VM configuration or enable in BIOS.\n"
3478             if !defined kvm_version();
3479     }
3480
3481     if ($kvmver =~ m/^(\d+)\.(\d+)$/) {
3482         $vernum = $1*1000000+$2*1000;
3483     } elsif ($kvmver =~ m/^(\d+)\.(\d+)\.(\d+)$/) {
3484         $vernum = $1*1000000+$2*1000+$3;
3485     }
3486
3487     die "detected old qemu-kvm binary ($kvmver)\n" if $vernum < 15000;
3488
3489     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
3490     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
3491     my $use_old_bios_files = undef;
3492     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3493
3494     my $cpuunits = defined($conf->{cpuunits}) ?
3495             $conf->{cpuunits} : $defaults->{cpuunits};
3496
3497     push @$cmd, $kvm_binary;
3498
3499     push @$cmd, '-id', $vmid;
3500
3501     my $vmname = $conf->{name} || "vm$vmid";
3502
3503     push @$cmd, '-name', $vmname;
3504
3505     my $use_virtio = 0;
3506
3507     my $qmpsocket = PVE::QemuServer::Helpers::qmp_socket($vmid);
3508     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server,nowait";
3509     push @$cmd, '-mon', "chardev=qmp,mode=control";
3510
3511     if (min_version($machine_version, 2, 12)) {
3512         push @$cmd, '-chardev', "socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5";
3513         push @$cmd, '-mon', "chardev=qmp-event,mode=control";
3514     }
3515
3516     push @$cmd, '-pidfile' , PVE::QemuServer::Helpers::pidfile_name($vmid);
3517
3518     push @$cmd, '-daemonize';
3519
3520     if ($conf->{smbios1}) {
3521         my $smbios_conf = parse_smbios1($conf->{smbios1});
3522         if ($smbios_conf->{base64}) {
3523             # Do not pass base64 flag to qemu
3524             delete $smbios_conf->{base64};
3525             my $smbios_string = "";
3526             foreach my $key (keys %$smbios_conf) {
3527                 my $value;
3528                 if ($key eq "uuid") {
3529                     $value = $smbios_conf->{uuid}
3530                 } else {
3531                     $value = decode_base64($smbios_conf->{$key});
3532                 }
3533                 # qemu accepts any binary data, only commas need escaping by double comma
3534                 $value =~ s/,/,,/g;
3535                 $smbios_string .= "," . $key . "=" . $value if $value;
3536             }
3537             push @$cmd, '-smbios', "type=1" . $smbios_string;
3538         } else {
3539             push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
3540         }
3541     }
3542
3543     if ($conf->{vmgenid}) {
3544         push @$devices, '-device', 'vmgenid,guid='.$conf->{vmgenid};
3545     }
3546
3547     my ($ovmf_code, $ovmf_vars) = get_ovmf_files($arch);
3548     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3549         die "uefi base image not found\n" if ! -f $ovmf_code;
3550
3551         my $path;
3552         my $format;
3553         if (my $efidisk = $conf->{efidisk0}) {
3554             my $d = PVE::JSONSchema::parse_property_string($efidisk_fmt, $efidisk);
3555             my ($storeid, $volname) = PVE::Storage::parse_volume_id($d->{file}, 1);
3556             $format = $d->{format};
3557             if ($storeid) {
3558                 $path = PVE::Storage::path($storecfg, $d->{file});
3559                 if (!defined($format)) {
3560                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
3561                     $format = qemu_img_format($scfg, $volname);
3562                 }
3563             } else {
3564                 $path = $d->{file};
3565                 die "efidisk format must be specified\n"
3566                     if !defined($format);
3567             }
3568         } else {
3569             warn "no efidisk configured! Using temporary efivars disk.\n";
3570             $path = "/tmp/$vmid-ovmf.fd";
3571             PVE::Tools::file_copy($ovmf_vars, $path, -s $ovmf_vars);
3572             $format = 'raw';
3573         }
3574
3575         push @$cmd, '-drive', "if=pflash,unit=0,format=raw,readonly,file=$ovmf_code";
3576         push @$cmd, '-drive', "if=pflash,unit=1,format=$format,id=drive-efidisk0,file=$path";
3577     }
3578
3579     # load q35 config
3580     if ($q35) {
3581         # we use different pcie-port hardware for qemu >= 4.0 for passthrough
3582         if (min_version($machine_version, 4, 0)) {
3583             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35-4.0.cfg';
3584         } else {
3585             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35.cfg';
3586         }
3587     }
3588
3589     # add usb controllers
3590     my @usbcontrollers = PVE::QemuServer::USB::get_usb_controllers($conf, $bridges, $arch, $machine_type, $usbdesc->{format}, $MAX_USB_DEVICES);
3591     push @$devices, @usbcontrollers if @usbcontrollers;
3592     my $vga = parse_vga($conf->{vga});
3593
3594     my $qxlnum = vga_conf_has_spice($conf->{vga});
3595     $vga->{type} = 'qxl' if $qxlnum;
3596
3597     if (!$vga->{type}) {
3598         if ($arch eq 'aarch64') {
3599             $vga->{type} = 'virtio';
3600         } elsif (min_version($machine_version, 2, 9)) {
3601             $vga->{type} = (!$winversion || $winversion >= 6) ? 'std' : 'cirrus';
3602         } else {
3603             $vga->{type} = ($winversion >= 6) ? 'std' : 'cirrus';
3604         }
3605     }
3606
3607     # enable absolute mouse coordinates (needed by vnc)
3608     my $tablet;
3609     if (defined($conf->{tablet})) {
3610         $tablet = $conf->{tablet};
3611     } else {
3612         $tablet = $defaults->{tablet};
3613         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
3614         $tablet = 0 if $vga->{type} =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
3615     }
3616
3617     if ($tablet) {
3618         push @$devices, '-device', print_tabletdevice_full($conf, $arch) if $tablet;
3619         my $kbd = print_keyboarddevice_full($conf, $arch);
3620         push @$devices, '-device', $kbd if defined($kbd);
3621     }
3622
3623     my $kvm_off = 0;
3624     my $gpu_passthrough;
3625
3626     # host pci devices
3627     for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
3628         my $id = "hostpci$i";
3629         my $d = parse_hostpci($conf->{$id});
3630         next if !$d;
3631
3632         if (my $pcie = $d->{pcie}) {
3633             die "q35 machine model is not enabled" if !$q35;
3634             # win7 wants to have the pcie devices directly on the pcie bus
3635             # instead of in the root port
3636             if ($winversion == 7) {
3637                 $pciaddr = print_pcie_addr("${id}bus0");
3638             } else {
3639                 # add more root ports if needed, 4 are present by default
3640                 # by pve-q35 cfgs, rest added here on demand.
3641                 if ($i > 3) {
3642                     push @$devices, '-device', print_pcie_root_port($i);
3643                 }
3644                 $pciaddr = print_pcie_addr($id);
3645             }
3646         } else {
3647             $pciaddr = print_pci_addr($id, $bridges, $arch, $machine_type);
3648         }
3649
3650         my $xvga = '';
3651         if ($d->{'x-vga'}) {
3652             $xvga = ',x-vga=on' if !($conf->{bios} && $conf->{bios} eq 'ovmf');
3653             $kvm_off = 1;
3654             $vga->{type} = 'none' if !defined($conf->{vga});
3655             $gpu_passthrough = 1;
3656         }
3657
3658         my $pcidevices = $d->{pciid};
3659         my $multifunction = 1 if @$pcidevices > 1;
3660
3661         my $sysfspath;
3662         if ($d->{mdev} && scalar(@$pcidevices) == 1) {
3663             my $pci_id = $pcidevices->[0]->{id};
3664             my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $i);
3665             $sysfspath = "/sys/bus/pci/devices/$pci_id/$uuid";
3666         } elsif ($d->{mdev}) {
3667             warn "ignoring mediated device '$id' with multifunction device\n";
3668         }
3669
3670         my $j=0;
3671         foreach my $pcidevice (@$pcidevices) {
3672             my $devicestr = "vfio-pci";
3673
3674             if ($sysfspath) {
3675                 $devicestr .= ",sysfsdev=$sysfspath";
3676             } else {
3677                 $devicestr .= ",host=$pcidevice->{id}";
3678             }
3679
3680             my $mf_addr = $multifunction ? ".$j" : '';
3681             $devicestr .= ",id=${id}${mf_addr}${pciaddr}${mf_addr}";
3682
3683             if ($j == 0) {
3684                 $devicestr .= ',rombar=0' if defined($d->{rombar}) && !$d->{rombar};
3685                 $devicestr .= "$xvga";
3686                 $devicestr .= ",multifunction=on" if $multifunction;
3687                 $devicestr .= ",romfile=/usr/share/kvm/$d->{romfile}" if $d->{romfile};
3688             }
3689
3690             push @$devices, '-device', $devicestr;
3691             $j++;
3692         }
3693     }
3694
3695     # usb devices
3696     my $usb_dev_features = {};
3697     $usb_dev_features->{spice_usb3} = 1 if min_version($machine_version, 4, 0);
3698
3699     my @usbdevices = PVE::QemuServer::USB::get_usb_devices($conf, $usbdesc->{format}, $MAX_USB_DEVICES, $usb_dev_features);
3700     push @$devices, @usbdevices if @usbdevices;
3701     # serial devices
3702     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3703         if (my $path = $conf->{"serial$i"}) {
3704             if ($path eq 'socket') {
3705                 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
3706                 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server,nowait";
3707                 # On aarch64, serial0 is the UART device. Qemu only allows
3708                 # connecting UART devices via the '-serial' command line, as
3709                 # the device has a fixed slot on the hardware...
3710                 if ($arch eq 'aarch64' && $i == 0) {
3711                     push @$devices, '-serial', "chardev:serial$i";
3712                 } else {
3713                     push @$devices, '-device', "isa-serial,chardev=serial$i";
3714                 }
3715             } else {
3716                 die "no such serial device\n" if ! -c $path;
3717                 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
3718                 push @$devices, '-device', "isa-serial,chardev=serial$i";
3719             }
3720         }
3721     }
3722
3723     # parallel devices
3724     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
3725         if (my $path = $conf->{"parallel$i"}) {
3726             die "no such parallel device\n" if ! -c $path;
3727             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
3728             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
3729             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
3730         }
3731     }
3732
3733     if (my $audio = conf_has_audio($conf)) {
3734
3735         my $audiopciaddr = print_pci_addr("audio0", $bridges, $arch, $machine_type);
3736
3737         my $id = $audio->{dev_id};
3738         if ($audio->{dev} eq 'AC97') {
3739             push @$devices, '-device', "AC97,id=${id}${audiopciaddr}";
3740         } elsif ($audio->{dev} =~ /intel\-hda$/) {
3741             push @$devices, '-device', "$audio->{dev},id=${id}${audiopciaddr}";
3742             push @$devices, '-device', "hda-micro,id=${id}-codec0,bus=${id}.0,cad=0";
3743             push @$devices, '-device', "hda-duplex,id=${id}-codec1,bus=${id}.0,cad=1";
3744         } else {
3745             die "unkown audio device '$audio->{dev}', implement me!";
3746         }
3747
3748         push @$devices, '-audiodev', "$audio->{backend},id=$audio->{backend_id}";
3749     }
3750
3751     my $sockets = 1;
3752     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3753     $sockets = $conf->{sockets} if  $conf->{sockets};
3754
3755     my $cores = $conf->{cores} || 1;
3756
3757     my $maxcpus = $sockets * $cores;
3758
3759     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
3760
3761     my $allowed_vcpus = $cpuinfo->{cpus};
3762
3763     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n"
3764         if ($allowed_vcpus < $maxcpus);
3765
3766     if($hotplug_features->{cpu} && min_version($machine_version, 2, 7)) {
3767
3768         push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3769         for (my $i = 2; $i <= $vcpus; $i++)  {
3770             my $cpustr = print_cpu_device($conf,$i);
3771             push @$cmd, '-device', $cpustr;
3772         }
3773
3774     } else {
3775
3776         push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3777     }
3778     push @$cmd, '-nodefaults';
3779
3780     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
3781
3782     my $bootindex_hash = {};
3783     my $i = 1;
3784     foreach my $o (split(//, $bootorder)) {
3785         $bootindex_hash->{$o} = $i*100;
3786         $i++;
3787     }
3788
3789     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
3790
3791     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
3792
3793     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
3794
3795     if ($vga->{type} && $vga->{type} !~ m/^serial\d+$/ && $vga->{type} ne 'none'){
3796         push @$devices, '-device', print_vga_device($conf, $vga, $arch, $machine_version, $machine_type, undef, $qxlnum, $bridges);
3797         my $socket = PVE::QemuServer::Helpers::vnc_socket($vmid);
3798         push @$cmd,  '-vnc', "unix:$socket,password";
3799     } else {
3800         push @$cmd, '-vga', 'none' if $vga->{type} eq 'none';
3801         push @$cmd, '-nographic';
3802     }
3803
3804     # time drift fix
3805     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
3806
3807     my $useLocaltime = $conf->{localtime};
3808
3809     if ($winversion >= 5) { # windows
3810         $useLocaltime = 1 if !defined($conf->{localtime});
3811
3812         # use time drift fix when acpi is enabled
3813         if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
3814             $tdf = 1 if !defined($conf->{tdf});
3815         }
3816     }
3817
3818     if ($winversion >= 6) {
3819         push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3820         push @$cmd, '-no-hpet';
3821     }
3822
3823     push @$rtcFlags, 'driftfix=slew' if $tdf;
3824
3825     if (!$kvm) {
3826         push @$machineFlags, 'accel=tcg';
3827     }
3828
3829     if ($machine_type) {
3830         push @$machineFlags, "type=${machine_type}";
3831     }
3832
3833     if (($conf->{startdate}) && ($conf->{startdate} ne 'now')) {
3834         push @$rtcFlags, "base=$conf->{startdate}";
3835     } elsif ($useLocaltime) {
3836         push @$rtcFlags, 'base=localtime';
3837     }
3838
3839     push @$cmd, get_cpu_options($conf, $arch, $kvm, $kvm_off, $machine_version, $winversion, $gpu_passthrough);
3840
3841     PVE::QemuServer::Memory::config($conf, $vmid, $sockets, $cores, $defaults, $hotplug_features, $cmd);
3842
3843     push @$cmd, '-S' if $conf->{freeze};
3844
3845     push @$cmd, '-k', $conf->{keyboard} if defined($conf->{keyboard});
3846
3847     my $guest_agent = parse_guest_agent($conf);
3848
3849     if ($guest_agent->{enabled}) {
3850         my $qgasocket = PVE::QemuServer::Helpers::qmp_socket($vmid, 1);
3851         push @$devices, '-chardev', "socket,path=$qgasocket,server,nowait,id=qga0";
3852
3853         if (!$guest_agent->{type} || $guest_agent->{type} eq 'virtio') {
3854             my $pciaddr = print_pci_addr("qga0", $bridges, $arch, $machine_type);
3855             push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3856             push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3857         } elsif ($guest_agent->{type} eq 'isa') {
3858             push @$devices, '-device', "isa-serial,chardev=qga0";
3859         }
3860     }
3861
3862     my $spice_port;
3863
3864     if ($qxlnum) {
3865         if ($qxlnum > 1) {
3866             if ($winversion){
3867                 for(my $i = 1; $i < $qxlnum; $i++){
3868                     push @$devices, '-device', print_vga_device($conf, $vga, $arch, $machine_version, $machine_type, $i, $qxlnum, $bridges);
3869                 }
3870             } else {
3871                 # assume other OS works like Linux
3872                 my ($ram, $vram) = ("134217728", "67108864");
3873                 if ($vga->{memory}) {
3874                     $ram = PVE::Tools::convert_size($qxlnum*4*$vga->{memory}, 'mb' => 'b');
3875                     $vram = PVE::Tools::convert_size($qxlnum*2*$vga->{memory}, 'mb' => 'b');
3876                 }
3877                 push @$cmd, '-global', "qxl-vga.ram_size=$ram";
3878                 push @$cmd, '-global', "qxl-vga.vram_size=$vram";
3879             }
3880         }
3881
3882         my $pciaddr = print_pci_addr("spice", $bridges, $arch, $machine_type);
3883
3884         my $nodename = PVE::INotify::nodename();
3885         my $pfamily = PVE::Tools::get_host_address_family($nodename);
3886         my @nodeaddrs = PVE::Tools::getaddrinfo_all('localhost', family => $pfamily);
3887         die "failed to get an ip address of type $pfamily for 'localhost'\n" if !@nodeaddrs;
3888
3889         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3890         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3891         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3892
3893         my $localhost = PVE::Network::addr_to_ip($nodeaddrs[0]->{addr});
3894         $spice_port = PVE::Tools::next_spice_port($pfamily, $localhost);
3895
3896         my $spice_enhancement = PVE::JSONSchema::parse_property_string($spice_enhancements_fmt, $conf->{spice_enhancements} // '');
3897         if ($spice_enhancement->{foldersharing}) {
3898             push @$devices, '-chardev', "spiceport,id=foldershare,name=org.spice-space.webdav.0";
3899             push @$devices, '-device', "virtserialport,chardev=foldershare,name=org.spice-space.webdav.0";
3900         }
3901
3902         my $spice_opts = "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";
3903         $spice_opts .= ",streaming-video=$spice_enhancement->{videostreaming}" if $spice_enhancement->{videostreaming};
3904         push @$devices, '-spice', "$spice_opts";
3905     }
3906
3907     # enable balloon by default, unless explicitly disabled
3908     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3909         $pciaddr = print_pci_addr("balloon0", $bridges, $arch, $machine_type);
3910         push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
3911     }
3912
3913     if ($conf->{watchdog}) {
3914         my $wdopts = parse_watchdog($conf->{watchdog});
3915         $pciaddr = print_pci_addr("watchdog", $bridges, $arch, $machine_type);
3916         my $watchdog = $wdopts->{model} || 'i6300esb';
3917         push @$devices, '-device', "$watchdog$pciaddr";
3918         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3919     }
3920
3921     my $vollist = [];
3922     my $scsicontroller = {};
3923     my $ahcicontroller = {};
3924     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3925
3926     # Add iscsi initiator name if available
3927     if (my $initiator = get_initiator_name()) {
3928         push @$devices, '-iscsi', "initiator-name=$initiator";
3929     }
3930
3931     foreach_drive($conf, sub {
3932         my ($ds, $drive) = @_;
3933
3934         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3935             push @$vollist, $drive->{file};
3936         }
3937
3938         # ignore efidisk here, already added in bios/fw handling code above
3939         return if $drive->{interface} eq 'efidisk';
3940
3941         $use_virtio = 1 if $ds =~ m/^virtio/;
3942
3943         if (drive_is_cdrom ($drive)) {
3944             if ($bootindex_hash->{d}) {
3945                 $drive->{bootindex} = $bootindex_hash->{d};
3946                 $bootindex_hash->{d} += 1;
3947             }
3948         } else {
3949             if ($bootindex_hash->{c}) {
3950                 $drive->{bootindex} = $bootindex_hash->{c} if $conf->{bootdisk} && ($conf->{bootdisk} eq $ds);
3951                 $bootindex_hash->{c} += 1;
3952             }
3953         }
3954
3955         if($drive->{interface} eq 'virtio'){
3956            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3957         }
3958
3959         if ($drive->{interface} eq 'scsi') {
3960
3961             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
3962
3963             $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges, $arch, $machine_type);
3964             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
3965
3966             my $iothread = '';
3967             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
3968                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3969                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3970             } elsif ($drive->{iothread}) {
3971                 warn "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n";
3972             }
3973
3974             my $queues = '';
3975             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
3976                 $queues = ",num_queues=$drive->{queues}";
3977             }
3978
3979             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues" if !$scsicontroller->{$controller};
3980             $scsicontroller->{$controller}=1;
3981         }
3982
3983         if ($drive->{interface} eq 'sata') {
3984            my $controller = int($drive->{index} / $MAX_SATA_DISKS);
3985            $pciaddr = print_pci_addr("ahci$controller", $bridges, $arch, $machine_type);
3986            push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr" if !$ahcicontroller->{$controller};
3987            $ahcicontroller->{$controller}=1;
3988         }
3989
3990         my $drive_cmd = print_drive_full($storecfg, $vmid, $drive);
3991         push @$devices, '-drive',$drive_cmd;
3992         push @$devices, '-device', print_drivedevice_full($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type);
3993     });
3994
3995     for (my $i = 0; $i < $MAX_NETS; $i++) {
3996          next if !$conf->{"net$i"};
3997          my $d = parse_net($conf->{"net$i"});
3998          next if !$d;
3999
4000          $use_virtio = 1 if $d->{model} eq 'virtio';
4001
4002          if ($bootindex_hash->{n}) {
4003             $d->{bootindex} = $bootindex_hash->{n};
4004             $bootindex_hash->{n} += 1;
4005          }
4006
4007          my $netdevfull = print_netdev_full($vmid, $conf, $arch, $d, "net$i");
4008          push @$devices, '-netdev', $netdevfull;
4009
4010          my $netdevicefull = print_netdevice_full($vmid, $conf, $d, "net$i", $bridges, $use_old_bios_files, $arch, $machine_type);
4011          push @$devices, '-device', $netdevicefull;
4012     }
4013
4014     if ($conf->{ivshmem}) {
4015         my $ivshmem = PVE::JSONSchema::parse_property_string($ivshmem_fmt, $conf->{ivshmem});
4016
4017         my $bus;
4018         if ($q35) {
4019             $bus = print_pcie_addr("ivshmem");
4020         } else {
4021             $bus = print_pci_addr("ivshmem", $bridges, $arch, $machine_type);
4022         }
4023
4024         my $ivshmem_name = $ivshmem->{name} // $vmid;
4025         my $path = '/dev/shm/pve-shm-' . $ivshmem_name;
4026
4027         push @$devices, '-device', "ivshmem-plain,memdev=ivshmem$bus,";
4028         push @$devices, '-object', "memory-backend-file,id=ivshmem,share=on,mem-path=$path,size=$ivshmem->{size}M";
4029     }
4030
4031     if (!$q35) {
4032         # add pci bridges
4033         if (min_version($machine_version, 2, 3)) {
4034            $bridges->{1} = 1;
4035            $bridges->{2} = 1;
4036         }
4037
4038         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
4039
4040         for my $k (sort {$b cmp $a} keys %$bridges) {
4041             $pciaddr = print_pci_addr("pci.$k", undef, $arch, $machine_type);
4042             unshift @$devices, '-device', "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr" if $k > 0;
4043         }
4044     }
4045
4046     push @$cmd, @$devices;
4047     push @$cmd, '-rtc', join(',', @$rtcFlags)
4048         if scalar(@$rtcFlags);
4049     push @$cmd, '-machine', join(',', @$machineFlags)
4050         if scalar(@$machineFlags);
4051     push @$cmd, '-global', join(',', @$globalFlags)
4052         if scalar(@$globalFlags);
4053
4054     if (my $vmstate = $conf->{vmstate}) {
4055         my $statepath = PVE::Storage::path($storecfg, $vmstate);
4056         push @$vollist, $vmstate;
4057         push @$cmd, '-loadstate', $statepath;
4058     }
4059
4060     # add custom args
4061     if ($conf->{args}) {
4062         my $aa = PVE::Tools::split_args($conf->{args});
4063         push @$cmd, @$aa;
4064     }
4065
4066     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
4067 }
4068
4069 sub spice_port {
4070     my ($vmid) = @_;
4071
4072     my $res = mon_cmd($vmid, 'query-spice');
4073
4074     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
4075 }
4076
4077 sub vm_devices_list {
4078     my ($vmid) = @_;
4079
4080     my $res = mon_cmd($vmid, 'query-pci');
4081     my $devices_to_check = [];
4082     my $devices = {};
4083     foreach my $pcibus (@$res) {
4084         push @$devices_to_check, @{$pcibus->{devices}},
4085     }
4086
4087     while (@$devices_to_check) {
4088         my $to_check = [];
4089         for my $d (@$devices_to_check) {
4090             $devices->{$d->{'qdev_id'}} = 1 if $d->{'qdev_id'};
4091             next if !$d->{'pci_bridge'};
4092
4093             $devices->{$d->{'qdev_id'}} += scalar(@{$d->{'pci_bridge'}->{devices}});
4094             push @$to_check, @{$d->{'pci_bridge'}->{devices}};
4095         }
4096         $devices_to_check = $to_check;
4097     }
4098
4099     my $resblock = mon_cmd($vmid, 'query-block');
4100     foreach my $block (@$resblock) {
4101         if($block->{device} =~ m/^drive-(\S+)/){
4102                 $devices->{$1} = 1;
4103         }
4104     }
4105
4106     my $resmice = mon_cmd($vmid, 'query-mice');
4107     foreach my $mice (@$resmice) {
4108         if ($mice->{name} eq 'QEMU HID Tablet') {
4109             $devices->{tablet} = 1;
4110             last;
4111         }
4112     }
4113
4114     # for usb devices there is no query-usb
4115     # but we can iterate over the entries in
4116     # qom-list path=/machine/peripheral
4117     my $resperipheral = mon_cmd($vmid, 'qom-list', path => '/machine/peripheral');
4118     foreach my $per (@$resperipheral) {
4119         if ($per->{name} =~ m/^usb\d+$/) {
4120             $devices->{$per->{name}} = 1;
4121         }
4122     }
4123
4124     return $devices;
4125 }
4126
4127 sub vm_deviceplug {
4128     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4129
4130     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
4131
4132     my $devices_list = vm_devices_list($vmid);
4133     return 1 if defined($devices_list->{$deviceid});
4134
4135     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid, $arch, $machine_type); # add PCI bridge if we need it for the device
4136
4137     if ($deviceid eq 'tablet') {
4138
4139         qemu_deviceadd($vmid, print_tabletdevice_full($conf, $arch));
4140
4141     } elsif ($deviceid eq 'keyboard') {
4142
4143         qemu_deviceadd($vmid, print_keyboarddevice_full($conf, $arch));
4144
4145     } elsif ($deviceid =~ m/^usb(\d+)$/) {
4146
4147         die "usb hotplug currently not reliable\n";
4148         # since we can't reliably hot unplug all added usb devices
4149         # and usb passthrough disables live migration
4150         # we disable usb hotplugging for now
4151         qemu_deviceadd($vmid, PVE::QemuServer::USB::print_usbdevice_full($conf, $deviceid, $device));
4152
4153     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4154
4155         qemu_iothread_add($vmid, $deviceid, $device);
4156
4157         qemu_driveadd($storecfg, $vmid, $device);
4158         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, $arch, $machine_type);
4159
4160         qemu_deviceadd($vmid, $devicefull);
4161         eval { qemu_deviceaddverify($vmid, $deviceid); };
4162         if (my $err = $@) {
4163             eval { qemu_drivedel($vmid, $deviceid); };
4164             warn $@ if $@;
4165             die $err;
4166         }
4167
4168     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4169
4170
4171         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
4172         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
4173         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
4174
4175         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
4176
4177         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
4178             qemu_iothread_add($vmid, $deviceid, $device);
4179             $devicefull .= ",iothread=iothread-$deviceid";
4180         }
4181
4182         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
4183             $devicefull .= ",num_queues=$device->{queues}";
4184         }
4185
4186         qemu_deviceadd($vmid, $devicefull);
4187         qemu_deviceaddverify($vmid, $deviceid);
4188
4189     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4190
4191         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device, $arch, $machine_type);
4192         qemu_driveadd($storecfg, $vmid, $device);
4193
4194         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, $arch, $machine_type);
4195         eval { qemu_deviceadd($vmid, $devicefull); };
4196         if (my $err = $@) {
4197             eval { qemu_drivedel($vmid, $deviceid); };
4198             warn $@ if $@;
4199             die $err;
4200         }
4201
4202     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4203
4204         return undef if !qemu_netdevadd($vmid, $conf, $arch, $device, $deviceid);
4205
4206         my $machine_type = PVE::QemuServer::Machine::qemu_machine_pxe($vmid, $conf);
4207         my $use_old_bios_files = undef;
4208         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
4209
4210         my $netdevicefull = print_netdevice_full($vmid, $conf, $device, $deviceid, undef, $use_old_bios_files, $arch, $machine_type);
4211         qemu_deviceadd($vmid, $netdevicefull);
4212         eval {
4213             qemu_deviceaddverify($vmid, $deviceid);
4214             qemu_set_link_status($vmid, $deviceid, !$device->{link_down});
4215         };
4216         if (my $err = $@) {
4217             eval { qemu_netdevdel($vmid, $deviceid); };
4218             warn $@ if $@;
4219             die $err;
4220         }
4221
4222     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
4223
4224         my $bridgeid = $2;
4225         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
4226         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
4227
4228         qemu_deviceadd($vmid, $devicefull);
4229         qemu_deviceaddverify($vmid, $deviceid);
4230
4231     } else {
4232         die "can't hotplug device '$deviceid'\n";
4233     }
4234
4235     return 1;
4236 }
4237
4238 # fixme: this should raise exceptions on error!
4239 sub vm_deviceunplug {
4240     my ($vmid, $conf, $deviceid) = @_;
4241
4242     my $devices_list = vm_devices_list($vmid);
4243     return 1 if !defined($devices_list->{$deviceid});
4244
4245     die "can't unplug bootdisk" if $conf->{bootdisk} && $conf->{bootdisk} eq $deviceid;
4246
4247     if ($deviceid eq 'tablet' || $deviceid eq 'keyboard') {
4248
4249         qemu_devicedel($vmid, $deviceid);
4250
4251     } elsif ($deviceid =~ m/^usb\d+$/) {
4252
4253         die "usb hotplug currently not reliable\n";
4254         # when unplugging usb devices this way,
4255         # there may be remaining usb controllers/hubs
4256         # so we disable it for now
4257         qemu_devicedel($vmid, $deviceid);
4258         qemu_devicedelverify($vmid, $deviceid);
4259
4260     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4261
4262         qemu_devicedel($vmid, $deviceid);
4263         qemu_devicedelverify($vmid, $deviceid);
4264         qemu_drivedel($vmid, $deviceid);
4265         qemu_iothread_del($conf, $vmid, $deviceid);
4266
4267     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4268
4269         qemu_devicedel($vmid, $deviceid);
4270         qemu_devicedelverify($vmid, $deviceid);
4271         qemu_iothread_del($conf, $vmid, $deviceid);
4272
4273     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4274
4275         qemu_devicedel($vmid, $deviceid);
4276         qemu_drivedel($vmid, $deviceid);
4277         qemu_deletescsihw($conf, $vmid, $deviceid);
4278
4279     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4280
4281         qemu_devicedel($vmid, $deviceid);
4282         qemu_devicedelverify($vmid, $deviceid);
4283         qemu_netdevdel($vmid, $deviceid);
4284
4285     } else {
4286         die "can't unplug device '$deviceid'\n";
4287     }
4288
4289     return 1;
4290 }
4291
4292 sub qemu_deviceadd {
4293     my ($vmid, $devicefull) = @_;
4294
4295     $devicefull = "driver=".$devicefull;
4296     my %options =  split(/[=,]/, $devicefull);
4297
4298     mon_cmd($vmid, "device_add" , %options);
4299 }
4300
4301 sub qemu_devicedel {
4302     my ($vmid, $deviceid) = @_;
4303
4304     my $ret = mon_cmd($vmid, "device_del", id => $deviceid);
4305 }
4306
4307 sub qemu_iothread_add {
4308     my($vmid, $deviceid, $device) = @_;
4309
4310     if ($device->{iothread}) {
4311         my $iothreads = vm_iothreads_list($vmid);
4312         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
4313     }
4314 }
4315
4316 sub qemu_iothread_del {
4317     my($conf, $vmid, $deviceid) = @_;
4318
4319     my $confid = $deviceid;
4320     if ($deviceid =~ m/^(?:virtioscsi|scsihw)(\d+)$/) {
4321         $confid = 'scsi' . $1;
4322     }
4323     my $device = parse_drive($confid, $conf->{$confid});
4324     if ($device->{iothread}) {
4325         my $iothreads = vm_iothreads_list($vmid);
4326         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
4327     }
4328 }
4329
4330 sub qemu_objectadd {
4331     my($vmid, $objectid, $qomtype) = @_;
4332
4333     mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
4334
4335     return 1;
4336 }
4337
4338 sub qemu_objectdel {
4339     my($vmid, $objectid) = @_;
4340
4341     mon_cmd($vmid, "object-del", id => $objectid);
4342
4343     return 1;
4344 }
4345
4346 sub qemu_driveadd {
4347     my ($storecfg, $vmid, $device) = @_;
4348
4349     my $drive = print_drive_full($storecfg, $vmid, $device);
4350     $drive =~ s/\\/\\\\/g;
4351     my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_add auto \"$drive\"");
4352
4353     # If the command succeeds qemu prints: "OK"
4354     return 1 if $ret =~ m/OK/s;
4355
4356     die "adding drive failed: $ret\n";
4357 }
4358
4359 sub qemu_drivedel {
4360     my($vmid, $deviceid) = @_;
4361
4362     my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_del drive-$deviceid");
4363     $ret =~ s/^\s+//;
4364
4365     return 1 if $ret eq "";
4366
4367     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
4368     return 1 if $ret =~ m/Device \'.*?\' not found/s;
4369
4370     die "deleting drive $deviceid failed : $ret\n";
4371 }
4372
4373 sub qemu_deviceaddverify {
4374     my ($vmid, $deviceid) = @_;
4375
4376     for (my $i = 0; $i <= 5; $i++) {
4377          my $devices_list = vm_devices_list($vmid);
4378          return 1 if defined($devices_list->{$deviceid});
4379          sleep 1;
4380     }
4381
4382     die "error on hotplug device '$deviceid'\n";
4383 }
4384
4385
4386 sub qemu_devicedelverify {
4387     my ($vmid, $deviceid) = @_;
4388
4389     # need to verify that the device is correctly removed as device_del
4390     # is async and empty return is not reliable
4391
4392     for (my $i = 0; $i <= 5; $i++) {
4393          my $devices_list = vm_devices_list($vmid);
4394          return 1 if !defined($devices_list->{$deviceid});
4395          sleep 1;
4396     }
4397
4398     die "error on hot-unplugging device '$deviceid'\n";
4399 }
4400
4401 sub qemu_findorcreatescsihw {
4402     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4403
4404     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4405
4406     my $scsihwid="$controller_prefix$controller";
4407     my $devices_list = vm_devices_list($vmid);
4408
4409     if(!defined($devices_list->{$scsihwid})) {
4410         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device, $arch, $machine_type);
4411     }
4412
4413     return 1;
4414 }
4415
4416 sub qemu_deletescsihw {
4417     my ($conf, $vmid, $opt) = @_;
4418
4419     my $device = parse_drive($opt, $conf->{$opt});
4420
4421     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
4422         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
4423         return 1;
4424     }
4425
4426     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4427
4428     my $devices_list = vm_devices_list($vmid);
4429     foreach my $opt (keys %{$devices_list}) {
4430         if (PVE::QemuServer::is_valid_drivename($opt)) {
4431             my $drive = PVE::QemuServer::parse_drive($opt, $conf->{$opt});
4432             if($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
4433                 return 1;
4434             }
4435         }
4436     }
4437
4438     my $scsihwid="scsihw$controller";
4439
4440     vm_deviceunplug($vmid, $conf, $scsihwid);
4441
4442     return 1;
4443 }
4444
4445 sub qemu_add_pci_bridge {
4446     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4447
4448     my $bridges = {};
4449
4450     my $bridgeid;
4451
4452     print_pci_addr($device, $bridges, $arch, $machine_type);
4453
4454     while (my ($k, $v) = each %$bridges) {
4455         $bridgeid = $k;
4456     }
4457     return 1 if !defined($bridgeid) || $bridgeid < 1;
4458
4459     my $bridge = "pci.$bridgeid";
4460     my $devices_list = vm_devices_list($vmid);
4461
4462     if (!defined($devices_list->{$bridge})) {
4463         vm_deviceplug($storecfg, $conf, $vmid, $bridge, $arch, $machine_type);
4464     }
4465
4466     return 1;
4467 }
4468
4469 sub qemu_set_link_status {
4470     my ($vmid, $device, $up) = @_;
4471
4472     mon_cmd($vmid, "set_link", name => $device,
4473                up => $up ? JSON::true : JSON::false);
4474 }
4475
4476 sub qemu_netdevadd {
4477     my ($vmid, $conf, $arch, $device, $deviceid) = @_;
4478
4479     my $netdev = print_netdev_full($vmid, $conf, $arch, $device, $deviceid, 1);
4480     my %options =  split(/[=,]/, $netdev);
4481
4482     mon_cmd($vmid, "netdev_add",  %options);
4483     return 1;
4484 }
4485
4486 sub qemu_netdevdel {
4487     my ($vmid, $deviceid) = @_;
4488
4489     mon_cmd($vmid, "netdev_del", id => $deviceid);
4490 }
4491
4492 sub qemu_usb_hotplug {
4493     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4494
4495     return if !$device;
4496
4497     # remove the old one first
4498     vm_deviceunplug($vmid, $conf, $deviceid);
4499
4500     # check if xhci controller is necessary and available
4501     if ($device->{usb3}) {
4502
4503         my $devicelist = vm_devices_list($vmid);
4504
4505         if (!$devicelist->{xhci}) {
4506             my $pciaddr = print_pci_addr("xhci", undef, $arch, $machine_type);
4507             qemu_deviceadd($vmid, "nec-usb-xhci,id=xhci$pciaddr");
4508         }
4509     }
4510     my $d = parse_usb_device($device->{host});
4511     $d->{usb3} = $device->{usb3};
4512
4513     # add the new one
4514     vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $d, $arch, $machine_type);
4515 }
4516
4517 sub qemu_cpu_hotplug {
4518     my ($vmid, $conf, $vcpus) = @_;
4519
4520     my $machine_type = PVE::QemuServer::Machine::get_current_qemu_machine($vmid);
4521
4522     my $sockets = 1;
4523     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
4524     $sockets = $conf->{sockets} if  $conf->{sockets};
4525     my $cores = $conf->{cores} || 1;
4526     my $maxcpus = $sockets * $cores;
4527
4528     $vcpus = $maxcpus if !$vcpus;
4529
4530     die "you can't add more vcpus than maxcpus\n"
4531         if $vcpus > $maxcpus;
4532
4533     my $currentvcpus = $conf->{vcpus} || $maxcpus;
4534
4535     if ($vcpus < $currentvcpus) {
4536
4537         if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4538
4539             for (my $i = $currentvcpus; $i > $vcpus; $i--) {
4540                 qemu_devicedel($vmid, "cpu$i");
4541                 my $retry = 0;
4542                 my $currentrunningvcpus = undef;
4543                 while (1) {
4544                     $currentrunningvcpus = mon_cmd($vmid, "query-cpus");
4545                     last if scalar(@{$currentrunningvcpus}) == $i-1;
4546                     raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
4547                     $retry++;
4548                     sleep 1;
4549                 }
4550                 #update conf after each succesfull cpu unplug
4551                 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4552                 PVE::QemuConfig->write_config($vmid, $conf);
4553             }
4554         } else {
4555             die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
4556         }
4557
4558         return;
4559     }
4560
4561     my $currentrunningvcpus = mon_cmd($vmid, "query-cpus");
4562     die "vcpus in running vm does not match its configuration\n"
4563         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
4564
4565     if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4566
4567         for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
4568             my $cpustr = print_cpu_device($conf, $i);
4569             qemu_deviceadd($vmid, $cpustr);
4570
4571             my $retry = 0;
4572             my $currentrunningvcpus = undef;
4573             while (1) {
4574                 $currentrunningvcpus = mon_cmd($vmid, "query-cpus");
4575                 last if scalar(@{$currentrunningvcpus}) == $i;
4576                 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
4577                 sleep 1;
4578                 $retry++;
4579             }
4580             #update conf after each succesfull cpu hotplug
4581             $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4582             PVE::QemuConfig->write_config($vmid, $conf);
4583         }
4584     } else {
4585
4586         for (my $i = $currentvcpus; $i < $vcpus; $i++) {
4587             mon_cmd($vmid, "cpu-add", id => int($i));
4588         }
4589     }
4590 }
4591
4592 sub qemu_block_set_io_throttle {
4593     my ($vmid, $deviceid,
4594         $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
4595         $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max,
4596         $bps_max_length, $bps_rd_max_length, $bps_wr_max_length,
4597         $iops_max_length, $iops_rd_max_length, $iops_wr_max_length) = @_;
4598
4599     return if !check_running($vmid) ;
4600
4601     mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
4602         bps => int($bps),
4603         bps_rd => int($bps_rd),
4604         bps_wr => int($bps_wr),
4605         iops => int($iops),
4606         iops_rd => int($iops_rd),
4607         iops_wr => int($iops_wr),
4608         bps_max => int($bps_max),
4609         bps_rd_max => int($bps_rd_max),
4610         bps_wr_max => int($bps_wr_max),
4611         iops_max => int($iops_max),
4612         iops_rd_max => int($iops_rd_max),
4613         iops_wr_max => int($iops_wr_max),
4614         bps_max_length => int($bps_max_length),
4615         bps_rd_max_length => int($bps_rd_max_length),
4616         bps_wr_max_length => int($bps_wr_max_length),
4617         iops_max_length => int($iops_max_length),
4618         iops_rd_max_length => int($iops_rd_max_length),
4619         iops_wr_max_length => int($iops_wr_max_length),
4620     );
4621
4622 }
4623
4624 # old code, only used to shutdown old VM after update
4625 sub __read_avail {
4626     my ($fh, $timeout) = @_;
4627
4628     my $sel = new IO::Select;
4629     $sel->add($fh);
4630
4631     my $res = '';
4632     my $buf;
4633
4634     my @ready;
4635     while (scalar (@ready = $sel->can_read($timeout))) {
4636         my $count;
4637         if ($count = $fh->sysread($buf, 8192)) {
4638             if ($buf =~ /^(.*)\(qemu\) $/s) {
4639                 $res .= $1;
4640                 last;
4641             } else {
4642                 $res .= $buf;
4643             }
4644         } else {
4645             if (!defined($count)) {
4646                 die "$!\n";
4647             }
4648             last;
4649         }
4650     }
4651
4652     die "monitor read timeout\n" if !scalar(@ready);
4653
4654     return $res;
4655 }
4656
4657 sub qemu_block_resize {
4658     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4659
4660     my $running = check_running($vmid);
4661
4662     $size = 0 if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4663
4664     return if !$running;
4665
4666     mon_cmd($vmid, "block_resize", device => $deviceid, size => int($size));
4667
4668 }
4669
4670 sub qemu_volume_snapshot {
4671     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4672
4673     my $running = check_running($vmid);
4674
4675     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4676         mon_cmd($vmid, 'blockdev-snapshot-internal-sync', device => $deviceid, name => $snap);
4677     } else {
4678         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4679     }
4680 }
4681
4682 sub qemu_volume_snapshot_delete {
4683     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4684
4685     my $running = check_running($vmid);
4686
4687     if($running) {
4688
4689         $running = undef;
4690         my $conf = PVE::QemuConfig->load_config($vmid);
4691         foreach_drive($conf, sub {
4692             my ($ds, $drive) = @_;
4693             $running = 1 if $drive->{file} eq $volid;
4694         });
4695     }
4696
4697     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4698         mon_cmd($vmid, 'blockdev-snapshot-delete-internal-sync', device => $deviceid, name => $snap);
4699     } else {
4700         PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4701     }
4702 }
4703
4704 sub set_migration_caps {
4705     my ($vmid) = @_;
4706
4707     my $cap_ref = [];
4708
4709     my $enabled_cap = {
4710         "auto-converge" => 1,
4711         "xbzrle" => 1,
4712         "x-rdma-pin-all" => 0,
4713         "zero-blocks" => 0,
4714         "compress" => 0
4715     };
4716
4717     my $supported_capabilities = mon_cmd($vmid, "query-migrate-capabilities");
4718
4719     for my $supported_capability (@$supported_capabilities) {
4720         push @$cap_ref, {
4721             capability => $supported_capability->{capability},
4722             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4723         };
4724     }
4725
4726     mon_cmd($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4727 }
4728
4729 my $fast_plug_option = {
4730     'lock' => 1,
4731     'name' => 1,
4732     'onboot' => 1,
4733     'shares' => 1,
4734     'startup' => 1,
4735     'description' => 1,
4736     'protection' => 1,
4737     'vmstatestorage' => 1,
4738     'hookscript' => 1,
4739     'tags' => 1,
4740 };
4741
4742 # hotplug changes in [PENDING]
4743 # $selection hash can be used to only apply specified options, for
4744 # example: { cores => 1 } (only apply changed 'cores')
4745 # $errors ref is used to return error messages
4746 sub vmconfig_hotplug_pending {
4747     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4748
4749     my $defaults = load_defaults();
4750     my $arch = get_vm_arch($conf);
4751     my $machine_type = get_vm_machine($conf, undef, $arch);
4752
4753     # commit values which do not have any impact on running VM first
4754     # Note: those option cannot raise errors, we we do not care about
4755     # $selection and always apply them.
4756
4757     my $add_error = sub {
4758         my ($opt, $msg) = @_;
4759         $errors->{$opt} = "hotplug problem - $msg";
4760     };
4761
4762     my $changes = 0;
4763     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4764         if ($fast_plug_option->{$opt}) {
4765             $conf->{$opt} = $conf->{pending}->{$opt};
4766             delete $conf->{pending}->{$opt};
4767             $changes = 1;
4768         }
4769     }
4770
4771     if ($changes) {
4772         PVE::QemuConfig->write_config($vmid, $conf);
4773         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4774     }
4775
4776     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4777
4778     my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4779     foreach my $opt (sort keys %$pending_delete_hash) {
4780         next if $selection && !$selection->{$opt};
4781         my $force = $pending_delete_hash->{$opt}->{force};
4782         eval {
4783             if ($opt eq 'hotplug') {
4784                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
4785             } elsif ($opt eq 'tablet') {
4786                 die "skip\n" if !$hotplug_features->{usb};
4787                 if ($defaults->{tablet}) {
4788                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4789                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4790                         if $arch eq 'aarch64';
4791                 } else {
4792                     vm_deviceunplug($vmid, $conf, 'tablet');
4793                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4794                 }
4795             } elsif ($opt =~ m/^usb\d+/) {
4796                 die "skip\n";
4797                 # since we cannot reliably hot unplug usb devices
4798                 # we are disabling it
4799                 die "skip\n" if !$hotplug_features->{usb} || $conf->{$opt} =~ m/spice/i;
4800                 vm_deviceunplug($vmid, $conf, $opt);
4801             } elsif ($opt eq 'vcpus') {
4802                 die "skip\n" if !$hotplug_features->{cpu};
4803                 qemu_cpu_hotplug($vmid, $conf, undef);
4804             } elsif ($opt eq 'balloon') {
4805                 # enable balloon device is not hotpluggable
4806                 die "skip\n" if defined($conf->{balloon}) && $conf->{balloon} == 0;
4807                 # here we reset the ballooning value to memory
4808                 my $balloon = $conf->{memory} || $defaults->{memory};
4809                 mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4810             } elsif ($fast_plug_option->{$opt}) {
4811                 # do nothing
4812             } elsif ($opt =~ m/^net(\d+)$/) {
4813                 die "skip\n" if !$hotplug_features->{network};
4814                 vm_deviceunplug($vmid, $conf, $opt);
4815             } elsif (is_valid_drivename($opt)) {
4816                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4817                 vm_deviceunplug($vmid, $conf, $opt);
4818                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4819             } elsif ($opt =~ m/^memory$/) {
4820                 die "skip\n" if !$hotplug_features->{memory};
4821                 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
4822             } elsif ($opt eq 'cpuunits') {
4823                 cgroups_write("cpu", $vmid, "cpu.shares", $defaults->{cpuunits});
4824             } elsif ($opt eq 'cpulimit') {
4825                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", -1);
4826             } else {
4827                 die "skip\n";
4828             }
4829         };
4830         if (my $err = $@) {
4831             &$add_error($opt, $err) if $err ne "skip\n";
4832         } else {
4833             # save new config if hotplug was successful
4834             delete $conf->{$opt};
4835             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4836             PVE::QemuConfig->write_config($vmid, $conf);
4837             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4838         }
4839     }
4840
4841     my ($apply_pending_cloudinit, $apply_pending_cloudinit_done);
4842     $apply_pending_cloudinit = sub {
4843         return if $apply_pending_cloudinit_done; # once is enough
4844         $apply_pending_cloudinit_done = 1; # once is enough
4845
4846         my ($key, $value) = @_;
4847
4848         my @cloudinit_opts = keys %$confdesc_cloudinit;
4849         foreach my $opt (keys %{$conf->{pending}}) {
4850             next if !grep { $_ eq $opt } @cloudinit_opts;
4851             $conf->{$opt} = delete $conf->{pending}->{$opt};
4852         }
4853
4854         my $new_conf = { %$conf };
4855         $new_conf->{$key} = $value;
4856         PVE::QemuServer::Cloudinit::generate_cloudinitconfig($new_conf, $vmid);
4857     };
4858
4859     foreach my $opt (keys %{$conf->{pending}}) {
4860         next if $selection && !$selection->{$opt};
4861         my $value = $conf->{pending}->{$opt};
4862         eval {
4863             if ($opt eq 'hotplug') {
4864                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
4865             } elsif ($opt eq 'tablet') {
4866                 die "skip\n" if !$hotplug_features->{usb};
4867                 if ($value == 1) {
4868                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4869                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4870                         if $arch eq 'aarch64';
4871                 } elsif ($value == 0) {
4872                     vm_deviceunplug($vmid, $conf, 'tablet');
4873                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4874                 }
4875             } elsif ($opt =~ m/^usb\d+$/) {
4876                 die "skip\n";
4877                 # since we cannot reliably hot unplug usb devices
4878                 # we are disabling it
4879                 die "skip\n" if !$hotplug_features->{usb} || $value =~ m/spice/i;
4880                 my $d = eval { PVE::JSONSchema::parse_property_string($usbdesc->{format}, $value) };
4881                 die "skip\n" if !$d;
4882                 qemu_usb_hotplug($storecfg, $conf, $vmid, $opt, $d, $arch, $machine_type);
4883             } elsif ($opt eq 'vcpus') {
4884                 die "skip\n" if !$hotplug_features->{cpu};
4885                 qemu_cpu_hotplug($vmid, $conf, $value);
4886             } elsif ($opt eq 'balloon') {
4887                 # enable/disable balloning device is not hotpluggable
4888                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
4889                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
4890                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
4891
4892                 # allow manual ballooning if shares is set to zero
4893                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
4894                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
4895                     mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4896                 }
4897             } elsif ($opt =~ m/^net(\d+)$/) {
4898                 # some changes can be done without hotplug
4899                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
4900                                     $vmid, $opt, $value, $arch, $machine_type);
4901             } elsif (is_valid_drivename($opt)) {
4902                 # some changes can be done without hotplug
4903                 my $drive = parse_drive($opt, $value);
4904                 if (drive_is_cloudinit($drive)) {
4905                     &$apply_pending_cloudinit($opt, $value);
4906                 }
4907                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
4908                                      $vmid, $opt, $value, 1, $arch, $machine_type);
4909             } elsif ($opt =~ m/^memory$/) { #dimms
4910                 die "skip\n" if !$hotplug_features->{memory};
4911                 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
4912             } elsif ($opt eq 'cpuunits') {
4913                 cgroups_write("cpu", $vmid, "cpu.shares", $conf->{pending}->{$opt});
4914             } elsif ($opt eq 'cpulimit') {
4915                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
4916                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", $cpulimit);
4917             } else {
4918                 die "skip\n";  # skip non-hot-pluggable options
4919             }
4920         };
4921         if (my $err = $@) {
4922             &$add_error($opt, $err) if $err ne "skip\n";
4923         } else {
4924             # save new config if hotplug was successful
4925             $conf->{$opt} = $value;
4926             delete $conf->{pending}->{$opt};
4927             PVE::QemuConfig->write_config($vmid, $conf);
4928             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4929         }
4930     }
4931 }
4932
4933 sub try_deallocate_drive {
4934     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
4935
4936     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
4937         my $volid = $drive->{file};
4938         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
4939             my $sid = PVE::Storage::parse_volume_id($volid);
4940             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
4941
4942             # check if the disk is really unused
4943             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
4944                 if is_volume_in_use($storecfg, $conf, $key, $volid);
4945             PVE::Storage::vdisk_free($storecfg, $volid);
4946             return 1;
4947         } else {
4948             # If vm is not owner of this disk remove from config
4949             return 1;
4950         }
4951     }
4952
4953     return undef;
4954 }
4955
4956 sub vmconfig_delete_or_detach_drive {
4957     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
4958
4959     my $drive = parse_drive($opt, $conf->{$opt});
4960
4961     my $rpcenv = PVE::RPCEnvironment::get();
4962     my $authuser = $rpcenv->get_user();
4963
4964     if ($force) {
4965         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
4966         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
4967     } else {
4968         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
4969     }
4970 }
4971
4972
4973
4974 sub vmconfig_apply_pending {
4975     my ($vmid, $conf, $storecfg) = @_;
4976
4977     # cold plug
4978
4979     my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4980     foreach my $opt (sort keys %$pending_delete_hash) {
4981         die "internal error" if $opt =~ m/^unused/;
4982         my $force = $pending_delete_hash->{$opt}->{force};
4983         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4984         if (!defined($conf->{$opt})) {
4985             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4986             PVE::QemuConfig->write_config($vmid, $conf);
4987         } elsif (is_valid_drivename($opt)) {
4988             vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4989             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4990             delete $conf->{$opt};
4991             PVE::QemuConfig->write_config($vmid, $conf);
4992         } else {
4993             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4994             delete $conf->{$opt};
4995             PVE::QemuConfig->write_config($vmid, $conf);
4996         }
4997     }
4998
4999     $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5000
5001     foreach my $opt (keys %{$conf->{pending}}) { # add/change
5002         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5003
5004         if (defined($conf->{$opt}) && ($conf->{$opt} eq $conf->{pending}->{$opt})) {
5005             # skip if nothing changed
5006         } elsif (is_valid_drivename($opt)) {
5007             vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
5008                 if defined($conf->{$opt});
5009             $conf->{$opt} = $conf->{pending}->{$opt};
5010         } else {
5011             $conf->{$opt} = $conf->{pending}->{$opt};
5012         }
5013
5014         delete $conf->{pending}->{$opt};
5015         PVE::QemuConfig->write_config($vmid, $conf);
5016     }
5017 }
5018
5019 my $safe_num_ne = sub {
5020     my ($a, $b) = @_;
5021
5022     return 0 if !defined($a) && !defined($b);
5023     return 1 if !defined($a);
5024     return 1 if !defined($b);
5025
5026     return $a != $b;
5027 };
5028
5029 my $safe_string_ne = sub {
5030     my ($a, $b) = @_;
5031
5032     return 0 if !defined($a) && !defined($b);
5033     return 1 if !defined($a);
5034     return 1 if !defined($b);
5035
5036     return $a ne $b;
5037 };
5038
5039 sub vmconfig_update_net {
5040     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
5041
5042     my $newnet = parse_net($value);
5043
5044     if ($conf->{$opt}) {
5045         my $oldnet = parse_net($conf->{$opt});
5046
5047         if (&$safe_string_ne($oldnet->{model}, $newnet->{model}) ||
5048             &$safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
5049             &$safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
5050             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
5051
5052             # for non online change, we try to hot-unplug
5053             die "skip\n" if !$hotplug;
5054             vm_deviceunplug($vmid, $conf, $opt);
5055         } else {
5056
5057             die "internal error" if $opt !~ m/net(\d+)/;
5058             my $iface = "tap${vmid}i$1";
5059
5060             if (&$safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
5061                 &$safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
5062                 &$safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
5063                 &$safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
5064                 PVE::Network::tap_unplug($iface);
5065                 PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
5066             } elsif (&$safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
5067                 # Rate can be applied on its own but any change above needs to
5068                 # include the rate in tap_plug since OVS resets everything.
5069                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
5070             }
5071
5072             if (&$safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
5073                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
5074             }
5075
5076             return 1;
5077         }
5078     }
5079
5080     if ($hotplug) {
5081         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet, $arch, $machine_type);
5082     } else {
5083         die "skip\n";
5084     }
5085 }
5086
5087 sub vmconfig_update_disk {
5088     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $force, $arch, $machine_type) = @_;
5089
5090     # fixme: do we need force?
5091
5092     my $drive = parse_drive($opt, $value);
5093
5094     if ($conf->{$opt}) {
5095
5096         if (my $old_drive = parse_drive($opt, $conf->{$opt}))  {
5097
5098             my $media = $drive->{media} || 'disk';
5099             my $oldmedia = $old_drive->{media} || 'disk';
5100             die "unable to change media type\n" if $media ne $oldmedia;
5101
5102             if (!drive_is_cdrom($old_drive)) {
5103
5104                 if ($drive->{file} ne $old_drive->{file}) {
5105
5106                     die "skip\n" if !$hotplug;
5107
5108                     # unplug and register as unused
5109                     vm_deviceunplug($vmid, $conf, $opt);
5110                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
5111
5112                 } else {
5113                     # update existing disk
5114
5115                     # skip non hotpluggable value
5116                     if (&$safe_string_ne($drive->{discard}, $old_drive->{discard}) ||
5117                         &$safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
5118                         &$safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
5119                         &$safe_string_ne($drive->{cache}, $old_drive->{cache})) {
5120                         die "skip\n";
5121                     }
5122
5123                     # apply throttle
5124                     if (&$safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
5125                         &$safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
5126                         &$safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
5127                         &$safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
5128                         &$safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
5129                         &$safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
5130                         &$safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
5131                         &$safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
5132                         &$safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
5133                         &$safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
5134                         &$safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
5135                         &$safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max}) ||
5136                         &$safe_num_ne($drive->{bps_max_length}, $old_drive->{bps_max_length}) ||
5137                         &$safe_num_ne($drive->{bps_rd_max_length}, $old_drive->{bps_rd_max_length}) ||
5138                         &$safe_num_ne($drive->{bps_wr_max_length}, $old_drive->{bps_wr_max_length}) ||
5139                         &$safe_num_ne($drive->{iops_max_length}, $old_drive->{iops_max_length}) ||
5140                         &$safe_num_ne($drive->{iops_rd_max_length}, $old_drive->{iops_rd_max_length}) ||
5141                         &$safe_num_ne($drive->{iops_wr_max_length}, $old_drive->{iops_wr_max_length})) {
5142
5143                         qemu_block_set_io_throttle($vmid,"drive-$opt",
5144                                                    ($drive->{mbps} || 0)*1024*1024,
5145                                                    ($drive->{mbps_rd} || 0)*1024*1024,
5146                                                    ($drive->{mbps_wr} || 0)*1024*1024,
5147                                                    $drive->{iops} || 0,
5148                                                    $drive->{iops_rd} || 0,
5149                                                    $drive->{iops_wr} || 0,
5150                                                    ($drive->{mbps_max} || 0)*1024*1024,
5151                                                    ($drive->{mbps_rd_max} || 0)*1024*1024,
5152                                                    ($drive->{mbps_wr_max} || 0)*1024*1024,
5153                                                    $drive->{iops_max} || 0,
5154                                                    $drive->{iops_rd_max} || 0,
5155                                                    $drive->{iops_wr_max} || 0,
5156                                                    $drive->{bps_max_length} || 1,
5157                                                    $drive->{bps_rd_max_length} || 1,
5158                                                    $drive->{bps_wr_max_length} || 1,
5159                                                    $drive->{iops_max_length} || 1,
5160                                                    $drive->{iops_rd_max_length} || 1,
5161                                                    $drive->{iops_wr_max_length} || 1);
5162
5163                     }
5164
5165                     return 1;
5166                 }
5167
5168             } else { # cdrom
5169
5170                 if ($drive->{file} eq 'none') {
5171                     mon_cmd($vmid, "eject",force => JSON::true,device => "drive-$opt");
5172                     if (drive_is_cloudinit($old_drive)) {
5173                         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive);
5174                     }
5175                 } else {
5176                     my $path = get_iso_path($storecfg, $vmid, $drive->{file});
5177                     mon_cmd($vmid, "eject", force => JSON::true,device => "drive-$opt"); # force eject if locked
5178                     mon_cmd($vmid, "change", device => "drive-$opt",target => "$path") if $path;
5179                 }
5180
5181                 return 1;
5182             }
5183         }
5184     }
5185
5186     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
5187     # hotplug new disks
5188     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
5189     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive, $arch, $machine_type);
5190 }
5191
5192 sub vm_start {
5193     my ($storecfg, $vmid, $statefile, $skiplock, $migratedfrom, $paused,
5194         $forcemachine, $spice_ticket, $migration_network, $migration_type, $targetstorage) = @_;
5195
5196     PVE::QemuConfig->lock_config($vmid, sub {
5197         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
5198
5199         die "you can't start a vm if it's a template\n" if PVE::QemuConfig->is_template($conf);
5200
5201         my $is_suspended = PVE::QemuConfig->has_lock($conf, 'suspended');
5202
5203         PVE::QemuConfig->check_lock($conf)
5204             if !($skiplock || $is_suspended);
5205
5206         die "VM $vmid already running\n" if check_running($vmid, undef, $migratedfrom);
5207
5208         # clean up leftover reboot request files
5209         eval { clear_reboot_request($vmid); };
5210         warn $@ if $@;
5211
5212         if (!$statefile && scalar(keys %{$conf->{pending}})) {
5213             vmconfig_apply_pending($vmid, $conf, $storecfg);
5214             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5215         }
5216
5217         PVE::QemuServer::Cloudinit::generate_cloudinitconfig($conf, $vmid);
5218
5219         my $defaults = load_defaults();
5220
5221         # set environment variable useful inside network script
5222         $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
5223
5224         my $local_volumes = {};
5225
5226         if ($targetstorage) {
5227             foreach_drive($conf, sub {
5228                 my ($ds, $drive) = @_;
5229
5230                 return if drive_is_cdrom($drive);
5231
5232                 my $volid = $drive->{file};
5233
5234                 return if !$volid;
5235
5236                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
5237
5238                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5239                 return if $scfg->{shared};
5240                 $local_volumes->{$ds} = [$volid, $storeid, $volname];
5241             });
5242
5243             my $format = undef;
5244
5245             foreach my $opt (sort keys %$local_volumes) {
5246
5247                 my ($volid, $storeid, $volname) = @{$local_volumes->{$opt}};
5248                 my $drive = parse_drive($opt, $conf->{$opt});
5249
5250                 #if remote storage is specified, use default format
5251                 if ($targetstorage && $targetstorage ne "1") {
5252                     $storeid = $targetstorage;
5253                     my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
5254                     $format = $defFormat;
5255                 } else {
5256                     #else we use same format than original
5257                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5258                     $format = qemu_img_format($scfg, $volid);
5259                 }
5260
5261                 my $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $format, undef, ($drive->{size}/1024));
5262                 my $newdrive = $drive;
5263                 $newdrive->{format} = $format;
5264                 $newdrive->{file} = $newvolid;
5265                 my $drivestr = PVE::QemuServer::print_drive($vmid, $newdrive);
5266                 $local_volumes->{$opt} = $drivestr;
5267                 #pass drive to conf for command line
5268                 $conf->{$opt} = $drivestr;
5269             }
5270         }
5271
5272         PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-start', 1);
5273
5274         if ($is_suspended) {
5275             # enforce machine type on suspended vm to ensure HW compatibility
5276             $forcemachine = $conf->{runningmachine};
5277             print "Resuming suspended VM\n";
5278         }
5279
5280         my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine);
5281
5282         my $migration_ip;
5283         my $get_migration_ip = sub {
5284             my ($cidr, $nodename) = @_;
5285
5286             return $migration_ip if defined($migration_ip);
5287
5288             if (!defined($cidr)) {
5289                 my $dc_conf = PVE::Cluster::cfs_read_file('datacenter.cfg');
5290                 $cidr = $dc_conf->{migration}->{network};
5291             }
5292
5293             if (defined($cidr)) {
5294                 my $ips = PVE::Network::get_local_ip_from_cidr($cidr);
5295
5296                 die "could not get IP: no address configured on local " .
5297                     "node for network '$cidr'\n" if scalar(@$ips) == 0;
5298
5299                 die "could not get IP: multiple addresses configured on local " .
5300                     "node for network '$cidr'\n" if scalar(@$ips) > 1;
5301
5302                 $migration_ip = @$ips[0];
5303             }
5304
5305             $migration_ip = PVE::Cluster::remote_node_ip($nodename, 1)
5306                 if !defined($migration_ip);
5307
5308             return $migration_ip;
5309         };
5310
5311         my $migrate_uri;
5312         if ($statefile) {
5313             if ($statefile eq 'tcp') {
5314                 my $localip = "localhost";
5315                 my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
5316                 my $nodename = PVE::INotify::nodename();
5317
5318                 if (!defined($migration_type)) {
5319                     if (defined($datacenterconf->{migration}->{type})) {
5320                         $migration_type = $datacenterconf->{migration}->{type};
5321                     } else {
5322                         $migration_type = 'secure';
5323                     }
5324                 }
5325
5326                 if ($migration_type eq 'insecure') {
5327                     $localip = $get_migration_ip->($migration_network, $nodename);
5328                     $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5329                 }
5330
5331                 my $pfamily = PVE::Tools::get_host_address_family($nodename);
5332                 my $migrate_port = PVE::Tools::next_migrate_port($pfamily);
5333                 $migrate_uri = "tcp:${localip}:${migrate_port}";
5334                 push @$cmd, '-incoming', $migrate_uri;
5335                 push @$cmd, '-S';
5336
5337             } elsif ($statefile eq 'unix') {
5338                 # should be default for secure migrations as a ssh TCP forward
5339                 # tunnel is not deterministic reliable ready and fails regurarly
5340                 # to set up in time, so use UNIX socket forwards
5341                 my $socket_addr = "/run/qemu-server/$vmid.migrate";
5342                 unlink $socket_addr;
5343
5344                 $migrate_uri = "unix:$socket_addr";
5345
5346                 push @$cmd, '-incoming', $migrate_uri;
5347                 push @$cmd, '-S';
5348
5349             } elsif (-e $statefile) {
5350                 push @$cmd, '-loadstate', $statefile;
5351             } else {
5352                 my $statepath = PVE::Storage::path($storecfg, $statefile);
5353                 push @$vollist, $statefile;
5354                 push @$cmd, '-loadstate', $statepath;
5355             }
5356         } elsif ($paused) {
5357             push @$cmd, '-S';
5358         }
5359
5360         # host pci devices
5361         for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
5362           my $d = parse_hostpci($conf->{"hostpci$i"});
5363           next if !$d;
5364           my $pcidevices = $d->{pciid};
5365           foreach my $pcidevice (@$pcidevices) {
5366                 my $pciid = $pcidevice->{id};
5367                 $pciid = "0000:$pciid" if $pciid !~ m/^[0-9a-f]{4}:/;
5368
5369                 my $info = PVE::SysFSTools::pci_device_info("$pciid");
5370                 die "IOMMU not present\n" if !PVE::SysFSTools::check_iommu_support();
5371                 die "no pci device info for device '$pciid'\n" if !$info;
5372
5373                 if ($d->{mdev}) {
5374                     my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $i);
5375                     PVE::SysFSTools::pci_create_mdev_device($pciid, $uuid, $d->{mdev});
5376                 } else {
5377                     die "can't unbind/bind pci group to vfio '$pciid'\n"
5378                         if !PVE::SysFSTools::pci_dev_group_bind_to_vfio($pciid);
5379                     die "can't reset pci device '$pciid'\n"
5380                         if $info->{has_fl_reset} and !PVE::SysFSTools::pci_dev_reset($info);
5381                 }
5382           }
5383         }
5384
5385         PVE::Storage::activate_volumes($storecfg, $vollist);
5386
5387         eval {
5388             run_command(['/bin/systemctl', 'stop', "$vmid.scope"],
5389                 outfunc => sub {}, errfunc => sub {});
5390         };
5391         # Issues with the above 'stop' not being fully completed are extremely rare, a very low
5392         # timeout should be more than enough here...
5393         PVE::Systemd::wait_for_unit_removed("$vmid.scope", 5);
5394
5395         my $cpuunits = defined($conf->{cpuunits}) ? $conf->{cpuunits}
5396                                                   : $defaults->{cpuunits};
5397
5398         my $start_timeout = ($conf->{hugepages} || $is_suspended) ? 300 : 30;
5399         my %run_params = (timeout => $statefile ? undef : $start_timeout, umask => 0077);
5400
5401         my %properties = (
5402             Slice => 'qemu.slice',
5403             KillMode => 'none',
5404             CPUShares => $cpuunits
5405         );
5406
5407         if (my $cpulimit = $conf->{cpulimit}) {
5408             $properties{CPUQuota} = int($cpulimit * 100);
5409         }
5410         $properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
5411
5412         my $run_qemu = sub {
5413             PVE::Tools::run_fork sub {
5414                 PVE::Systemd::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
5415                 run_command($cmd, %run_params);
5416             };
5417         };
5418
5419         if ($conf->{hugepages}) {
5420
5421             my $code = sub {
5422                 my $hugepages_topology = PVE::QemuServer::Memory::hugepages_topology($conf);
5423                 my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
5424
5425                 PVE::QemuServer::Memory::hugepages_mount();
5426                 PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
5427
5428                 eval { $run_qemu->() };
5429                 if (my $err = $@) {
5430                     PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology);
5431                     die $err;
5432                 }
5433
5434                 PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology);
5435             };
5436             eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
5437
5438         } else {
5439             eval { $run_qemu->() };
5440         }
5441
5442         if (my $err = $@) {
5443             # deactivate volumes if start fails
5444             eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
5445             die "start failed: $err";
5446         }
5447
5448         print "migration listens on $migrate_uri\n" if $migrate_uri;
5449
5450         if ($statefile && $statefile ne 'tcp' && $statefile ne 'unix')  {
5451             eval { mon_cmd($vmid, "cont"); };
5452             warn $@ if $@;
5453         }
5454
5455         #start nbd server for storage migration
5456         if ($targetstorage) {
5457             my $nodename = PVE::INotify::nodename();
5458             my $localip = $get_migration_ip->($migration_network, $nodename);
5459             my $pfamily = PVE::Tools::get_host_address_family($nodename);
5460             my $storage_migrate_port = PVE::Tools::next_migrate_port($pfamily);
5461
5462             mon_cmd($vmid, "nbd-server-start", addr => { type => 'inet', data => { host => "${localip}", port => "${storage_migrate_port}" } } );
5463
5464             $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5465
5466             foreach my $opt (sort keys %$local_volumes) {
5467                 my $volid = $local_volumes->{$opt};
5468                 mon_cmd($vmid, "nbd-server-add", device => "drive-$opt", writable => JSON::true );
5469                 my $migrate_storage_uri = "nbd:${localip}:${storage_migrate_port}:exportname=drive-$opt";
5470                 print "storage migration listens on $migrate_storage_uri volume:$volid\n";
5471             }
5472         }
5473
5474         if ($migratedfrom) {
5475             eval {
5476                 set_migration_caps($vmid);
5477             };
5478             warn $@ if $@;
5479
5480             if ($spice_port) {
5481                 print "spice listens on port $spice_port\n";
5482                 if ($spice_ticket) {
5483                     mon_cmd($vmid, "set_password", protocol => 'spice', password => $spice_ticket);
5484                     mon_cmd($vmid, "expire_password", protocol => 'spice', time => "+30");
5485                 }
5486             }
5487
5488         } else {
5489             mon_cmd($vmid, "balloon", value => $conf->{balloon}*1024*1024)
5490                 if !$statefile && $conf->{balloon};
5491
5492             foreach my $opt (keys %$conf) {
5493                 next if $opt !~  m/^net\d+$/;
5494                 my $nicconf = parse_net($conf->{$opt});
5495                 qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
5496             }
5497         }
5498
5499         mon_cmd($vmid, 'qom-set',
5500                     path => "machine/peripheral/balloon0",
5501                     property => "guest-stats-polling-interval",
5502                     value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
5503
5504         if ($is_suspended && (my $vmstate = $conf->{vmstate})) {
5505             print "Resumed VM, removing state\n";
5506             delete $conf->@{qw(lock vmstate runningmachine)};
5507             PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
5508             PVE::Storage::vdisk_free($storecfg, $vmstate);
5509             PVE::QemuConfig->write_config($vmid, $conf);
5510         }
5511
5512         PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'post-start');
5513     });
5514 }
5515
5516 sub vm_commandline {
5517     my ($storecfg, $vmid, $snapname) = @_;
5518
5519     my $conf = PVE::QemuConfig->load_config($vmid);
5520     my $forcemachine;
5521
5522     if ($snapname) {
5523         my $snapshot = $conf->{snapshots}->{$snapname};
5524         die "snapshot '$snapname' does not exist\n" if !defined($snapshot);
5525
5526         # check for a 'runningmachine' in snapshot
5527         $forcemachine = $snapshot->{runningmachine} if $snapshot->{runningmachine};
5528
5529         $snapshot->{digest} = $conf->{digest}; # keep file digest for API
5530
5531         $conf = $snapshot;
5532     }
5533
5534     my $defaults = load_defaults();
5535
5536     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine);
5537
5538     return PVE::Tools::cmd2string($cmd);
5539 }
5540
5541 sub vm_reset {
5542     my ($vmid, $skiplock) = @_;
5543
5544     PVE::QemuConfig->lock_config($vmid, sub {
5545
5546         my $conf = PVE::QemuConfig->load_config($vmid);
5547
5548         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5549
5550         mon_cmd($vmid, "system_reset");
5551     });
5552 }
5553
5554 sub get_vm_volumes {
5555     my ($conf) = @_;
5556
5557     my $vollist = [];
5558     foreach_volid($conf, sub {
5559         my ($volid, $attr) = @_;
5560
5561         return if $volid =~ m|^/|;
5562
5563         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5564         return if !$sid;
5565
5566         push @$vollist, $volid;
5567     });
5568
5569     return $vollist;
5570 }
5571
5572 sub vm_stop_cleanup {
5573     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
5574
5575     eval {
5576
5577         if (!$keepActive) {
5578             my $vollist = get_vm_volumes($conf);
5579             PVE::Storage::deactivate_volumes($storecfg, $vollist);
5580         }
5581
5582         foreach my $ext (qw(mon qmp pid vnc qga)) {
5583             unlink "/var/run/qemu-server/${vmid}.$ext";
5584         }
5585
5586         if ($conf->{ivshmem}) {
5587             my $ivshmem = PVE::JSONSchema::parse_property_string($ivshmem_fmt, $conf->{ivshmem});
5588             # just delete it for now, VMs which have this already open do not
5589             # are affected, but new VMs will get a separated one. If this
5590             # becomes an issue we either add some sort of ref-counting or just
5591             # add a "don't delete on stop" flag to the ivshmem format.
5592             unlink '/dev/shm/pve-shm-' . ($ivshmem->{name} // $vmid);
5593         }
5594
5595         foreach my $key (keys %$conf) {
5596             next if $key !~ m/^hostpci(\d+)$/;
5597             my $hostpciindex = $1;
5598             my $d = parse_hostpci($conf->{$key});
5599             my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $hostpciindex);
5600
5601             foreach my $pci (@{$d->{pciid}}) {
5602                 my $pciid = $pci->{id};
5603                 PVE::SysFSTools::pci_cleanup_mdev_device($pciid, $uuid);
5604             }
5605         }
5606
5607         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
5608     };
5609     warn $@ if $@; # avoid errors - just warn
5610 }
5611
5612 # call only in locked context
5613 sub _do_vm_stop {
5614     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive) = @_;
5615
5616     my $pid = check_running($vmid, $nocheck);
5617     return if !$pid;
5618
5619     my $conf;
5620     if (!$nocheck) {
5621         $conf = PVE::QemuConfig->load_config($vmid);
5622         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5623         if (!defined($timeout) && $shutdown && $conf->{startup}) {
5624             my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
5625             $timeout = $opts->{down} if $opts->{down};
5626         }
5627         PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-stop');
5628     }
5629
5630     eval {
5631         if ($shutdown) {
5632             if (defined($conf) && parse_guest_agent($conf)->{enabled}) {
5633                 mon_cmd($vmid, "guest-shutdown", timeout => $timeout);
5634             } else {
5635                 mon_cmd($vmid, "system_powerdown");
5636             }
5637         } else {
5638             mon_cmd($vmid, "quit");
5639         }
5640     };
5641     my $err = $@;
5642
5643     if (!$err) {
5644         $timeout = 60 if !defined($timeout);
5645
5646         my $count = 0;
5647         while (($count < $timeout) && check_running($vmid, $nocheck)) {
5648             $count++;
5649             sleep 1;
5650         }
5651
5652         if ($count >= $timeout) {
5653             if ($force) {
5654                 warn "VM still running - terminating now with SIGTERM\n";
5655                 kill 15, $pid;
5656             } else {
5657                 die "VM quit/powerdown failed - got timeout\n";
5658             }
5659         } else {
5660             vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5661             return;
5662         }
5663     } else {
5664         if ($force) {
5665             warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
5666             kill 15, $pid;
5667         } else {
5668             die "VM quit/powerdown failed\n";
5669         }
5670     }
5671
5672     # wait again
5673     $timeout = 10;
5674
5675     my $count = 0;
5676     while (($count < $timeout) && check_running($vmid, $nocheck)) {
5677         $count++;
5678         sleep 1;
5679     }
5680
5681     if ($count >= $timeout) {
5682         warn "VM still running - terminating now with SIGKILL\n";
5683         kill 9, $pid;
5684         sleep 1;
5685     }
5686
5687     vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5688 }
5689
5690 # Note: use $nocheck to skip tests if VM configuration file exists.
5691 # We need that when migration VMs to other nodes (files already moved)
5692 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
5693 sub vm_stop {
5694     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
5695
5696     $force = 1 if !defined($force) && !$shutdown;
5697
5698     if ($migratedfrom){
5699         my $pid = check_running($vmid, $nocheck, $migratedfrom);
5700         kill 15, $pid if $pid;
5701         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
5702         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
5703         return;
5704     }
5705
5706     PVE::QemuConfig->lock_config($vmid, sub {
5707         _do_vm_stop($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive);
5708    });
5709 }
5710
5711 sub vm_reboot {
5712     my ($vmid, $timeout) = @_;
5713
5714     PVE::QemuConfig->lock_config($vmid, sub {
5715         eval {
5716
5717             # only reboot if running, as qmeventd starts it again on a stop event
5718             return if !check_running($vmid);
5719
5720             create_reboot_request($vmid);
5721
5722             my $storecfg = PVE::Storage::config();
5723             _do_vm_stop($storecfg, $vmid, undef, undef, $timeout, 1);
5724
5725         };
5726         if (my $err = $@) {
5727             # avoid that the next normal shutdown will be confused for a reboot
5728             clear_reboot_request($vmid);
5729             die $err;
5730         }
5731    });
5732 }
5733
5734 sub vm_suspend {
5735     my ($vmid, $skiplock, $includestate, $statestorage) = @_;
5736
5737     my $conf;
5738     my $path;
5739     my $storecfg;
5740     my $vmstate;
5741
5742     PVE::QemuConfig->lock_config($vmid, sub {
5743
5744         $conf = PVE::QemuConfig->load_config($vmid);
5745
5746         my $is_backing_up = PVE::QemuConfig->has_lock($conf, 'backup');
5747         PVE::QemuConfig->check_lock($conf)
5748             if !($skiplock || $is_backing_up);
5749
5750         die "cannot suspend to disk during backup\n"
5751             if $is_backing_up && $includestate;
5752
5753         if ($includestate) {
5754             $conf->{lock} = 'suspending';
5755             my $date = strftime("%Y-%m-%d", localtime(time()));
5756             $storecfg = PVE::Storage::config();
5757             $vmstate = PVE::QemuConfig->__snapshot_save_vmstate($vmid, $conf, "suspend-$date", $storecfg, $statestorage, 1);
5758             $path = PVE::Storage::path($storecfg, $vmstate);
5759             PVE::QemuConfig->write_config($vmid, $conf);
5760         } else {
5761             mon_cmd($vmid, "stop");
5762         }
5763     });
5764
5765     if ($includestate) {
5766         # save vm state
5767         PVE::Storage::activate_volumes($storecfg, [$vmstate]);
5768
5769         eval {
5770             mon_cmd($vmid, "savevm-start", statefile => $path);
5771             for(;;) {
5772                 my $state = mon_cmd($vmid, "query-savevm");
5773                 if (!$state->{status}) {
5774                     die "savevm not active\n";
5775                 } elsif ($state->{status} eq 'active') {
5776                     sleep(1);
5777                     next;
5778                 } elsif ($state->{status} eq 'completed') {
5779                     print "State saved, quitting\n";
5780                     last;
5781                 } elsif ($state->{status} eq 'failed' && $state->{error}) {
5782                     die "query-savevm failed with error '$state->{error}'\n"
5783                 } else {
5784                     die "query-savevm returned status '$state->{status}'\n";
5785                 }
5786             }
5787         };
5788         my $err = $@;
5789
5790         PVE::QemuConfig->lock_config($vmid, sub {
5791             $conf = PVE::QemuConfig->load_config($vmid);
5792             if ($err) {
5793                 # cleanup, but leave suspending lock, to indicate something went wrong
5794                 eval {
5795                     mon_cmd($vmid, "savevm-end");
5796                     PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
5797                     PVE::Storage::vdisk_free($storecfg, $vmstate);
5798                     delete $conf->@{qw(vmstate runningmachine)};
5799                     PVE::QemuConfig->write_config($vmid, $conf);
5800                 };
5801                 warn $@ if $@;
5802                 die $err;
5803             }
5804
5805             die "lock changed unexpectedly\n"
5806                 if !PVE::QemuConfig->has_lock($conf, 'suspending');
5807
5808             mon_cmd($vmid, "quit");
5809             $conf->{lock} = 'suspended';
5810             PVE::QemuConfig->write_config($vmid, $conf);
5811         });
5812     }
5813 }
5814
5815 sub vm_resume {
5816     my ($vmid, $skiplock, $nocheck) = @_;
5817
5818     PVE::QemuConfig->lock_config($vmid, sub {
5819         my $res = mon_cmd($vmid, 'query-status');
5820         my $resume_cmd = 'cont';
5821
5822         if ($res->{status} && $res->{status} eq 'suspended') {
5823             $resume_cmd = 'system_wakeup';
5824         }
5825
5826         if (!$nocheck) {
5827
5828             my $conf = PVE::QemuConfig->load_config($vmid);
5829
5830             PVE::QemuConfig->check_lock($conf)
5831                 if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
5832         }
5833
5834         mon_cmd($vmid, $resume_cmd);
5835     });
5836 }
5837
5838 sub vm_sendkey {
5839     my ($vmid, $skiplock, $key) = @_;
5840
5841     PVE::QemuConfig->lock_config($vmid, sub {
5842
5843         my $conf = PVE::QemuConfig->load_config($vmid);
5844
5845         # there is no qmp command, so we use the human monitor command
5846         my $res = PVE::QemuServer::Monitor::hmp_cmd($vmid, "sendkey $key");
5847         die $res if $res ne '';
5848     });
5849 }
5850
5851 # vzdump restore implementaion
5852
5853 sub tar_archive_read_firstfile {
5854     my $archive = shift;
5855
5856     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
5857
5858     # try to detect archive type first
5859     my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
5860         die "unable to open file '$archive'\n";
5861     my $firstfile = <$fh>;
5862     kill 15, $pid;
5863     close $fh;
5864
5865     die "ERROR: archive contaions no data\n" if !$firstfile;
5866     chomp $firstfile;
5867
5868     return $firstfile;
5869 }
5870
5871 sub tar_restore_cleanup {
5872     my ($storecfg, $statfile) = @_;
5873
5874     print STDERR "starting cleanup\n";
5875
5876     if (my $fd = IO::File->new($statfile, "r")) {
5877         while (defined(my $line = <$fd>)) {
5878             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5879                 my $volid = $2;
5880                 eval {
5881                     if ($volid =~ m|^/|) {
5882                         unlink $volid || die 'unlink failed\n';
5883                     } else {
5884                         PVE::Storage::vdisk_free($storecfg, $volid);
5885                     }
5886                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
5887                 };
5888                 print STDERR "unable to cleanup '$volid' - $@" if $@;
5889             } else {
5890                 print STDERR "unable to parse line in statfile - $line";
5891             }
5892         }
5893         $fd->close();
5894     }
5895 }
5896
5897 sub restore_archive {
5898     my ($archive, $vmid, $user, $opts) = @_;
5899
5900     my $format = $opts->{format};
5901     my $comp;
5902
5903     if ($archive =~ m/\.tgz$/ || $archive =~ m/\.tar\.gz$/) {
5904         $format = 'tar' if !$format;
5905         $comp = 'gzip';
5906     } elsif ($archive =~ m/\.tar$/) {
5907         $format = 'tar' if !$format;
5908     } elsif ($archive =~ m/.tar.lzo$/) {
5909         $format = 'tar' if !$format;
5910         $comp = 'lzop';
5911     } elsif ($archive =~ m/\.vma$/) {
5912         $format = 'vma' if !$format;
5913     } elsif ($archive =~ m/\.vma\.gz$/) {
5914         $format = 'vma' if !$format;
5915         $comp = 'gzip';
5916     } elsif ($archive =~ m/\.vma\.lzo$/) {
5917         $format = 'vma' if !$format;
5918         $comp = 'lzop';
5919     } else {
5920         $format = 'vma' if !$format; # default
5921     }
5922
5923     # try to detect archive format
5924     if ($format eq 'tar') {
5925         return restore_tar_archive($archive, $vmid, $user, $opts);
5926     } else {
5927         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
5928     }
5929 }
5930
5931 sub restore_update_config_line {
5932     my ($outfd, $cookie, $vmid, $map, $line, $unique) = @_;
5933
5934     return if $line =~ m/^\#qmdump\#/;
5935     return if $line =~ m/^\#vzdump\#/;
5936     return if $line =~ m/^lock:/;
5937     return if $line =~ m/^unused\d+:/;
5938     return if $line =~ m/^parent:/;
5939
5940     my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
5941     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
5942         # try to convert old 1.X settings
5943         my ($id, $ind, $ethcfg) = ($1, $2, $3);
5944         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
5945             my ($model, $macaddr) = split(/\=/, $devconfig);
5946             $macaddr = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if !$macaddr || $unique;
5947             my $net = {
5948                 model => $model,
5949                 bridge => "vmbr$ind",
5950                 macaddr => $macaddr,
5951             };
5952             my $netstr = print_net($net);
5953
5954             print $outfd "net$cookie->{netcount}: $netstr\n";
5955             $cookie->{netcount}++;
5956         }
5957     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
5958         my ($id, $netstr) = ($1, $2);
5959         my $net = parse_net($netstr);
5960         $net->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if $net->{macaddr};
5961         $netstr = print_net($net);
5962         print $outfd "$id: $netstr\n";
5963     } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk)\d+):\s*(\S+)\s*$/) {
5964         my $virtdev = $1;
5965         my $value = $3;
5966         my $di = parse_drive($virtdev, $value);
5967         if (defined($di->{backup}) && !$di->{backup}) {
5968             print $outfd "#$line";
5969         } elsif ($map->{$virtdev}) {
5970             delete $di->{format}; # format can change on restore
5971             $di->{file} = $map->{$virtdev};
5972             $value = print_drive($vmid, $di);
5973             print $outfd "$virtdev: $value\n";
5974         } else {
5975             print $outfd $line;
5976         }
5977     } elsif (($line =~ m/^vmgenid: (.*)/)) {
5978         my $vmgenid = $1;
5979         if ($vmgenid ne '0') {
5980             # always generate a new vmgenid if there was a valid one setup
5981             $vmgenid = generate_uuid();
5982         }
5983         print $outfd "vmgenid: $vmgenid\n";
5984     } elsif (($line =~ m/^(smbios1: )(.*)/) && $unique) {
5985         my ($uuid, $uuid_str);
5986         UUID::generate($uuid);
5987         UUID::unparse($uuid, $uuid_str);
5988         my $smbios1 = parse_smbios1($2);
5989         $smbios1->{uuid} = $uuid_str;
5990         print $outfd $1.print_smbios1($smbios1)."\n";
5991     } else {
5992         print $outfd $line;
5993     }
5994 }
5995
5996 sub scan_volids {
5997     my ($cfg, $vmid) = @_;
5998
5999     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid);
6000
6001     my $volid_hash = {};
6002     foreach my $storeid (keys %$info) {
6003         foreach my $item (@{$info->{$storeid}}) {
6004             next if !($item->{volid} && $item->{size});
6005             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
6006             $volid_hash->{$item->{volid}} = $item;
6007         }
6008     }
6009
6010     return $volid_hash;
6011 }
6012
6013 sub is_volume_in_use {
6014     my ($storecfg, $conf, $skip_drive, $volid) = @_;
6015
6016     my $path = PVE::Storage::path($storecfg, $volid);
6017
6018     my $scan_config = sub {
6019         my ($cref, $snapname) = @_;
6020
6021         foreach my $key (keys %$cref) {
6022             my $value = $cref->{$key};
6023             if (is_valid_drivename($key)) {
6024                 next if $skip_drive && $key eq $skip_drive;
6025                 my $drive = parse_drive($key, $value);
6026                 next if !$drive || !$drive->{file} || drive_is_cdrom($drive);
6027                 return 1 if $volid eq $drive->{file};
6028                 if ($drive->{file} =~ m!^/!) {
6029                     return 1 if $drive->{file} eq $path;
6030                 } else {
6031                     my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file}, 1);
6032                     next if !$storeid;
6033                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid, 1);
6034                     next if !$scfg;
6035                     return 1 if $path eq PVE::Storage::path($storecfg, $drive->{file}, $snapname);
6036                 }
6037             }
6038         }
6039
6040         return 0;
6041     };
6042
6043     return 1 if &$scan_config($conf);
6044
6045     undef $skip_drive;
6046
6047     foreach my $snapname (keys %{$conf->{snapshots}}) {
6048         return 1 if &$scan_config($conf->{snapshots}->{$snapname}, $snapname);
6049     }
6050
6051     return 0;
6052 }
6053
6054 sub update_disksize {
6055     my ($vmid, $conf, $volid_hash) = @_;
6056
6057     my $changes;
6058     my $prefix = "VM $vmid:";
6059
6060     # used and unused disks
6061     my $referenced = {};
6062
6063     # Note: it is allowed to define multiple storages with same path (alias), so
6064     # we need to check both 'volid' and real 'path' (two different volid can point
6065     # to the same path).
6066
6067     my $referencedpath = {};
6068
6069     # update size info
6070     foreach my $opt (keys %$conf) {
6071         if (is_valid_drivename($opt)) {
6072             my $drive = parse_drive($opt, $conf->{$opt});
6073             my $volid = $drive->{file};
6074             next if !$volid;
6075
6076             $referenced->{$volid} = 1;
6077             if ($volid_hash->{$volid} &&
6078                 (my $path = $volid_hash->{$volid}->{path})) {
6079                 $referencedpath->{$path} = 1;
6080             }
6081
6082             next if drive_is_cdrom($drive);
6083             next if !$volid_hash->{$volid};
6084
6085             $drive->{size} = $volid_hash->{$volid}->{size};
6086             my $new = print_drive($vmid, $drive);
6087             if ($new ne $conf->{$opt}) {
6088                 $changes = 1;
6089                 $conf->{$opt} = $new;
6090                 print "$prefix update disk '$opt' information.\n";
6091             }
6092         }
6093     }
6094
6095     # remove 'unusedX' entry if volume is used
6096     foreach my $opt (keys %$conf) {
6097         next if $opt !~ m/^unused\d+$/;
6098         my $volid = $conf->{$opt};
6099         my $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
6100         if ($referenced->{$volid} || ($path && $referencedpath->{$path})) {
6101             print "$prefix remove entry '$opt', its volume '$volid' is in use.\n";
6102             $changes = 1;
6103             delete $conf->{$opt};
6104         }
6105
6106         $referenced->{$volid} = 1;
6107         $referencedpath->{$path} = 1 if $path;
6108     }
6109
6110     foreach my $volid (sort keys %$volid_hash) {
6111         next if $volid =~ m/vm-$vmid-state-/;
6112         next if $referenced->{$volid};
6113         my $path = $volid_hash->{$volid}->{path};
6114         next if !$path; # just to be sure
6115         next if $referencedpath->{$path};
6116         $changes = 1;
6117         my $key = PVE::QemuConfig->add_unused_volume($conf, $volid);
6118         print "$prefix add unreferenced volume '$volid' as '$key' to config.\n";
6119         $referencedpath->{$path} = 1; # avoid to add more than once (aliases)
6120     }
6121
6122     return $changes;
6123 }
6124
6125 sub rescan {
6126     my ($vmid, $nolock, $dryrun) = @_;
6127
6128     my $cfg = PVE::Storage::config();
6129
6130     # FIXME: Remove once our RBD plugin can handle CT and VM on a single storage
6131     # see: https://pve.proxmox.com/pipermail/pve-devel/2018-July/032900.html
6132     foreach my $stor (keys %{$cfg->{ids}}) {
6133         delete($cfg->{ids}->{$stor}) if ! $cfg->{ids}->{$stor}->{content}->{images};
6134     }
6135
6136     print "rescan volumes...\n";
6137     my $volid_hash = scan_volids($cfg, $vmid);
6138
6139     my $updatefn =  sub {
6140         my ($vmid) = @_;
6141
6142         my $conf = PVE::QemuConfig->load_config($vmid);
6143
6144         PVE::QemuConfig->check_lock($conf);
6145
6146         my $vm_volids = {};
6147         foreach my $volid (keys %$volid_hash) {
6148             my $info = $volid_hash->{$volid};
6149             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
6150         }
6151
6152         my $changes = update_disksize($vmid, $conf, $vm_volids);
6153
6154         PVE::QemuConfig->write_config($vmid, $conf) if $changes && !$dryrun;
6155     };
6156
6157     if (defined($vmid)) {
6158         if ($nolock) {
6159             &$updatefn($vmid);
6160         } else {
6161             PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6162         }
6163     } else {
6164         my $vmlist = config_list();
6165         foreach my $vmid (keys %$vmlist) {
6166             if ($nolock) {
6167                 &$updatefn($vmid);
6168             } else {
6169                 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6170             }
6171         }
6172     }
6173 }
6174
6175 sub restore_vma_archive {
6176     my ($archive, $vmid, $user, $opts, $comp) = @_;
6177
6178     my $readfrom = $archive;
6179
6180     my $cfg = PVE::Storage::config();
6181     my $commands = [];
6182     my $bwlimit = $opts->{bwlimit};
6183
6184     my $dbg_cmdstring = '';
6185     my $add_pipe = sub {
6186         my ($cmd) = @_;
6187         push @$commands, $cmd;
6188         $dbg_cmdstring .= ' | ' if length($dbg_cmdstring);
6189         $dbg_cmdstring .= PVE::Tools::cmd2string($cmd);
6190         $readfrom = '-';
6191     };
6192
6193     my $input = undef;
6194     if ($archive eq '-') {
6195         $input = '<&STDIN';
6196     } else {
6197         # If we use a backup from a PVE defined storage we also consider that
6198         # storage's rate limit:
6199         my (undef, $volid) = PVE::Storage::path_to_volume_id($cfg, $archive);
6200         if (defined($volid)) {
6201             my ($sid, undef) = PVE::Storage::parse_volume_id($volid);
6202             my $readlimit = PVE::Storage::get_bandwidth_limit('restore', [$sid], $bwlimit);
6203             if ($readlimit) {
6204                 print STDERR "applying read rate limit: $readlimit\n";
6205                 my $cstream = ['cstream', '-t', $readlimit*1024, '--', $readfrom];
6206                 $add_pipe->($cstream);
6207             }
6208         }
6209     }
6210
6211     if ($comp) {
6212         my $cmd;
6213         if ($comp eq 'gzip') {
6214             $cmd = ['zcat', $readfrom];
6215         } elsif ($comp eq 'lzop') {
6216             $cmd = ['lzop', '-d', '-c', $readfrom];
6217         } else {
6218             die "unknown compression method '$comp'\n";
6219         }
6220         $add_pipe->($cmd);
6221     }
6222
6223     my $tmpdir = "/var/tmp/vzdumptmp$$";
6224     rmtree $tmpdir;
6225
6226     # disable interrupts (always do cleanups)
6227     local $SIG{INT} =
6228         local $SIG{TERM} =
6229         local $SIG{QUIT} =
6230         local $SIG{HUP} = sub { warn "got interrupt - ignored\n"; };
6231
6232     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
6233     POSIX::mkfifo($mapfifo, 0600);
6234     my $fifofh;
6235
6236     my $openfifo = sub {
6237         open($fifofh, '>', $mapfifo) || die $!;
6238     };
6239
6240     $add_pipe->(['vma', 'extract', '-v', '-r', $mapfifo, $readfrom, $tmpdir]);
6241
6242     my $oldtimeout;
6243     my $timeout = 5;
6244
6245     my $devinfo = {};
6246
6247     my $rpcenv = PVE::RPCEnvironment::get();
6248
6249     my $conffile = PVE::QemuConfig->config_file($vmid);
6250     my $tmpfn = "$conffile.$$.tmp";
6251
6252     # Note: $oldconf is undef if VM does not exists
6253     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
6254     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
6255
6256     my %storage_limits;
6257
6258     my $print_devmap = sub {
6259         my $virtdev_hash = {};
6260
6261         my $cfgfn = "$tmpdir/qemu-server.conf";
6262
6263         # we can read the config - that is already extracted
6264         my $fh = IO::File->new($cfgfn, "r") ||
6265             "unable to read qemu-server.conf - $!\n";
6266
6267         my $fwcfgfn = "$tmpdir/qemu-server.fw";
6268         if (-f $fwcfgfn) {
6269             my $pve_firewall_dir = '/etc/pve/firewall';
6270             mkdir $pve_firewall_dir; # make sure the dir exists
6271             PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
6272         }
6273
6274         while (defined(my $line = <$fh>)) {
6275             if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
6276                 my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
6277                 die "archive does not contain data for drive '$virtdev'\n"
6278                     if !$devinfo->{$devname};
6279                 if (defined($opts->{storage})) {
6280                     $storeid = $opts->{storage} || 'local';
6281                 } elsif (!$storeid) {
6282                     $storeid = 'local';
6283                 }
6284                 $format = 'raw' if !$format;
6285                 $devinfo->{$devname}->{devname} = $devname;
6286                 $devinfo->{$devname}->{virtdev} = $virtdev;
6287                 $devinfo->{$devname}->{format} = $format;
6288                 $devinfo->{$devname}->{storeid} = $storeid;
6289
6290                 # check permission on storage
6291                 my $pool = $opts->{pool}; # todo: do we need that?
6292                 if ($user ne 'root@pam') {
6293                     $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
6294                 }
6295
6296                 $storage_limits{$storeid} = $bwlimit;
6297
6298                 $virtdev_hash->{$virtdev} = $devinfo->{$devname};
6299             } elsif ($line =~ m/^((?:ide|sata|scsi)\d+):\s*(.*)\s*$/) {
6300                 my $virtdev = $1;
6301                 my $drive = parse_drive($virtdev, $2);
6302                 if (drive_is_cloudinit($drive)) {
6303                     my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6304                     my $scfg = PVE::Storage::storage_config($cfg, $storeid);
6305                     my $format = qemu_img_format($scfg, $volname); # has 'raw' fallback
6306
6307                     my $d = {
6308                         format => $format,
6309                         storeid => $opts->{storage} // $storeid,
6310                         size => PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE,
6311                         file => $drive->{file}, # to make drive_is_cloudinit check possible
6312                         name => "vm-$vmid-cloudinit",
6313                         is_cloudinit => 1,
6314                     };
6315                     $virtdev_hash->{$virtdev} = $d;
6316                 }
6317             }
6318         }
6319
6320         foreach my $key (keys %storage_limits) {
6321             my $limit = PVE::Storage::get_bandwidth_limit('restore', [$key], $bwlimit);
6322             next if !$limit;
6323             print STDERR "rate limit for storage $key: $limit KiB/s\n";
6324             $storage_limits{$key} = $limit * 1024;
6325         }
6326
6327         foreach my $devname (keys %$devinfo) {
6328             die "found no device mapping information for device '$devname'\n"
6329                 if !$devinfo->{$devname}->{virtdev};
6330         }
6331
6332         # create empty/temp config
6333         if ($oldconf) {
6334             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
6335             foreach_drive($oldconf, sub {
6336                 my ($ds, $drive) = @_;
6337
6338                 return if drive_is_cdrom($drive, 1);
6339
6340                 my $volid = $drive->{file};
6341                 return if !$volid || $volid =~ m|^/|;
6342
6343                 my ($path, $owner) = PVE::Storage::path($cfg, $volid);
6344                 return if !$path || !$owner || ($owner != $vmid);
6345
6346                 # Note: only delete disk we want to restore
6347                 # other volumes will become unused
6348                 if ($virtdev_hash->{$ds}) {
6349                     eval { PVE::Storage::vdisk_free($cfg, $volid); };
6350                     if (my $err = $@) {
6351                         warn $err;
6352                     }
6353                 }
6354             });
6355
6356             # delete vmstate files, after the restore we have no snapshots anymore
6357             foreach my $snapname (keys %{$oldconf->{snapshots}}) {
6358                 my $snap = $oldconf->{snapshots}->{$snapname};
6359                 if ($snap->{vmstate}) {
6360                     eval { PVE::Storage::vdisk_free($cfg, $snap->{vmstate}); };
6361                     if (my $err = $@) {
6362                         warn $err;
6363                     }
6364                 }
6365             }
6366         }
6367
6368         my $map = {};
6369         foreach my $virtdev (sort keys %$virtdev_hash) {
6370             my $d = $virtdev_hash->{$virtdev};
6371             my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
6372             my $storeid = $d->{storeid};
6373             my $scfg = PVE::Storage::storage_config($cfg, $storeid);
6374
6375             my $map_opts = '';
6376             if (my $limit = $storage_limits{$storeid}) {
6377                 $map_opts .= "throttling.bps=$limit:throttling.group=$storeid:";
6378             }
6379
6380             # test if requested format is supported
6381             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($cfg, $storeid);
6382             my $supported = grep { $_ eq $d->{format} } @$validFormats;
6383             $d->{format} = $defFormat if !$supported;
6384
6385             my $name;
6386             if ($d->{is_cloudinit}) {
6387                 $name = $d->{name};
6388                 $name .= ".$d->{format}" if $d->{format} ne 'raw';
6389             }
6390
6391             my $volid = PVE::Storage::vdisk_alloc($cfg, $storeid, $vmid, $d->{format}, $name, $alloc_size);
6392             print STDERR "new volume ID is '$volid'\n";
6393             $d->{volid} = $volid;
6394
6395             PVE::Storage::activate_volumes($cfg, [$volid]);
6396
6397             my $write_zeros = 1;
6398             if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
6399                 $write_zeros = 0;
6400             }
6401
6402             if (!$d->{is_cloudinit}) {
6403                 my $path = PVE::Storage::path($cfg, $volid);
6404
6405                 print $fifofh "${map_opts}format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
6406
6407                 print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
6408             }
6409             $map->{$virtdev} = $volid;
6410         }
6411
6412         $fh->seek(0, 0) || die "seek failed - $!\n";
6413
6414         my $outfd = new IO::File ($tmpfn, "w") ||
6415             die "unable to write config for VM $vmid\n";
6416
6417         my $cookie = { netcount => 0 };
6418         while (defined(my $line = <$fh>)) {
6419             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
6420         }
6421
6422         $fh->close();
6423         $outfd->close();
6424     };
6425
6426     eval {
6427         # enable interrupts
6428         local $SIG{INT} =
6429             local $SIG{TERM} =
6430             local $SIG{QUIT} =
6431             local $SIG{HUP} =
6432             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6433         local $SIG{ALRM} = sub { die "got timeout\n"; };
6434
6435         $oldtimeout = alarm($timeout);
6436
6437         my $parser = sub {
6438             my $line = shift;
6439
6440             print "$line\n";
6441
6442             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
6443                 my ($dev_id, $size, $devname) = ($1, $2, $3);
6444                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
6445             } elsif ($line =~ m/^CTIME: /) {
6446                 # we correctly received the vma config, so we can disable
6447                 # the timeout now for disk allocation (set to 10 minutes, so
6448                 # that we always timeout if something goes wrong)
6449                 alarm(600);
6450                 &$print_devmap();
6451                 print $fifofh "done\n";
6452                 my $tmp = $oldtimeout || 0;
6453                 $oldtimeout = undef;
6454                 alarm($tmp);
6455                 close($fifofh);
6456             }
6457         };
6458
6459         print "restore vma archive: $dbg_cmdstring\n";
6460         run_command($commands, input => $input, outfunc => $parser, afterfork => $openfifo);
6461     };
6462     my $err = $@;
6463
6464     alarm($oldtimeout) if $oldtimeout;
6465
6466     my $vollist = [];
6467     foreach my $devname (keys %$devinfo) {
6468         my $volid = $devinfo->{$devname}->{volid};
6469         push @$vollist, $volid if $volid;
6470     }
6471
6472     PVE::Storage::deactivate_volumes($cfg, $vollist);
6473
6474     unlink $mapfifo;
6475
6476     if ($err) {
6477         rmtree $tmpdir;
6478         unlink $tmpfn;
6479
6480         foreach my $devname (keys %$devinfo) {
6481             my $volid = $devinfo->{$devname}->{volid};
6482             next if !$volid;
6483             eval {
6484                 if ($volid =~ m|^/|) {
6485                     unlink $volid || die 'unlink failed\n';
6486                 } else {
6487                     PVE::Storage::vdisk_free($cfg, $volid);
6488                 }
6489                 print STDERR "temporary volume '$volid' sucessfuly removed\n";
6490             };
6491             print STDERR "unable to cleanup '$volid' - $@" if $@;
6492         }
6493         die $err;
6494     }
6495
6496     rmtree $tmpdir;
6497
6498     rename($tmpfn, $conffile) ||
6499         die "unable to commit configuration file '$conffile'\n";
6500
6501     PVE::Cluster::cfs_update(); # make sure we read new file
6502
6503     eval { rescan($vmid, 1); };
6504     warn $@ if $@;
6505 }
6506
6507 sub restore_tar_archive {
6508     my ($archive, $vmid, $user, $opts) = @_;
6509
6510     if ($archive ne '-') {
6511         my $firstfile = tar_archive_read_firstfile($archive);
6512         die "ERROR: file '$archive' dos not lock like a QemuServer vzdump backup\n"
6513             if $firstfile ne 'qemu-server.conf';
6514     }
6515
6516     my $storecfg = PVE::Storage::config();
6517
6518     # avoid zombie disks when restoring over an existing VM -> cleanup first
6519     # pass keep_empty_config=1 to keep the config (thus VMID) reserved for us
6520     # skiplock=1 because qmrestore has set the 'create' lock itself already
6521     my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
6522     destroy_vm($storecfg, $vmid, 1, { lock => 'restore' }) if -f $vmcfgfn;
6523
6524     my $tocmd = "/usr/lib/qemu-server/qmextract";
6525
6526     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
6527     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
6528     $tocmd .= ' --prealloc' if $opts->{prealloc};
6529     $tocmd .= ' --info' if $opts->{info};
6530
6531     # tar option "xf" does not autodetect compression when read from STDIN,
6532     # so we pipe to zcat
6533     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
6534         PVE::Tools::shellquote("--to-command=$tocmd");
6535
6536     my $tmpdir = "/var/tmp/vzdumptmp$$";
6537     mkpath $tmpdir;
6538
6539     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
6540     local $ENV{VZDUMP_VMID} = $vmid;
6541     local $ENV{VZDUMP_USER} = $user;
6542
6543     my $conffile = PVE::QemuConfig->config_file($vmid);
6544     my $tmpfn = "$conffile.$$.tmp";
6545
6546     # disable interrupts (always do cleanups)
6547     local $SIG{INT} =
6548         local $SIG{TERM} =
6549         local $SIG{QUIT} =
6550         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
6551
6552     eval {
6553         # enable interrupts
6554         local $SIG{INT} =
6555             local $SIG{TERM} =
6556             local $SIG{QUIT} =
6557             local $SIG{HUP} =
6558             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6559
6560         if ($archive eq '-') {
6561             print "extracting archive from STDIN\n";
6562             run_command($cmd, input => "<&STDIN");
6563         } else {
6564             print "extracting archive '$archive'\n";
6565             run_command($cmd);
6566         }
6567
6568         return if $opts->{info};
6569
6570         # read new mapping
6571         my $map = {};
6572         my $statfile = "$tmpdir/qmrestore.stat";
6573         if (my $fd = IO::File->new($statfile, "r")) {
6574             while (defined (my $line = <$fd>)) {
6575                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
6576                     $map->{$1} = $2 if $1;
6577                 } else {
6578                     print STDERR "unable to parse line in statfile - $line\n";
6579                 }
6580             }
6581             $fd->close();
6582         }
6583
6584         my $confsrc = "$tmpdir/qemu-server.conf";
6585
6586         my $srcfd = new IO::File($confsrc, "r") ||
6587             die "unable to open file '$confsrc'\n";
6588
6589         my $outfd = new IO::File ($tmpfn, "w") ||
6590             die "unable to write config for VM $vmid\n";
6591
6592         my $cookie = { netcount => 0 };
6593         while (defined (my $line = <$srcfd>)) {
6594             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
6595         }
6596
6597         $srcfd->close();
6598         $outfd->close();
6599     };
6600     if (my $err = $@) {
6601         unlink $tmpfn;
6602         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
6603         die $err;
6604     }
6605
6606     rmtree $tmpdir;
6607
6608     rename $tmpfn, $conffile ||
6609         die "unable to commit configuration file '$conffile'\n";
6610
6611     PVE::Cluster::cfs_update(); # make sure we read new file
6612
6613     eval { rescan($vmid, 1); };
6614     warn $@ if $@;
6615 };
6616
6617 sub foreach_storage_used_by_vm {
6618     my ($conf, $func) = @_;
6619
6620     my $sidhash = {};
6621
6622     foreach_drive($conf, sub {
6623         my ($ds, $drive) = @_;
6624         return if drive_is_cdrom($drive);
6625
6626         my $volid = $drive->{file};
6627
6628         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
6629         $sidhash->{$sid} = $sid if $sid;
6630     });
6631
6632     foreach my $sid (sort keys %$sidhash) {
6633         &$func($sid);
6634     }
6635 }
6636
6637 my $qemu_snap_storage = {
6638     rbd => 1,
6639 };
6640 sub do_snapshots_with_qemu {
6641     my ($storecfg, $volid) = @_;
6642
6643     my $storage_name = PVE::Storage::parse_volume_id($volid);
6644     my $scfg = $storecfg->{ids}->{$storage_name};
6645
6646     if ($qemu_snap_storage->{$scfg->{type}} && !$scfg->{krbd}){
6647         return 1;
6648     }
6649
6650     if ($volid =~ m/\.(qcow2|qed)$/){
6651         return 1;
6652     }
6653
6654     return undef;
6655 }
6656
6657 sub qga_check_running {
6658     my ($vmid, $nowarn) = @_;
6659
6660     eval { mon_cmd($vmid, "guest-ping", timeout => 3); };
6661     if ($@) {
6662         warn "Qemu Guest Agent is not running - $@" if !$nowarn;
6663         return 0;
6664     }
6665     return 1;
6666 }
6667
6668 sub template_create {
6669     my ($vmid, $conf, $disk) = @_;
6670
6671     my $storecfg = PVE::Storage::config();
6672
6673     foreach_drive($conf, sub {
6674         my ($ds, $drive) = @_;
6675
6676         return if drive_is_cdrom($drive);
6677         return if $disk && $ds ne $disk;
6678
6679         my $volid = $drive->{file};
6680         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
6681
6682         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
6683         $drive->{file} = $voliddst;
6684         $conf->{$ds} = print_drive($vmid, $drive);
6685         PVE::QemuConfig->write_config($vmid, $conf);
6686     });
6687 }
6688
6689 sub convert_iscsi_path {
6690     my ($path) = @_;
6691
6692     if ($path =~ m|^iscsi://([^/]+)/([^/]+)/(.+)$|) {
6693         my $portal = $1;
6694         my $target = $2;
6695         my $lun = $3;
6696
6697         my $initiator_name = get_initiator_name();
6698
6699         return "file.driver=iscsi,file.transport=tcp,file.initiator-name=$initiator_name,".
6700                "file.portal=$portal,file.target=$target,file.lun=$lun,driver=raw";
6701     }
6702
6703     die "cannot convert iscsi path '$path', unkown format\n";
6704 }
6705
6706 sub qemu_img_convert {
6707     my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized) = @_;
6708
6709     my $storecfg = PVE::Storage::config();
6710     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
6711     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
6712
6713     die "destination '$dst_volid' is not a valid volid form qemu-img convert\n" if !$dst_storeid;
6714
6715     my $cachemode;
6716     my $src_path;
6717     my $src_is_iscsi = 0;
6718     my $src_format = 'raw';
6719
6720     if ($src_storeid) {
6721         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
6722         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
6723         $src_format = qemu_img_format($src_scfg, $src_volname);
6724         $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
6725         $src_is_iscsi = ($src_path =~ m|^iscsi://|);
6726         $cachemode = 'none' if $src_scfg->{type} eq 'zfspool';
6727     } elsif (-f $src_volid) {
6728         $src_path = $src_volid;
6729         if ($src_path =~ m/\.($QEMU_FORMAT_RE)$/) {
6730             $src_format = $1;
6731         }
6732     }
6733
6734     die "source '$src_volid' is not a valid volid nor path for qemu-img convert\n" if !$src_path;
6735
6736     my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6737     my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
6738     my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6739     my $dst_is_iscsi = ($dst_path =~ m|^iscsi://|);
6740
6741     my $cmd = [];
6742     push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
6743     push @$cmd, '-l', "snapshot.name=$snapname" if($snapname && $src_format eq "qcow2");
6744     push @$cmd, '-t', 'none' if $dst_scfg->{type} eq 'zfspool';
6745     push @$cmd, '-T', $cachemode if defined($cachemode);
6746
6747     if ($src_is_iscsi) {
6748         push @$cmd, '--image-opts';
6749         $src_path = convert_iscsi_path($src_path);
6750     } else {
6751         push @$cmd, '-f', $src_format;
6752     }
6753
6754     if ($dst_is_iscsi) {
6755         push @$cmd, '--target-image-opts';
6756         $dst_path = convert_iscsi_path($dst_path);
6757     } else {
6758         push @$cmd, '-O', $dst_format;
6759     }
6760
6761     push @$cmd, $src_path;
6762
6763     if (!$dst_is_iscsi && $is_zero_initialized) {
6764         push @$cmd, "zeroinit:$dst_path";
6765     } else {
6766         push @$cmd, $dst_path;
6767     }
6768
6769     my $parser = sub {
6770         my $line = shift;
6771         if($line =~ m/\((\S+)\/100\%\)/){
6772             my $percent = $1;
6773             my $transferred = int($size * $percent / 100);
6774             my $remaining = $size - $transferred;
6775
6776             print "transferred: $transferred bytes remaining: $remaining bytes total: $size bytes progression: $percent %\n";
6777         }
6778
6779     };
6780
6781     eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
6782     my $err = $@;
6783     die "copy failed: $err" if $err;
6784 }
6785
6786 sub qemu_img_format {
6787     my ($scfg, $volname) = @_;
6788
6789     if ($scfg->{path} && $volname =~ m/\.($QEMU_FORMAT_RE)$/) {
6790         return $1;
6791     } else {
6792         return "raw";
6793     }
6794 }
6795
6796 sub qemu_drive_mirror {
6797     my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $skipcomplete, $qga, $bwlimit) = @_;
6798
6799     $jobs = {} if !$jobs;
6800
6801     my $qemu_target;
6802     my $format;
6803     $jobs->{"drive-$drive"} = {};
6804
6805     if ($dst_volid =~ /^nbd:/) {
6806         $qemu_target = $dst_volid;
6807         $format = "nbd";
6808     } else {
6809         my $storecfg = PVE::Storage::config();
6810         my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
6811
6812         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6813
6814         $format = qemu_img_format($dst_scfg, $dst_volname);
6815
6816         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6817
6818         $qemu_target = $is_zero_initialized ? "zeroinit:$dst_path" : $dst_path;
6819     }
6820
6821     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $qemu_target };
6822     $opts->{format} = $format if $format;
6823
6824     if (defined($bwlimit)) {
6825         $opts->{speed} = $bwlimit * 1024;
6826         print "drive mirror is starting for drive-$drive with bandwidth limit: ${bwlimit} KB/s\n";
6827     } else {
6828         print "drive mirror is starting for drive-$drive\n";
6829     }
6830
6831     # if a job already runs for this device we get an error, catch it for cleanup
6832     eval { mon_cmd($vmid, "drive-mirror", %$opts); };
6833     if (my $err = $@) {
6834         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6835         warn "$@\n" if $@;
6836         die "mirroring error: $err\n";
6837     }
6838
6839     qemu_drive_mirror_monitor ($vmid, $vmiddst, $jobs, $skipcomplete, $qga);
6840 }
6841
6842 sub qemu_drive_mirror_monitor {
6843     my ($vmid, $vmiddst, $jobs, $skipcomplete, $qga) = @_;
6844
6845     eval {
6846         my $err_complete = 0;
6847
6848         while (1) {
6849             die "storage migration timed out\n" if $err_complete > 300;
6850
6851             my $stats = mon_cmd($vmid, "query-block-jobs");
6852
6853             my $running_mirror_jobs = {};
6854             foreach my $stat (@$stats) {
6855                 next if $stat->{type} ne 'mirror';
6856                 $running_mirror_jobs->{$stat->{device}} = $stat;
6857             }
6858
6859             my $readycounter = 0;
6860
6861             foreach my $job (keys %$jobs) {
6862
6863                 if(defined($jobs->{$job}->{complete}) && !defined($running_mirror_jobs->{$job})) {
6864                     print "$job : finished\n";
6865                     delete $jobs->{$job};
6866                     next;
6867                 }
6868
6869                 die "$job: mirroring has been cancelled\n" if !defined($running_mirror_jobs->{$job});
6870
6871                 my $busy = $running_mirror_jobs->{$job}->{busy};
6872                 my $ready = $running_mirror_jobs->{$job}->{ready};
6873                 if (my $total = $running_mirror_jobs->{$job}->{len}) {
6874                     my $transferred = $running_mirror_jobs->{$job}->{offset} || 0;
6875                     my $remaining = $total - $transferred;
6876                     my $percent = sprintf "%.2f", ($transferred * 100 / $total);
6877
6878                     print "$job: transferred: $transferred bytes remaining: $remaining bytes total: $total bytes progression: $percent % busy: $busy ready: $ready \n";
6879                 }
6880
6881                 $readycounter++ if $running_mirror_jobs->{$job}->{ready};
6882             }
6883
6884             last if scalar(keys %$jobs) == 0;
6885
6886             if ($readycounter == scalar(keys %$jobs)) {
6887                 print "all mirroring jobs are ready \n";
6888                 last if $skipcomplete; #do the complete later
6889
6890                 if ($vmiddst && $vmiddst != $vmid) {
6891                     my $agent_running = $qga && qga_check_running($vmid);
6892                     if ($agent_running) {
6893                         print "freeze filesystem\n";
6894                         eval { mon_cmd($vmid, "guest-fsfreeze-freeze"); };
6895                     } else {
6896                         print "suspend vm\n";
6897                         eval { PVE::QemuServer::vm_suspend($vmid, 1); };
6898                     }
6899
6900                     # if we clone a disk for a new target vm, we don't switch the disk
6901                     PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs);
6902
6903                     if ($agent_running) {
6904                         print "unfreeze filesystem\n";
6905                         eval { mon_cmd($vmid, "guest-fsfreeze-thaw"); };
6906                     } else {
6907                         print "resume vm\n";
6908                         eval {  PVE::QemuServer::vm_resume($vmid, 1, 1); };
6909                     }
6910
6911                     last;
6912                 } else {
6913
6914                     foreach my $job (keys %$jobs) {
6915                         # try to switch the disk if source and destination are on the same guest
6916                         print "$job: Completing block job...\n";
6917
6918                         eval { mon_cmd($vmid, "block-job-complete", device => $job) };
6919                         if ($@ =~ m/cannot be completed/) {
6920                             print "$job: Block job cannot be completed, try again.\n";
6921                             $err_complete++;
6922                         }else {
6923                             print "$job: Completed successfully.\n";
6924                             $jobs->{$job}->{complete} = 1;
6925                         }
6926                     }
6927                 }
6928             }
6929             sleep 1;
6930         }
6931     };
6932     my $err = $@;
6933
6934     if ($err) {
6935         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6936         die "mirroring error: $err";
6937     }
6938
6939 }
6940
6941 sub qemu_blockjobs_cancel {
6942     my ($vmid, $jobs) = @_;
6943
6944     foreach my $job (keys %$jobs) {
6945         print "$job: Cancelling block job\n";
6946         eval { mon_cmd($vmid, "block-job-cancel", device => $job); };
6947         $jobs->{$job}->{cancel} = 1;
6948     }
6949
6950     while (1) {
6951         my $stats = mon_cmd($vmid, "query-block-jobs");
6952
6953         my $running_jobs = {};
6954         foreach my $stat (@$stats) {
6955             $running_jobs->{$stat->{device}} = $stat;
6956         }
6957
6958         foreach my $job (keys %$jobs) {
6959
6960             if (defined($jobs->{$job}->{cancel}) && !defined($running_jobs->{$job})) {
6961                 print "$job: Done.\n";
6962                 delete $jobs->{$job};
6963             }
6964         }
6965
6966         last if scalar(keys %$jobs) == 0;
6967
6968         sleep 1;
6969     }
6970 }
6971
6972 sub clone_disk {
6973     my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
6974         $newvmid, $storage, $format, $full, $newvollist, $jobs, $skipcomplete, $qga, $bwlimit) = @_;
6975
6976     my $newvolid;
6977
6978     if (!$full) {
6979         print "create linked clone of drive $drivename ($drive->{file})\n";
6980         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
6981         push @$newvollist, $newvolid;
6982     } else {
6983
6984         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6985         $storeid = $storage if $storage;
6986
6987         my $dst_format = resolve_dst_disk_format($storecfg, $storeid, $volname, $format);
6988         my ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 3);
6989
6990         print "create full clone of drive $drivename ($drive->{file})\n";
6991         my $name = undef;
6992         if (drive_is_cloudinit($drive)) {
6993             $name = "vm-$newvmid-cloudinit";
6994             $name .= ".$dst_format" if $dst_format ne 'raw';
6995             $snapname = undef;
6996             $size = PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE;
6997         }
6998         $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $newvmid, $dst_format, $name, ($size/1024));
6999         push @$newvollist, $newvolid;
7000
7001         PVE::Storage::activate_volumes($storecfg, [$newvolid]);
7002
7003         if (drive_is_cloudinit($drive)) {
7004             goto no_data_clone;
7005         }
7006
7007         my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
7008         if (!$running || $snapname) {
7009             # TODO: handle bwlimits
7010             qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit);
7011         } else {
7012
7013             my $kvmver = get_running_qemu_version ($vmid);
7014             if (!min_version($kvmver, 2, 7)) {
7015                 die "drive-mirror with iothread requires qemu version 2.7 or higher\n"
7016                     if $drive->{iothread};
7017             }
7018
7019             qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid, $sparseinit, $jobs, $skipcomplete, $qga, $bwlimit);
7020         }
7021     }
7022
7023 no_data_clone:
7024     my ($size) = PVE::Storage::volume_size_info($storecfg, $newvolid, 3);
7025
7026     my $disk = $drive;
7027     $disk->{format} = undef;
7028     $disk->{file} = $newvolid;
7029     $disk->{size} = $size;
7030
7031     return $disk;
7032 }
7033
7034 sub get_running_qemu_version {
7035     my ($vmid) = @_;
7036     my $res = mon_cmd($vmid, "query-version");
7037     return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
7038 }
7039
7040 sub qemu_use_old_bios_files {
7041     my ($machine_type) = @_;
7042
7043     return if !$machine_type;
7044
7045     my $use_old_bios_files = undef;
7046
7047     if ($machine_type =~ m/^(\S+)\.pxe$/) {
7048         $machine_type = $1;
7049         $use_old_bios_files = 1;
7050     } else {
7051         my $version = PVE::QemuServer::Machine::extract_version($machine_type) // kvm_user_version();
7052         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
7053         # load new efi bios files on migration. So this hack is required to allow
7054         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
7055         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
7056         $use_old_bios_files = !min_version($version, 2, 4);
7057     }
7058
7059     return ($use_old_bios_files, $machine_type);
7060 }
7061
7062 sub create_efidisk($$$$$) {
7063     my ($storecfg, $storeid, $vmid, $fmt, $arch) = @_;
7064
7065     my (undef, $ovmf_vars) = get_ovmf_files($arch);
7066     die "EFI vars default image not found\n" if ! -f $ovmf_vars;
7067
7068     my $vars_size_b = -s $ovmf_vars;
7069     my $vars_size = PVE::Tools::convert_size($vars_size_b, 'b' => 'kb');
7070     my $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $fmt, undef, $vars_size);
7071     PVE::Storage::activate_volumes($storecfg, [$volid]);
7072
7073     qemu_img_convert($ovmf_vars, $volid, $vars_size_b, undef, 0);
7074
7075     return ($volid, $vars_size);
7076 }
7077
7078 sub vm_iothreads_list {
7079     my ($vmid) = @_;
7080
7081     my $res = mon_cmd($vmid, 'query-iothreads');
7082
7083     my $iothreads = {};
7084     foreach my $iothread (@$res) {
7085         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
7086     }
7087
7088     return $iothreads;
7089 }
7090
7091 sub scsihw_infos {
7092     my ($conf, $drive) = @_;
7093
7094     my $maxdev = 0;
7095
7096     if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)) {
7097         $maxdev = 7;
7098     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
7099         $maxdev = 1;
7100     } else {
7101         $maxdev = 256;
7102     }
7103
7104     my $controller = int($drive->{index} / $maxdev);
7105     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single') ? "virtioscsi" : "scsihw";
7106
7107     return ($maxdev, $controller, $controller_prefix);
7108 }
7109
7110 sub add_hyperv_enlightenments {
7111     my ($cpuFlags, $winversion, $machine_version, $bios, $gpu_passthrough, $hv_vendor_id) = @_;
7112
7113     return if $winversion < 6;
7114     return if $bios && $bios eq 'ovmf' && $winversion < 8;
7115
7116     if ($gpu_passthrough || defined($hv_vendor_id)) {
7117         $hv_vendor_id //= 'proxmox';
7118         push @$cpuFlags , "hv_vendor_id=$hv_vendor_id";
7119     }
7120
7121     if (min_version($machine_version, 2, 3)) {
7122         push @$cpuFlags , 'hv_spinlocks=0x1fff';
7123         push @$cpuFlags , 'hv_vapic';
7124         push @$cpuFlags , 'hv_time';
7125     } else {
7126         push @$cpuFlags , 'hv_spinlocks=0xffff';
7127     }
7128
7129     if (min_version($machine_version, 2, 6)) {
7130         push @$cpuFlags , 'hv_reset';
7131         push @$cpuFlags , 'hv_vpindex';
7132         push @$cpuFlags , 'hv_runtime';
7133     }
7134
7135     if ($winversion >= 7) {
7136         push @$cpuFlags , 'hv_relaxed';
7137
7138         if (min_version($machine_version, 2, 12)) {
7139             push @$cpuFlags , 'hv_synic';
7140             push @$cpuFlags , 'hv_stimer';
7141         }
7142
7143         if (min_version($machine_version, 3, 1)) {
7144             push @$cpuFlags , 'hv_ipi';
7145         }
7146     }
7147 }
7148
7149 sub windows_version {
7150     my ($ostype) = @_;
7151
7152     return 0 if !$ostype;
7153
7154     my $winversion = 0;
7155
7156     if($ostype eq 'wxp' || $ostype eq 'w2k3' || $ostype eq 'w2k') {
7157         $winversion = 5;
7158     } elsif($ostype eq 'w2k8' || $ostype eq 'wvista') {
7159         $winversion = 6;
7160     } elsif ($ostype =~ m/^win(\d+)$/) {
7161         $winversion = $1;
7162     }
7163
7164     return $winversion;
7165 }
7166
7167 sub resolve_dst_disk_format {
7168         my ($storecfg, $storeid, $src_volname, $format) = @_;
7169         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
7170
7171         if (!$format) {
7172             # if no target format is specified, use the source disk format as hint
7173             if ($src_volname) {
7174                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
7175                 $format = qemu_img_format($scfg, $src_volname);
7176             } else {
7177                 return $defFormat;
7178             }
7179         }
7180
7181         # test if requested format is supported - else use default
7182         my $supported = grep { $_ eq $format } @$validFormats;
7183         $format = $defFormat if !$supported;
7184         return $format;
7185 }
7186
7187 sub resolve_first_disk {
7188     my $conf = shift;
7189     my @disks = PVE::QemuServer::valid_drive_names();
7190     my $firstdisk;
7191     foreach my $ds (reverse @disks) {
7192         next if !$conf->{$ds};
7193         my $disk = PVE::QemuServer::parse_drive($ds, $conf->{$ds});
7194         next if PVE::QemuServer::drive_is_cdrom($disk);
7195         $firstdisk = $ds;
7196     }
7197     return $firstdisk;
7198 }
7199
7200 sub generate_uuid {
7201     my ($uuid, $uuid_str);
7202     UUID::generate($uuid);
7203     UUID::unparse($uuid, $uuid_str);
7204     return $uuid_str;
7205 }
7206
7207 sub generate_smbios1_uuid {
7208     return "uuid=".generate_uuid();
7209 }
7210
7211 sub nbd_stop {
7212     my ($vmid) = @_;
7213
7214     mon_cmd($vmid, 'nbd-server-stop');
7215 }
7216
7217 sub create_reboot_request {
7218     my ($vmid) = @_;
7219     open(my $fh, '>', "/run/qemu-server/$vmid.reboot")
7220         or die "failed to create reboot trigger file: $!\n";
7221     close($fh);
7222 }
7223
7224 sub clear_reboot_request {
7225     my ($vmid) = @_;
7226     my $path = "/run/qemu-server/$vmid.reboot";
7227     my $res = 0;
7228
7229     $res = unlink($path);
7230     die "could not remove reboot request for $vmid: $!"
7231         if !$res && $! != POSIX::ENOENT;
7232
7233     return $res;
7234 }
7235
7236 # bash completion helper
7237
7238 sub complete_backup_archives {
7239     my ($cmdname, $pname, $cvalue) = @_;
7240
7241     my $cfg = PVE::Storage::config();
7242
7243     my $storeid;
7244
7245     if ($cvalue =~ m/^([^:]+):/) {
7246         $storeid = $1;
7247     }
7248
7249     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
7250
7251     my $res = [];
7252     foreach my $id (keys %$data) {
7253         foreach my $item (@{$data->{$id}}) {
7254             next if $item->{format} !~ m/^vma\.(gz|lzo)$/;
7255             push @$res, $item->{volid} if defined($item->{volid});
7256         }
7257     }
7258
7259     return $res;
7260 }
7261
7262 my $complete_vmid_full = sub {
7263     my ($running) = @_;
7264
7265     my $idlist = vmstatus();
7266
7267     my $res = [];
7268
7269     foreach my $id (keys %$idlist) {
7270         my $d = $idlist->{$id};
7271         if (defined($running)) {
7272             next if $d->{template};
7273             next if $running && $d->{status} ne 'running';
7274             next if !$running && $d->{status} eq 'running';
7275         }
7276         push @$res, $id;
7277
7278     }
7279     return $res;
7280 };
7281
7282 sub complete_vmid {
7283     return &$complete_vmid_full();
7284 }
7285
7286 sub complete_vmid_stopped {
7287     return &$complete_vmid_full(0);
7288 }
7289
7290 sub complete_vmid_running {
7291     return &$complete_vmid_full(1);
7292 }
7293
7294 sub complete_storage {
7295
7296     my $cfg = PVE::Storage::config();
7297     my $ids = $cfg->{ids};
7298
7299     my $res = [];
7300     foreach my $sid (keys %$ids) {
7301         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
7302         next if !$ids->{$sid}->{content}->{images};
7303         push @$res, $sid;
7304     }
7305
7306     return $res;
7307 }
7308
7309 sub complete_migration_storage {
7310     my ($cmd, $param, $current_value, $all_args) = @_;
7311
7312     my $targetnode = @$all_args[1];
7313
7314     my $cfg = PVE::Storage::config();
7315     my $ids = $cfg->{ids};
7316
7317     my $res = [];
7318     foreach my $sid (keys %$ids) {
7319         next if !PVE::Storage::storage_check_enabled($cfg, $sid, $targetnode, 1);
7320         next if !$ids->{$sid}->{content}->{images};
7321         push @$res, $sid;
7322     }
7323
7324     return $res;
7325 }
7326
7327 1;