PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5
   6 use POSIX;
   7 use IO::Handle;
   8 use IO::Select;
   9 use IO::File;
  10 use IO::Dir;
  11 use IO::Socket::UNIX;
  12 use File::Basename;
  13 use File::Path;
  14 use File::stat;
  15 use Getopt::Long;
  16 use Digest::SHA;
  17 use Fcntl ':flock';
  18 use Cwd 'abs_path';
  19 use IPC::Open3;
  20 use JSON;
  21 use Fcntl;
  22 use PVE::SafeSyslog;
  23 use Storable qw(dclone);
  24 use MIME::Base64;
  25 use PVE::Exception qw(raise raise_param_exc);
  26 use PVE::Storage;
  27 use PVE::Tools qw(run_command lock_file lock_file_full file_read_firstline dir_glob_foreach $IPV6RE);
  28 use PVE::JSONSchema qw(get_standard_option);
  29 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file cfs_lock_file);
  30 use PVE::INotify;
  31 use PVE::ProcFSTools;
  32 use PVE::QemuConfig;
  33 use PVE::QMPClient;
  34 use PVE::RPCEnvironment;
  35 use PVE::GuestHelpers;
  36 use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr print_pcie_root_port);
  37 use PVE::QemuServer::Memory;
  38 use PVE::QemuServer::USB qw(parse_usb_device);
  39 use PVE::QemuServer::Cloudinit;
  40 use PVE::SysFSTools;
  41 use PVE::Systemd;
  42 use Time::HiRes qw(gettimeofday);
  43 use File::Copy qw(copy);
  44 use URI::Escape;
  45
  46 my $EDK2_FW_BASE = '/usr/share/pve-edk2-firmware/';
  47 my $OVMF = {
  48     x86_64 => [
  49         "$EDK2_FW_BASE/OVMF_CODE.fd",
  50         "$EDK2_FW_BASE/OVMF_VARS.fd"
  51     ],
  52     aarch64 => [
  53         "$EDK2_FW_BASE/AAVMF_CODE.fd",
  54         "$EDK2_FW_BASE/AAVMF_VARS.fd"
  55     ],
  56 };
  57
  58 my $qemu_snap_storage = { rbd => 1 };
  59
  60 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  61
  62 my $QEMU_FORMAT_RE = qr/raw|cow|qcow|qcow2|qed|vmdk|cloop/;
  63
  64 # Note about locking: we use flock on the config file protect
  65 # against concurent actions.
  66 # Aditionaly, we have a 'lock' setting in the config file. This
  67 # can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
  68 # allowed when such lock is set. But you can ignore this kind of
  69 # lock with the --skiplock flag.
  70
  71 cfs_register_file('/qemu-server/',
  72                   \&parse_vm_config,
  73                   \&write_vm_config);
  74
  75 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
  76     description => "Some command save/restore state from this location.",
  77     type => 'string',
  78     maxLength => 128,
  79     optional => 1,
  80 });
  81
  82 PVE::JSONSchema::register_standard_option('pve-qm-image-format', {
  83     type => 'string',
  84     enum => [qw(raw cow qcow qed qcow2 vmdk cloop)],
  85     description => "The drive's backing file's data format.",
  86     optional => 1,
  87 });
  88
  89 PVE::JSONSchema::register_standard_option('pve-qemu-machine', {
  90         description => "Specifies the Qemu machine type.",
  91         type => 'string',
  92         pattern => '(pc|pc(-i440fx)?-\d+\.\d+(\.pxe)?|q35|pc-q35-\d+\.\d+(\.pxe)?|virt(?:-\d+\.\d+)?)',
  93         maxLength => 40,
  94         optional => 1,
  95 });
  96
  97 #no warnings 'redefine';
  98
  99 sub cgroups_write {
 100    my ($controller, $vmid, $option, $value) = @_;
 101
 102    my $path = "/sys/fs/cgroup/$controller/qemu.slice/$vmid.scope/$option";
 103    PVE::ProcFSTools::write_proc_entry($path, $value);
 104
 105 }
 106
 107 my $nodename = PVE::INotify::nodename();
 108
 109 mkdir "/etc/pve/nodes/$nodename";
 110 my $confdir = "/etc/pve/nodes/$nodename/qemu-server";
 111 mkdir $confdir;
 112
 113 my $var_run_tmpdir = "/var/run/qemu-server";
 114 mkdir $var_run_tmpdir;
 115
 116 my $lock_dir = "/var/lock/qemu-server";
 117 mkdir $lock_dir;
 118
 119 my $cpu_vendor_list = {
 120     # Intel CPUs
 121     486 => 'GenuineIntel',
 122     pentium => 'GenuineIntel',
 123     pentium2  => 'GenuineIntel',
 124     pentium3  => 'GenuineIntel',
 125     coreduo => 'GenuineIntel',
 126     core2duo => 'GenuineIntel',
 127     Conroe  => 'GenuineIntel',
 128     Penryn  => 'GenuineIntel',
 129     Nehalem  => 'GenuineIntel',
 130     'Nehalem-IBRS'  => 'GenuineIntel',
 131     Westmere => 'GenuineIntel',
 132     'Westmere-IBRS' => 'GenuineIntel',
 133     SandyBridge => 'GenuineIntel',
 134     'SandyBridge-IBRS' => 'GenuineIntel',
 135     IvyBridge => 'GenuineIntel',
 136     'IvyBridge-IBRS' => 'GenuineIntel',
 137     Haswell => 'GenuineIntel',
 138     'Haswell-IBRS' => 'GenuineIntel',
 139     'Haswell-noTSX' => 'GenuineIntel',
 140     'Haswell-noTSX-IBRS' => 'GenuineIntel',
 141     Broadwell => 'GenuineIntel',
 142     'Broadwell-IBRS' => 'GenuineIntel',
 143     'Broadwell-noTSX' => 'GenuineIntel',
 144     'Broadwell-noTSX-IBRS' => 'GenuineIntel',
 145     'Skylake-Client' => 'GenuineIntel',
 146     'Skylake-Client-IBRS' => 'GenuineIntel',
 147     'Skylake-Server' => 'GenuineIntel',
 148     'Skylake-Server-IBRS' => 'GenuineIntel',
 149     'Cascadelake-Server' => 'GenuineIntel',
 150     KnightsMill => 'GenuineIntel',
 151
 152
 153     # AMD CPUs
 154     athlon => 'AuthenticAMD',
 155     phenom  => 'AuthenticAMD',
 156     Opteron_G1  => 'AuthenticAMD',
 157     Opteron_G2  => 'AuthenticAMD',
 158     Opteron_G3  => 'AuthenticAMD',
 159     Opteron_G4  => 'AuthenticAMD',
 160     Opteron_G5  => 'AuthenticAMD',
 161     EPYC => 'AuthenticAMD',
 162     'EPYC-IBPB' => 'AuthenticAMD',
 163
 164     # generic types, use vendor from host node
 165     host => 'default',
 166     kvm32 => 'default',
 167     kvm64 => 'default',
 168     qemu32 => 'default',
 169     qemu64 => 'default',
 170     max => 'default',
 171 };
 172
 173 my @supported_cpu_flags = (
 174     'pcid',
 175     'spec-ctrl',
 176     'ibpb',
 177     'ssbd',
 178     'virt-ssbd',
 179     'amd-ssbd',
 180     'amd-no-ssb',
 181     'pdpe1gb',
 182     'md-clear',
 183     'hv-tlbflush',
 184     'hv-evmcs',
 185     'aes'
 186 );
 187 my $cpu_flag = qr/[+-](@{[join('|', @supported_cpu_flags)]})/;
 188
 189 my $cpu_fmt = {
 190     cputype => {
 191         description => "Emulated CPU type.",
 192         type => 'string',
 193         enum => [ sort { "\L$a" cmp "\L$b" } keys %$cpu_vendor_list ],
 194         default => 'kvm64',
 195         default_key => 1,
 196     },
 197     hidden => {
 198         description => "Do not identify as a KVM virtual machine.",
 199         type => 'boolean',
 200         optional => 1,
 201         default => 0
 202     },
 203     'hv-vendor-id' => {
 204         type => 'string',
 205         pattern => qr/[a-zA-Z0-9]{1,12}/,
 206         format_description => 'vendor-id',
 207         description => 'The Hyper-V vendor ID. Some drivers or programs inside Windows guests need a specific ID.',
 208         optional => 1,
 209     },
 210     flags => {
 211         description => "List of additional CPU flags separated by ';'."
 212                      . " Use '+FLAG' to enable, '-FLAG' to disable a flag."
 213                      . " Currently supported flags: @{[join(', ', @supported_cpu_flags)]}.",
 214         format_description => '+FLAG[;-FLAG...]',
 215         type => 'string',
 216         pattern => qr/$cpu_flag(;$cpu_flag)*/,
 217         optional => 1,
 218     },
 219 };
 220
 221 my $watchdog_fmt = {
 222     model => {
 223         default_key => 1,
 224         type => 'string',
 225         enum => [qw(i6300esb ib700)],
 226         description => "Watchdog type to emulate.",
 227         default => 'i6300esb',
 228         optional => 1,
 229     },
 230     action => {
 231         type => 'string',
 232         enum => [qw(reset shutdown poweroff pause debug none)],
 233         description => "The action to perform if after activation the guest fails to poll the watchdog in time.",
 234         optional => 1,
 235     },
 236 };
 237 PVE::JSONSchema::register_format('pve-qm-watchdog', $watchdog_fmt);
 238
 239 my $agent_fmt = {
 240     enabled => {
 241         description => "Enable/disable Qemu GuestAgent.",
 242         type => 'boolean',
 243         default => 0,
 244         default_key => 1,
 245     },
 246     fstrim_cloned_disks => {
 247         description => "Run fstrim after cloning/moving a disk.",
 248         type => 'boolean',
 249         optional => 1,
 250         default => 0
 251     },
 252 };
 253
 254 my $vga_fmt = {
 255     type => {
 256         description => "Select the VGA type.",
 257         type => 'string',
 258         default => 'std',
 259         optional => 1,
 260         default_key => 1,
 261         enum => [qw(cirrus qxl qxl2 qxl3 qxl4 none serial0 serial1 serial2 serial3 std virtio vmware)],
 262     },
 263     memory => {
 264         description => "Sets the VGA memory (in MiB). Has no effect with serial display.",
 265         type => 'integer',
 266         optional => 1,
 267         minimum => 4,
 268         maximum => 512,
 269     },
 270 };
 271
 272 my $ivshmem_fmt = {
 273     size => {
 274         type => 'integer',
 275         minimum => 1,
 276         description => "The size of the file in MB.",
 277     },
 278     name => {
 279         type => 'string',
 280         pattern => '[a-zA-Z0-9\-]+',
 281         optional => 1,
 282         format_description => 'string',
 283         description => "The name of the file. Will be prefixed with 'pve-shm-'. Default is the VMID. Will be deleted when the VM is stopped.",
 284     },
 285 };
 286
 287 my $audio_fmt = {
 288     device => {
 289         type => 'string',
 290         enum => [qw(ich9-intel-hda intel-hda AC97)],
 291         description =>  "Configure an audio device."
 292     },
 293     driver =>  {
 294         type => 'string',
 295         enum => ['spice'],
 296         default => 'spice',
 297         optional => 1,
 298         description => "Driver backend for the audio device."
 299     },
 300 };
 301
 302 my $spice_enhancements_fmt = {
 303     foldersharing => {
 304         type => 'boolean',
 305         optional => 1,
 306         default => '0',
 307         description =>  "Enable folder sharing via SPICE. Needs Spice-WebDAV daemon installed in the VM."
 308     },
 309     videostreaming =>  {
 310         type => 'string',
 311         enum => ['off', 'all', 'filter'],
 312         default => 'off',
 313         optional => 1,
 314         description => "Enable video streaming. Uses compression for detected video streams."
 315     },
 316 };
 317
 318 my $confdesc = {
 319     onboot => {
 320         optional => 1,
 321         type => 'boolean',
 322         description => "Specifies whether a VM will be started during system bootup.",
 323         default => 0,
 324     },
 325     autostart => {
 326         optional => 1,
 327         type => 'boolean',
 328         description => "Automatic restart after crash (currently ignored).",
 329         default => 0,
 330     },
 331     hotplug => {
 332         optional => 1,
 333         type => 'string', format => 'pve-hotplug-features',
 334         description => "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
 335         default => 'network,disk,usb',
 336     },
 337     reboot => {
 338         optional => 1,
 339         type => 'boolean',
 340         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 341         default => 1,
 342     },
 343     lock => {
 344         optional => 1,
 345         type => 'string',
 346         description => "Lock/unlock the VM.",
 347         enum => [qw(backup clone create migrate rollback snapshot snapshot-delete suspending suspended)],
 348     },
 349     cpulimit => {
 350         optional => 1,
 351         type => 'number',
 352         description => "Limit of CPU usage.",
 353         verbose_description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
 354         minimum => 0,
 355         maximum => 128,
 356         default => 0,
 357     },
 358     cpuunits => {
 359         optional => 1,
 360         type => 'integer',
 361         description => "CPU weight for a VM.",
 362         verbose_description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.",
 363         minimum => 2,
 364         maximum => 262144,
 365         default => 1024,
 366     },
 367     memory => {
 368         optional => 1,
 369         type => 'integer',
 370         description => "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
 371         minimum => 16,
 372         default => 512,
 373     },
 374     balloon => {
 375         optional => 1,
 376         type => 'integer',
 377         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 378         minimum => 0,
 379     },
 380     shares => {
 381         optional => 1,
 382         type => 'integer',
 383         description => "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning. Auto-ballooning is done by pvestatd.",
 384         minimum => 0,
 385         maximum => 50000,
 386         default => 1000,
 387     },
 388     keyboard => {
 389         optional => 1,
 390         type => 'string',
 391         description => "Keybord layout for vnc server. Default is read from the '/etc/pve/datacenter.cfg' configuration file.".
 392                        "It should not be necessary to set it.",
 393         enum => PVE::Tools::kvmkeymaplist(),
 394         default => undef,
 395     },
 396     name => {
 397         optional => 1,
 398         type => 'string', format => 'dns-name',
 399         description => "Set a name for the VM. Only used on the configuration web interface.",
 400     },
 401     scsihw => {
 402         optional => 1,
 403         type => 'string',
 404         description => "SCSI controller model",
 405         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 406         default => 'lsi',
 407     },
 408     description => {
 409         optional => 1,
 410         type => 'string',
 411         description => "Description for the VM. Only used on the configuration web interface. This is saved as comment inside the configuration file.",
 412     },
 413     ostype => {
 414         optional => 1,
 415         type => 'string',
 416         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 l24 l26 solaris)],
 417         description => "Specify guest operating system.",
 418         verbose_description => <<EODESC,
 419 Specify guest operating system. This is used to enable special
 420 optimization/features for specific operating systems:
 421
 422 [horizontal]
 423 other;; unspecified OS
 424 wxp;; Microsoft Windows XP
 425 w2k;; Microsoft Windows 2000
 426 w2k3;; Microsoft Windows 2003
 427 w2k8;; Microsoft Windows 2008
 428 wvista;; Microsoft Windows Vista
 429 win7;; Microsoft Windows 7
 430 win8;; Microsoft Windows 8/2012/2012r2
 431 win10;; Microsoft Windows 10/2016
 432 l24;; Linux 2.4 Kernel
 433 l26;; Linux 2.6/3.X Kernel
 434 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
 435 EODESC
 436     },
 437     boot => {
 438         optional => 1,
 439         type => 'string',
 440         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n).",
 441         pattern => '[acdn]{1,4}',
 442         default => 'cdn',
 443     },
 444     bootdisk => {
 445         optional => 1,
 446         type => 'string', format => 'pve-qm-bootdisk',
 447         description => "Enable booting from specified disk.",
 448         pattern => '(ide|sata|scsi|virtio)\d+',
 449     },
 450     smp => {
 451         optional => 1,
 452         type => 'integer',
 453         description => "The number of CPUs. Please use option -sockets instead.",
 454         minimum => 1,
 455         default => 1,
 456     },
 457     sockets => {
 458         optional => 1,
 459         type => 'integer',
 460         description => "The number of CPU sockets.",
 461         minimum => 1,
 462         default => 1,
 463     },
 464     cores => {
 465         optional => 1,
 466         type => 'integer',
 467         description => "The number of cores per socket.",
 468         minimum => 1,
 469         default => 1,
 470     },
 471     numa => {
 472         optional => 1,
 473         type => 'boolean',
 474         description => "Enable/disable NUMA.",
 475         default => 0,
 476     },
 477     hugepages => {
 478         optional => 1,
 479         type => 'string',
 480         description => "Enable/disable hugepages memory.",
 481         enum => [qw(any 2 1024)],
 482     },
 483     vcpus => {
 484         optional => 1,
 485         type => 'integer',
 486         description => "Number of hotplugged vcpus.",
 487         minimum => 1,
 488         default => 0,
 489     },
 490     acpi => {
 491         optional => 1,
 492         type => 'boolean',
 493         description => "Enable/disable ACPI.",
 494         default => 1,
 495     },
 496     agent => {
 497         optional => 1,
 498         description => "Enable/disable Qemu GuestAgent and its properties.",
 499         type => 'string',
 500         format => $agent_fmt,
 501     },
 502     kvm => {
 503         optional => 1,
 504         type => 'boolean',
 505         description => "Enable/disable KVM hardware virtualization.",
 506         default => 1,
 507     },
 508     tdf => {
 509         optional => 1,
 510         type => 'boolean',
 511         description => "Enable/disable time drift fix.",
 512         default => 0,
 513     },
 514     localtime => {
 515         optional => 1,
 516         type => 'boolean',
 517         description => "Set the real time clock to local time. This is enabled by default if ostype indicates a Microsoft OS.",
 518     },
 519     freeze => {
 520         optional => 1,
 521         type => 'boolean',
 522         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 523     },
 524     vga => {
 525         optional => 1,
 526         type => 'string', format => $vga_fmt,
 527         description => "Configure the VGA hardware.",
 528         verbose_description => "Configure the VGA Hardware. If you want to use ".
 529             "high resolution modes (>= 1280x1024x16) you may need to increase " .
 530             "the vga memory option. Since QEMU 2.9 the default VGA display type " .
 531             "is 'std' for all OS types besides some Windows versions (XP and " .
 532             "older) which use 'cirrus'. The 'qxl' option enables the SPICE " .
 533             "display server. For win* OS you can select how many independent " .
 534             "displays you want, Linux guests can add displays them self.\n".
 535             "You can also run without any graphic card, using a serial device as terminal.",
 536     },
 537     watchdog => {
 538         optional => 1,
 539         type => 'string', format => 'pve-qm-watchdog',
 540         description => "Create a virtual hardware watchdog device.",
 541         verbose_description => "Create a virtual hardware watchdog device. Once enabled" .
 542             " (by a guest action), the watchdog must be periodically polled " .
 543             "by an agent inside the guest or else the watchdog will reset " .
 544             "the guest (or execute the respective action specified)",
 545     },
 546     startdate => {
 547         optional => 1,
 548         type => 'string',
 549         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 550         description => "Set the initial date of the real time clock. Valid format for date are: 'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 551         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 552         default => 'now',
 553     },
 554     startup =>  get_standard_option('pve-startup-order'),
 555     template => {
 556         optional => 1,
 557         type => 'boolean',
 558         description => "Enable/disable Template.",
 559         default => 0,
 560     },
 561     args => {
 562         optional => 1,
 563         type => 'string',
 564         description => "Arbitrary arguments passed to kvm.",
 565         verbose_description => <<EODESCR,
 566 Arbitrary arguments passed to kvm, for example:
 567
 568 args: -no-reboot -no-hpet
 569
 570 NOTE: this option is for experts only.
 571 EODESCR
 572     },
 573     tablet => {
 574         optional => 1,
 575         type => 'boolean',
 576         default => 1,
 577         description => "Enable/disable the USB tablet device.",
 578         verbose_description => "Enable/disable the USB tablet device. This device is " .
 579             "usually needed to allow absolute mouse positioning with VNC. " .
 580             "Else the mouse runs out of sync with normal VNC clients. " .
 581             "If you're running lots of console-only guests on one host, " .
 582             "you may consider disabling this to save some context switches. " .
 583             "This is turned off by default if you use spice (-vga=qxl).",
 584     },
 585     migrate_speed => {
 586         optional => 1,
 587         type => 'integer',
 588         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 589         minimum => 0,
 590         default => 0,
 591     },
 592     migrate_downtime => {
 593         optional => 1,
 594         type => 'number',
 595         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 596         minimum => 0,
 597         default => 0.1,
 598     },
 599     cdrom => {
 600         optional => 1,
 601         type => 'string', format => 'pve-qm-ide',
 602         typetext => '<volume>',
 603         description => "This is an alias for option -ide2",
 604     },
 605     cpu => {
 606         optional => 1,
 607         description => "Emulated CPU type.",
 608         type => 'string',
 609         format => $cpu_fmt,
 610     },
 611     parent => get_standard_option('pve-snapshot-name', {
 612         optional => 1,
 613         description => "Parent snapshot name. This is used internally, and should not be modified.",
 614     }),
 615     snaptime => {
 616         optional => 1,
 617         description => "Timestamp for snapshots.",
 618         type => 'integer',
 619         minimum => 0,
 620     },
 621     vmstate => {
 622         optional => 1,
 623         type => 'string', format => 'pve-volume-id',
 624         description => "Reference to a volume which stores the VM state. This is used internally for snapshots.",
 625     },
 626     vmstatestorage => get_standard_option('pve-storage-id', {
 627         description => "Default storage for VM state volumes/files.",
 628         optional => 1,
 629     }),
 630     runningmachine => get_standard_option('pve-qemu-machine', {
 631         description => "Specifies the Qemu machine type of the running vm. This is used internally for snapshots.",
 632     }),
 633     machine => get_standard_option('pve-qemu-machine'),
 634     arch => {
 635         description => "Virtual processor architecture. Defaults to the host.",
 636         optional => 1,
 637         type => 'string',
 638         enum => [qw(x86_64 aarch64)],
 639     },
 640     smbios1 => {
 641         description => "Specify SMBIOS type 1 fields.",
 642         type => 'string', format => 'pve-qm-smbios1',
 643         maxLength => 512,
 644         optional => 1,
 645     },
 646     protection => {
 647         optional => 1,
 648         type => 'boolean',
 649         description => "Sets the protection flag of the VM. This will disable the remove VM and remove disk operations.",
 650         default => 0,
 651     },
 652     bios => {
 653         optional => 1,
 654         type => 'string',
 655         enum => [ qw(seabios ovmf) ],
 656         description => "Select BIOS implementation.",
 657         default => 'seabios',
 658     },
 659     vmgenid => {
 660         type => 'string',
 661         pattern => '(?:[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}|[01])',
 662         format_description => 'UUID',
 663         description => "Set VM Generation ID. Use '1' to autogenerate on create or update, pass '0' to disable explicitly.",
 664         verbose_description => "The VM generation ID (vmgenid) device exposes a".
 665             " 128-bit integer value identifier to the guest OS. This allows to".
 666             " notify the guest operating system when the virtual machine is".
 667             " executed with a different configuration (e.g. snapshot execution".
 668             " or creation from a template). The guest operating system notices".
 669             " the change, and is then able to react as appropriate by marking".
 670             " its copies of distributed databases as dirty, re-initializing its".
 671             " random number generator, etc.\n".
 672             "Note that auto-creation only works when done throug API/CLI create".
 673             " or update methods, but not when manually editing the config file.",
 674         default => "1 (autogenerated)",
 675         optional => 1,
 676     },
 677     hookscript => {
 678         type => 'string',
 679         format => 'pve-volume-id',
 680         optional => 1,
 681         description => "Script that will be executed during various steps in the vms lifetime.",
 682     },
 683     ivshmem => {
 684         type => 'string',
 685         format => $ivshmem_fmt,
 686         description => "Inter-VM shared memory. Useful for direct communication between VMs, or to the host.",
 687         optional => 1,
 688     },
 689     audio0 => {
 690         type => 'string',
 691         format => $audio_fmt,
 692         description => "Configure a audio device, useful in combination with QXL/Spice.",
 693         optional => 1
 694     },
 695     spice_enhancements => {
 696         type => 'string',
 697         format => $spice_enhancements_fmt,
 698         description => "Configure additional enhancements for SPICE.",
 699         optional => 1
 700     },
 701 };
 702
 703 my $cicustom_fmt = {
 704     meta => {
 705         type => 'string',
 706         optional => 1,
 707         description => 'Specify a custom file containing all meta data passed to the VM via cloud-init. This is provider specific meaning configdrive2 and nocloud differ.',
 708         format => 'pve-volume-id',
 709         format_description => 'volume',
 710     },
 711     network => {
 712         type => 'string',
 713         optional => 1,
 714         description => 'Specify a custom file containing all network data passed to the VM via cloud-init.',
 715         format => 'pve-volume-id',
 716         format_description => 'volume',
 717     },
 718     user => {
 719         type => 'string',
 720         optional => 1,
 721         description => 'Specify a custom file containing all user data passed to the VM via cloud-init.',
 722         format => 'pve-volume-id',
 723         format_description => 'volume',
 724     },
 725 };
 726 PVE::JSONSchema::register_format('pve-qm-cicustom', $cicustom_fmt);
 727
 728 my $confdesc_cloudinit = {
 729     citype => {
 730         optional => 1,
 731         type => 'string',
 732         description => 'Specifies the cloud-init configuration format. The default depends on the configured operating system type (`ostype`. We use the `nocloud` format for Linux, and `configdrive2` for windows.',
 733         enum => ['configdrive2', 'nocloud'],
 734     },
 735     ciuser => {
 736         optional => 1,
 737         type => 'string',
 738         description => "cloud-init: User name to change ssh keys and password for instead of the image's configured default user.",
 739     },
 740     cipassword => {
 741         optional => 1,
 742         type => 'string',
 743         description => 'cloud-init: Password to assign the user. Using this is generally not recommended. Use ssh keys instead. Also note that older cloud-init versions do not support hashed passwords.',
 744     },
 745     cicustom => {
 746         optional => 1,
 747         type => 'string',
 748         description => 'cloud-init: Specify custom files to replace the automatically generated ones at start.',
 749         format => 'pve-qm-cicustom',
 750     },
 751     searchdomain => {
 752         optional => 1,
 753         type => 'string',
 754         description => "cloud-init: Sets DNS search domains for a container. Create will automatically use the setting from the host if neither searchdomain nor nameserver are set.",
 755     },
 756     nameserver => {
 757         optional => 1,
 758         type => 'string', format => 'address-list',
 759         description => "cloud-init: Sets DNS server IP address for a container. Create will automatically use the setting from the host if neither searchdomain nor nameserver are set.",
 760     },
 761     sshkeys => {
 762         optional => 1,
 763         type => 'string',
 764         format => 'urlencoded',
 765         description => "cloud-init: Setup public SSH keys (one key per line, OpenSSH format).",
 766     },
 767 };
 768
 769 # what about other qemu settings ?
 770 #cpu => 'string',
 771 #machine => 'string',
 772 #fda => 'file',
 773 #fdb => 'file',
 774 #mtdblock => 'file',
 775 #sd => 'file',
 776 #pflash => 'file',
 777 #snapshot => 'bool',
 778 #bootp => 'file',
 779 ##tftp => 'dir',
 780 ##smb => 'dir',
 781 #kernel => 'file',
 782 #append => 'string',
 783 #initrd => 'file',
 784 ##soundhw => 'string',
 785
 786 while (my ($k, $v) = each %$confdesc) {
 787     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 788 }
 789
 790 my $MAX_IDE_DISKS = 4;
 791 my $MAX_SCSI_DISKS = 14;
 792 my $MAX_VIRTIO_DISKS = 16;
 793 my $MAX_SATA_DISKS = 6;
 794 my $MAX_USB_DEVICES = 5;
 795 my $MAX_NETS = 32;
 796 my $MAX_UNUSED_DISKS = 256;
 797 my $MAX_HOSTPCI_DEVICES = 16;
 798 my $MAX_SERIAL_PORTS = 4;
 799 my $MAX_PARALLEL_PORTS = 3;
 800 my $MAX_NUMA = 8;
 801
 802 my $numa_fmt = {
 803     cpus => {
 804         type => "string",
 805         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 806         description => "CPUs accessing this NUMA node.",
 807         format_description => "id[-id];...",
 808     },
 809     memory => {
 810         type => "number",
 811         description => "Amount of memory this NUMA node provides.",
 812         optional => 1,
 813     },
 814     hostnodes => {
 815         type => "string",
 816         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 817         description => "Host NUMA nodes to use.",
 818         format_description => "id[-id];...",
 819         optional => 1,
 820     },
 821     policy => {
 822         type => 'string',
 823         enum => [qw(preferred bind interleave)],
 824         description => "NUMA allocation policy.",
 825         optional => 1,
 826     },
 827 };
 828 PVE::JSONSchema::register_format('pve-qm-numanode', $numa_fmt);
 829 my $numadesc = {
 830     optional => 1,
 831     type => 'string', format => $numa_fmt,
 832     description => "NUMA topology.",
 833 };
 834 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 835
 836 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 837     $confdesc->{"numa$i"} = $numadesc;
 838 }
 839
 840 my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000',  'pcnet',  'virtio',
 841                       'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3',
 842                       'e1000-82540em', 'e1000-82544gc', 'e1000-82545em'];
 843 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 844
 845 my $net_fmt_bridge_descr = <<__EOD__;
 846 Bridge to attach the network device to. The Proxmox VE standard bridge
 847 is called 'vmbr0'.
 848
 849 If you do not specify a bridge, we create a kvm user (NATed) network
 850 device, which provides DHCP and DNS services. The following addresses
 851 are used:
 852
 853  10.0.2.2   Gateway
 854  10.0.2.3   DNS Server
 855  10.0.2.4   SMB Server
 856
 857 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 858 __EOD__
 859
 860 my $net_fmt = {
 861     macaddr  => get_standard_option('mac-addr', {
 862         description => "MAC address. That address must be unique withing your network. This is automatically generated if not specified.",
 863     }),
 864     model => {
 865         type => 'string',
 866         description => "Network Card Model. The 'virtio' model provides the best performance with very low CPU overhead. If your guest does not support this driver, it is usually best to use 'e1000'.",
 867         enum => $nic_model_list,
 868         default_key => 1,
 869     },
 870     (map { $_ => { keyAlias => 'model', alias => 'macaddr' }} @$nic_model_list),
 871     bridge => {
 872         type => 'string',
 873         description => $net_fmt_bridge_descr,
 874         format_description => 'bridge',
 875         optional => 1,
 876     },
 877     queues => {
 878         type => 'integer',
 879         minimum => 0, maximum => 16,
 880         description => 'Number of packet queues to be used on the device.',
 881         optional => 1,
 882     },
 883     rate => {
 884         type => 'number',
 885         minimum => 0,
 886         description => "Rate limit in mbps (megabytes per second) as floating point number.",
 887         optional => 1,
 888     },
 889     tag => {
 890         type => 'integer',
 891         minimum => 1, maximum => 4094,
 892         description => 'VLAN tag to apply to packets on this interface.',
 893         optional => 1,
 894     },
 895     trunks => {
 896         type => 'string',
 897         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 898         description => 'VLAN trunks to pass through this interface.',
 899         format_description => 'vlanid[;vlanid...]',
 900         optional => 1,
 901     },
 902     firewall => {
 903         type => 'boolean',
 904         description => 'Whether this interface should be protected by the firewall.',
 905         optional => 1,
 906     },
 907     link_down => {
 908         type => 'boolean',
 909         description => 'Whether this interface should be disconnected (like pulling the plug).',
 910         optional => 1,
 911     },
 912 };
 913
 914 my $netdesc = {
 915     optional => 1,
 916     type => 'string', format => $net_fmt,
 917     description => "Specify network devices.",
 918 };
 919
 920 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 921
 922 my $ipconfig_fmt = {
 923     ip => {
 924         type => 'string',
 925         format => 'pve-ipv4-config',
 926         format_description => 'IPv4Format/CIDR',
 927         description => 'IPv4 address in CIDR format.',
 928         optional => 1,
 929         default => 'dhcp',
 930     },
 931     gw => {
 932         type => 'string',
 933         format => 'ipv4',
 934         format_description => 'GatewayIPv4',
 935         description => 'Default gateway for IPv4 traffic.',
 936         optional => 1,
 937         requires => 'ip',
 938     },
 939     ip6 => {
 940         type => 'string',
 941         format => 'pve-ipv6-config',
 942         format_description => 'IPv6Format/CIDR',
 943         description => 'IPv6 address in CIDR format.',
 944         optional => 1,
 945         default => 'dhcp',
 946     },
 947     gw6 => {
 948         type => 'string',
 949         format => 'ipv6',
 950         format_description => 'GatewayIPv6',
 951         description => 'Default gateway for IPv6 traffic.',
 952         optional => 1,
 953         requires => 'ip6',
 954     },
 955 };
 956 PVE::JSONSchema::register_format('pve-qm-ipconfig', $ipconfig_fmt);
 957 my $ipconfigdesc = {
 958     optional => 1,
 959     type => 'string', format => 'pve-qm-ipconfig',
 960     description => <<'EODESCR',
 961 cloud-init: Specify IP addresses and gateways for the corresponding interface.
 962
 963 IP addresses use CIDR notation, gateways are optional but need an IP of the same type specified.
 964
 965 The special string 'dhcp' can be used for IP addresses to use DHCP, in which case no explicit gateway should be provided.
 966 For IPv6 the special string 'auto' can be used to use stateless autoconfiguration.
 967
 968 If cloud-init is enabled and neither an IPv4 nor an IPv6 address is specified, it defaults to using dhcp on IPv4.
 969 EODESCR
 970 };
 971 PVE::JSONSchema::register_standard_option("pve-qm-ipconfig", $netdesc);
 972
 973 for (my $i = 0; $i < $MAX_NETS; $i++)  {
 974     $confdesc->{"net$i"} = $netdesc;
 975     $confdesc_cloudinit->{"ipconfig$i"} = $ipconfigdesc;
 976 }
 977
 978 foreach my $key (keys %$confdesc_cloudinit) {
 979     $confdesc->{$key} = $confdesc_cloudinit->{$key};
 980 }
 981
 982 PVE::JSONSchema::register_format('pve-volume-id-or-qm-path', \&verify_volume_id_or_qm_path);
 983 sub verify_volume_id_or_qm_path {
 984     my ($volid, $noerr) = @_;
 985
 986     if ($volid eq 'none' || $volid eq 'cdrom' || $volid =~ m|^/|) {
 987         return $volid;
 988     }
 989
 990     # if its neither 'none' nor 'cdrom' nor a path, check if its a volume-id
 991     $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') };
 992     if ($@) {
 993         return undef if $noerr;
 994         die $@;
 995     }
 996     return $volid;
 997 }
 998
 999 my $drivename_hash;
1000
1001 my %drivedesc_base = (
1002     volume => { alias => 'file' },
1003     file => {
1004         type => 'string',
1005         format => 'pve-volume-id-or-qm-path',
1006         default_key => 1,
1007         format_description => 'volume',
1008         description => "The drive's backing volume.",
1009     },
1010     media => {
1011         type => 'string',
1012         enum => [qw(cdrom disk)],
1013         description => "The drive's media type.",
1014         default => 'disk',
1015         optional => 1
1016     },
1017     cyls => {
1018         type => 'integer',
1019         description => "Force the drive's physical geometry to have a specific cylinder count.",
1020         optional => 1
1021     },
1022     heads => {
1023         type => 'integer',
1024         description => "Force the drive's physical geometry to have a specific head count.",
1025         optional => 1
1026     },
1027     secs => {
1028         type => 'integer',
1029         description => "Force the drive's physical geometry to have a specific sector count.",
1030         optional => 1
1031     },
1032     trans => {
1033         type => 'string',
1034         enum => [qw(none lba auto)],
1035         description => "Force disk geometry bios translation mode.",
1036         optional => 1,
1037     },
1038     snapshot => {
1039         type => 'boolean',
1040         description => "Controls qemu's snapshot mode feature."
1041             . " If activated, changes made to the disk are temporary and will"
1042             . " be discarded when the VM is shutdown.",
1043         optional => 1,
1044     },
1045     cache => {
1046         type => 'string',
1047         enum => [qw(none writethrough writeback unsafe directsync)],
1048         description => "The drive's cache mode",
1049         optional => 1,
1050     },
1051     format => get_standard_option('pve-qm-image-format'),
1052     size => {
1053         type => 'string',
1054         format => 'disk-size',
1055         format_description => 'DiskSize',
1056         description => "Disk size. This is purely informational and has no effect.",
1057         optional => 1,
1058     },
1059     backup => {
1060         type => 'boolean',
1061         description => "Whether the drive should be included when making backups.",
1062         optional => 1,
1063     },
1064     replicate => {
1065         type => 'boolean',
1066         description => 'Whether the drive should considered for replication jobs.',
1067         optional => 1,
1068         default => 1,
1069     },
1070     rerror => {
1071         type => 'string',
1072         enum => [qw(ignore report stop)],
1073         description => 'Read error action.',
1074         optional => 1,
1075     },
1076     werror => {
1077         type => 'string',
1078         enum => [qw(enospc ignore report stop)],
1079         description => 'Write error action.',
1080         optional => 1,
1081     },
1082     aio => {
1083         type => 'string',
1084         enum => [qw(native threads)],
1085         description => 'AIO type to use.',
1086         optional => 1,
1087     },
1088     discard => {
1089         type => 'string',
1090         enum => [qw(ignore on)],
1091         description => 'Controls whether to pass discard/trim requests to the underlying storage.',
1092         optional => 1,
1093     },
1094     detect_zeroes => {
1095         type => 'boolean',
1096         description => 'Controls whether to detect and try to optimize writes of zeroes.',
1097         optional => 1,
1098     },
1099     serial => {
1100         type => 'string',
1101         format => 'urlencoded',
1102         format_description => 'serial',
1103         maxLength => 20*3, # *3 since it's %xx url enoded
1104         description => "The drive's reported serial number, url-encoded, up to 20 bytes long.",
1105         optional => 1,
1106     },
1107     shared => {
1108         type => 'boolean',
1109         description => 'Mark this locally-managed volume as available on all nodes',
1110         verbose_description => "Mark this locally-managed volume as available on all nodes.\n\nWARNING: This option does not share the volume automatically, it assumes it is shared already!",
1111         optional => 1,
1112         default => 0,
1113     }
1114 );
1115
1116 my %iothread_fmt = ( iothread => {
1117         type => 'boolean',
1118         description => "Whether to use iothreads for this drive",
1119         optional => 1,
1120 });
1121
1122 my %model_fmt = (
1123     model => {
1124         type => 'string',
1125         format => 'urlencoded',
1126         format_description => 'model',
1127         maxLength => 40*3, # *3 since it's %xx url enoded
1128         description => "The drive's reported model name, url-encoded, up to 40 bytes long.",
1129         optional => 1,
1130     },
1131 );
1132
1133 my %queues_fmt = (
1134     queues => {
1135         type => 'integer',
1136         description => "Number of queues.",
1137         minimum => 2,
1138         optional => 1
1139     }
1140 );
1141
1142 my %scsiblock_fmt = (
1143     scsiblock => {
1144         type => 'boolean',
1145         description => "whether to use scsi-block for full passthrough of host block device\n\nWARNING: can lead to I/O errors in combination with low memory or high memory fragmentation on host",
1146         optional => 1,
1147         default => 0,
1148     },
1149 );
1150
1151 my %ssd_fmt = (
1152     ssd => {
1153         type => 'boolean',
1154         description => "Whether to expose this drive as an SSD, rather than a rotational hard disk.",
1155         optional => 1,
1156     },
1157 );
1158
1159 my %wwn_fmt = (
1160     wwn => {
1161         type => 'string',
1162         pattern => qr/^(0x)[0-9a-fA-F]{16}/,
1163         format_description => 'wwn',
1164         description => "The drive's worldwide name, encoded as 16 bytes hex string, prefixed by '0x'.",
1165         optional => 1,
1166     },
1167 );
1168
1169 my $add_throttle_desc = sub {
1170     my ($key, $type, $what, $unit, $longunit, $minimum) = @_;
1171     my $d = {
1172         type => $type,
1173         format_description => $unit,
1174         description => "Maximum $what in $longunit.",
1175         optional => 1,
1176     };
1177     $d->{minimum} = $minimum if defined($minimum);
1178     $drivedesc_base{$key} = $d;
1179 };
1180 # throughput: (leaky bucket)
1181 $add_throttle_desc->('bps',     'integer', 'r/w speed',   'bps',  'bytes per second');
1182 $add_throttle_desc->('bps_rd',  'integer', 'read speed',  'bps',  'bytes per second');
1183 $add_throttle_desc->('bps_wr',  'integer', 'write speed', 'bps',  'bytes per second');
1184 $add_throttle_desc->('mbps',    'number',  'r/w speed',   'mbps', 'megabytes per second');
1185 $add_throttle_desc->('mbps_rd', 'number',  'read speed',  'mbps', 'megabytes per second');
1186 $add_throttle_desc->('mbps_wr', 'number',  'write speed', 'mbps', 'megabytes per second');
1187 $add_throttle_desc->('iops',    'integer', 'r/w I/O',     'iops', 'operations per second');
1188 $add_throttle_desc->('iops_rd', 'integer', 'read I/O',    'iops', 'operations per second');
1189 $add_throttle_desc->('iops_wr', 'integer', 'write I/O',   'iops', 'operations per second');
1190
1191 # pools: (pool of IO before throttling starts taking effect)
1192 $add_throttle_desc->('mbps_max',    'number',  'unthrottled r/w pool',       'mbps', 'megabytes per second');
1193 $add_throttle_desc->('mbps_rd_max', 'number',  'unthrottled read pool',      'mbps', 'megabytes per second');
1194 $add_throttle_desc->('mbps_wr_max', 'number',  'unthrottled write pool',     'mbps', 'megabytes per second');
1195 $add_throttle_desc->('iops_max',    'integer', 'unthrottled r/w I/O pool',   'iops', 'operations per second');
1196 $add_throttle_desc->('iops_rd_max', 'integer', 'unthrottled read I/O pool',  'iops', 'operations per second');
1197 $add_throttle_desc->('iops_wr_max', 'integer', 'unthrottled write I/O pool', 'iops', 'operations per second');
1198
1199 # burst lengths
1200 $add_throttle_desc->('bps_max_length',     'integer', 'length of I/O bursts',       'seconds', 'seconds', 1);
1201 $add_throttle_desc->('bps_rd_max_length',  'integer', 'length of read I/O bursts',  'seconds', 'seconds', 1);
1202 $add_throttle_desc->('bps_wr_max_length',  'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
1203 $add_throttle_desc->('iops_max_length',    'integer', 'length of I/O bursts',       'seconds', 'seconds', 1);
1204 $add_throttle_desc->('iops_rd_max_length', 'integer', 'length of read I/O bursts',  'seconds', 'seconds', 1);
1205 $add_throttle_desc->('iops_wr_max_length', 'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
1206
1207 # legacy support
1208 $drivedesc_base{'bps_rd_length'} = { alias => 'bps_rd_max_length' };
1209 $drivedesc_base{'bps_wr_length'} = { alias => 'bps_wr_max_length' };
1210 $drivedesc_base{'iops_rd_length'} = { alias => 'iops_rd_max_length' };
1211 $drivedesc_base{'iops_wr_length'} = { alias => 'iops_wr_max_length' };
1212
1213 my $ide_fmt = {
1214     %drivedesc_base,
1215     %model_fmt,
1216     %ssd_fmt,
1217     %wwn_fmt,
1218 };
1219 PVE::JSONSchema::register_format("pve-qm-ide", $ide_fmt);
1220
1221 my $idedesc = {
1222     optional => 1,
1223     type => 'string', format => $ide_fmt,
1224     description => "Use volume as IDE hard disk or CD-ROM (n is 0 to " .($MAX_IDE_DISKS -1) . ").",
1225 };
1226 PVE::JSONSchema::register_standard_option("pve-qm-ide", $idedesc);
1227
1228 my $scsi_fmt = {
1229     %drivedesc_base,
1230     %iothread_fmt,
1231     %queues_fmt,
1232     %scsiblock_fmt,
1233     %ssd_fmt,
1234     %wwn_fmt,
1235 };
1236 my $scsidesc = {
1237     optional => 1,
1238     type => 'string', format => $scsi_fmt,
1239     description => "Use volume as SCSI hard disk or CD-ROM (n is 0 to " . ($MAX_SCSI_DISKS - 1) . ").",
1240 };
1241 PVE::JSONSchema::register_standard_option("pve-qm-scsi", $scsidesc);
1242
1243 my $sata_fmt = {
1244     %drivedesc_base,
1245     %ssd_fmt,
1246     %wwn_fmt,
1247 };
1248 my $satadesc = {
1249     optional => 1,
1250     type => 'string', format => $sata_fmt,
1251     description => "Use volume as SATA hard disk or CD-ROM (n is 0 to " . ($MAX_SATA_DISKS - 1). ").",
1252 };
1253 PVE::JSONSchema::register_standard_option("pve-qm-sata", $satadesc);
1254
1255 my $virtio_fmt = {
1256     %drivedesc_base,
1257     %iothread_fmt,
1258 };
1259 my $virtiodesc = {
1260     optional => 1,
1261     type => 'string', format => $virtio_fmt,
1262     description => "Use volume as VIRTIO hard disk (n is 0 to " . ($MAX_VIRTIO_DISKS - 1) . ").",
1263 };
1264 PVE::JSONSchema::register_standard_option("pve-qm-virtio", $virtiodesc);
1265
1266 my $alldrive_fmt = {
1267     %drivedesc_base,
1268     %iothread_fmt,
1269     %model_fmt,
1270     %queues_fmt,
1271     %scsiblock_fmt,
1272     %ssd_fmt,
1273     %wwn_fmt,
1274 };
1275
1276 my $efidisk_fmt = {
1277     volume => { alias => 'file' },
1278     file => {
1279         type => 'string',
1280         format => 'pve-volume-id-or-qm-path',
1281         default_key => 1,
1282         format_description => 'volume',
1283         description => "The drive's backing volume.",
1284     },
1285     format => get_standard_option('pve-qm-image-format'),
1286     size => {
1287         type => 'string',
1288         format => 'disk-size',
1289         format_description => 'DiskSize',
1290         description => "Disk size. This is purely informational and has no effect.",
1291         optional => 1,
1292     },
1293 };
1294
1295 my $efidisk_desc = {
1296     optional => 1,
1297     type => 'string', format => $efidisk_fmt,
1298     description => "Configure a Disk for storing EFI vars",
1299 };
1300
1301 PVE::JSONSchema::register_standard_option("pve-qm-efidisk", $efidisk_desc);
1302
1303 my $usb_fmt = {
1304     host => {
1305         default_key => 1,
1306         type => 'string', format => 'pve-qm-usb-device',
1307         format_description => 'HOSTUSBDEVICE|spice',
1308         description => <<EODESCR,
1309 The Host USB device or port or the value 'spice'. HOSTUSBDEVICE syntax is:
1310
1311  'bus-port(.port)*' (decimal numbers) or
1312  'vendor_id:product_id' (hexadeciaml numbers) or
1313  'spice'
1314
1315 You can use the 'lsusb -t' command to list existing usb devices.
1316
1317 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1318
1319 The value 'spice' can be used to add a usb redirection devices for spice.
1320 EODESCR
1321     },
1322     usb3 => {
1323         optional => 1,
1324         type => 'boolean',
1325         description => "Specifies whether if given host option is a USB3 device or port.",
1326         default => 0,
1327     },
1328 };
1329
1330 my $usbdesc = {
1331     optional => 1,
1332     type => 'string', format => $usb_fmt,
1333     description => "Configure an USB device (n is 0 to 4).",
1334 };
1335 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
1336
1337 my $PCIRE = qr/[a-f0-9]{2}:[a-f0-9]{2}(?:\.[a-f0-9])?/;
1338 my $hostpci_fmt = {
1339     host => {
1340         default_key => 1,
1341         type => 'string',
1342         pattern => qr/$PCIRE(;$PCIRE)*/,
1343         format_description => 'HOSTPCIID[;HOSTPCIID2...]',
1344         description => <<EODESCR,
1345 Host PCI device pass through. The PCI ID of a host's PCI device or a list
1346 of PCI virtual functions of the host. HOSTPCIID syntax is:
1347
1348 'bus:dev.func' (hexadecimal numbers)
1349
1350 You can us the 'lspci' command to list existing PCI devices.
1351 EODESCR
1352     },
1353     rombar => {
1354         type => 'boolean',
1355         description =>  "Specify whether or not the device's ROM will be visible in the guest's memory map.",
1356         optional => 1,
1357         default => 1,
1358     },
1359     romfile => {
1360         type => 'string',
1361         pattern => '[^,;]+',
1362         format_description => 'string',
1363         description => "Custom pci device rom filename (must be located in /usr/share/kvm/).",
1364         optional => 1,
1365     },
1366     pcie => {
1367         type => 'boolean',
1368         description =>  "Choose the PCI-express bus (needs the 'q35' machine model).",
1369         optional => 1,
1370         default => 0,
1371     },
1372     'x-vga' => {
1373         type => 'boolean',
1374         description =>  "Enable vfio-vga device support.",
1375         optional => 1,
1376         default => 0,
1377     },
1378     'mdev' => {
1379         type => 'string',
1380         format_description => 'string',
1381         pattern => '[^/\.:]+',
1382         optional => 1,
1383         description => <<EODESCR
1384 The type of mediated device to use.
1385 An instance of this type will be created on startup of the VM and
1386 will be cleaned up when the VM stops.
1387 EODESCR
1388     }
1389 };
1390 PVE::JSONSchema::register_format('pve-qm-hostpci', $hostpci_fmt);
1391
1392 my $hostpcidesc = {
1393         optional => 1,
1394         type => 'string', format => 'pve-qm-hostpci',
1395         description => "Map host PCI devices into guest.",
1396         verbose_description =>  <<EODESCR,
1397 Map host PCI devices into guest.
1398
1399 NOTE: This option allows direct access to host hardware. So it is no longer
1400 possible to migrate such machines - use with special care.
1401
1402 CAUTION: Experimental! User reported problems with this option.
1403 EODESCR
1404 };
1405 PVE::JSONSchema::register_standard_option("pve-qm-hostpci", $hostpcidesc);
1406
1407 my $serialdesc = {
1408         optional => 1,
1409         type => 'string',
1410         pattern => '(/dev/.+|socket)',
1411         description =>  "Create a serial device inside the VM (n is 0 to 3)",
1412         verbose_description =>  <<EODESCR,
1413 Create a serial device inside the VM (n is 0 to 3), and pass through a
1414 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1415 host side (use 'qm terminal' to open a terminal connection).
1416
1417 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines - use with special care.
1418
1419 CAUTION: Experimental! User reported problems with this option.
1420 EODESCR
1421 };
1422
1423 my $paralleldesc= {
1424         optional => 1,
1425         type => 'string',
1426         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
1427         description =>  "Map host parallel devices (n is 0 to 2).",
1428         verbose_description =>  <<EODESCR,
1429 Map host parallel devices (n is 0 to 2).
1430
1431 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1432
1433 CAUTION: Experimental! User reported problems with this option.
1434 EODESCR
1435 };
1436
1437 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
1438     $confdesc->{"parallel$i"} = $paralleldesc;
1439 }
1440
1441 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
1442     $confdesc->{"serial$i"} = $serialdesc;
1443 }
1444
1445 for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
1446     $confdesc->{"hostpci$i"} = $hostpcidesc;
1447 }
1448
1449 for (my $i = 0; $i < $MAX_IDE_DISKS; $i++)  {
1450     $drivename_hash->{"ide$i"} = 1;
1451     $confdesc->{"ide$i"} = $idedesc;
1452 }
1453
1454 for (my $i = 0; $i < $MAX_SATA_DISKS; $i++)  {
1455     $drivename_hash->{"sata$i"} = 1;
1456     $confdesc->{"sata$i"} = $satadesc;
1457 }
1458
1459 for (my $i = 0; $i < $MAX_SCSI_DISKS; $i++)  {
1460     $drivename_hash->{"scsi$i"} = 1;
1461     $confdesc->{"scsi$i"} = $scsidesc ;
1462 }
1463
1464 for (my $i = 0; $i < $MAX_VIRTIO_DISKS; $i++)  {
1465     $drivename_hash->{"virtio$i"} = 1;
1466     $confdesc->{"virtio$i"} = $virtiodesc;
1467 }
1468
1469 $drivename_hash->{efidisk0} = 1;
1470 $confdesc->{efidisk0} = $efidisk_desc;
1471
1472 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
1473     $confdesc->{"usb$i"} = $usbdesc;
1474 }
1475
1476 my $unuseddesc = {
1477     optional => 1,
1478     type => 'string', format => 'pve-volume-id',
1479     description => "Reference to unused volumes. This is used internally, and should not be modified manually.",
1480 };
1481
1482 for (my $i = 0; $i < $MAX_UNUSED_DISKS; $i++)  {
1483     $confdesc->{"unused$i"} = $unuseddesc;
1484 }
1485
1486 my $kvm_api_version = 0;
1487
1488 sub kvm_version {
1489     return $kvm_api_version if $kvm_api_version;
1490
1491     open my $fh, '<', '/dev/kvm'
1492         or return undef;
1493
1494     # 0xae00 => KVM_GET_API_VERSION
1495     $kvm_api_version = ioctl($fh, 0xae00, 0);
1496
1497     return $kvm_api_version;
1498 }
1499
1500 my $kvm_user_version = {};
1501 my $kvm_mtime = {};
1502
1503 sub kvm_user_version {
1504     my ($binary) = @_;
1505
1506     $binary //= get_command_for_arch(get_host_arch()); # get the native arch by default
1507     my $st = stat($binary);
1508
1509     my $cachedmtime = $kvm_mtime->{$binary} // -1;
1510     return $kvm_user_version->{$binary} if $kvm_user_version->{$binary} &&
1511         $cachedmtime == $st->mtime;
1512
1513     $kvm_user_version->{$binary} = 'unknown';
1514     $kvm_mtime->{$binary} = $st->mtime;
1515
1516     my $code = sub {
1517         my $line = shift;
1518         if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1519             $kvm_user_version->{$binary} = $2;
1520         }
1521     };
1522
1523     eval { run_command([$binary, '--version'], outfunc => $code); };
1524     warn $@ if $@;
1525
1526     return $kvm_user_version->{$binary};
1527
1528 }
1529
1530 sub kernel_has_vhost_net {
1531     return -c '/dev/vhost-net';
1532 }
1533
1534 sub valid_drive_names {
1535     # order is important - used to autoselect boot disk
1536     return ((map { "ide$_" } (0 .. ($MAX_IDE_DISKS - 1))),
1537             (map { "scsi$_" } (0 .. ($MAX_SCSI_DISKS - 1))),
1538             (map { "virtio$_" } (0 .. ($MAX_VIRTIO_DISKS - 1))),
1539             (map { "sata$_" } (0 .. ($MAX_SATA_DISKS - 1))),
1540             'efidisk0');
1541 }
1542
1543 sub is_valid_drivename {
1544     my $dev = shift;
1545
1546     return defined($drivename_hash->{$dev});
1547 }
1548
1549 sub option_exists {
1550     my $key = shift;
1551     return defined($confdesc->{$key});
1552 }
1553
1554 sub nic_models {
1555     return $nic_model_list;
1556 }
1557
1558 sub os_list_description {
1559
1560     return {
1561         other => 'Other',
1562         wxp => 'Windows XP',
1563         w2k => 'Windows 2000',
1564         w2k3 =>, 'Windows 2003',
1565         w2k8 => 'Windows 2008',
1566         wvista => 'Windows Vista',
1567         win7 => 'Windows 7',
1568         win8 => 'Windows 8/2012',
1569         win10 => 'Windows 10/2016',
1570         l24 => 'Linux 2.4',
1571         l26 => 'Linux 2.6',
1572     };
1573 }
1574
1575 my $cdrom_path;
1576
1577 sub get_cdrom_path {
1578
1579     return  $cdrom_path if $cdrom_path;
1580
1581     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
1582     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
1583     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
1584 }
1585
1586 sub get_iso_path {
1587     my ($storecfg, $vmid, $cdrom) = @_;
1588
1589     if ($cdrom eq 'cdrom') {
1590         return get_cdrom_path();
1591     } elsif ($cdrom eq 'none') {
1592         return '';
1593     } elsif ($cdrom =~ m|^/|) {
1594         return $cdrom;
1595     } else {
1596         return PVE::Storage::path($storecfg, $cdrom);
1597     }
1598 }
1599
1600 # try to convert old style file names to volume IDs
1601 sub filename_to_volume_id {
1602     my ($vmid, $file, $media) = @_;
1603
1604      if (!($file eq 'none' || $file eq 'cdrom' ||
1605           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1606
1607         return undef if $file =~ m|/|;
1608
1609         if ($media && $media eq 'cdrom') {
1610             $file = "local:iso/$file";
1611         } else {
1612             $file = "local:$vmid/$file";
1613         }
1614     }
1615
1616     return $file;
1617 }
1618
1619 sub verify_media_type {
1620     my ($opt, $vtype, $media) = @_;
1621
1622     return if !$media;
1623
1624     my $etype;
1625     if ($media eq 'disk') {
1626         $etype = 'images';
1627     } elsif ($media eq 'cdrom') {
1628         $etype = 'iso';
1629     } else {
1630         die "internal error";
1631     }
1632
1633     return if ($vtype eq $etype);
1634
1635     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1636 }
1637
1638 sub cleanup_drive_path {
1639     my ($opt, $storecfg, $drive) = @_;
1640
1641     # try to convert filesystem paths to volume IDs
1642
1643     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1644         ($drive->{file} !~ m|^/dev/.+|) &&
1645         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1646         ($drive->{file} !~ m/^\d+$/)) {
1647         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1648         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"}) if !$vtype;
1649         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1650         verify_media_type($opt, $vtype, $drive->{media});
1651         $drive->{file} = $volid;
1652     }
1653
1654     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1655 }
1656
1657 sub parse_hotplug_features {
1658     my ($data) = @_;
1659
1660     my $res = {};
1661
1662     return $res if $data eq '0';
1663
1664     $data = $confdesc->{hotplug}->{default} if $data eq '1';
1665
1666     foreach my $feature (PVE::Tools::split_list($data)) {
1667         if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
1668             $res->{$1} = 1;
1669         } else {
1670             die "invalid hotplug feature '$feature'\n";
1671         }
1672     }
1673     return $res;
1674 }
1675
1676 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1677 sub pve_verify_hotplug_features {
1678     my ($value, $noerr) = @_;
1679
1680     return $value if parse_hotplug_features($value);
1681
1682     return undef if $noerr;
1683
1684     die "unable to parse hotplug option\n";
1685 }
1686
1687 # ideX = [volume=]volume-id[,media=d][,cyls=c,heads=h,secs=s[,trans=t]]
1688 #        [,snapshot=on|off][,cache=on|off][,format=f][,backup=yes|no]
1689 #        [,rerror=ignore|report|stop][,werror=enospc|ignore|report|stop]
1690 #        [,aio=native|threads][,discard=ignore|on][,detect_zeroes=on|off]
1691 #        [,iothread=on][,serial=serial][,model=model]
1692
1693 sub parse_drive {
1694     my ($key, $data) = @_;
1695
1696     my ($interface, $index);
1697
1698     if ($key =~ m/^([^\d]+)(\d+)$/) {
1699         $interface = $1;
1700         $index = $2;
1701     } else {
1702         return undef;
1703     }
1704
1705     my $desc = $key =~ /^unused\d+$/ ? $alldrive_fmt
1706                                      : $confdesc->{$key}->{format};
1707     if (!$desc) {
1708         warn "invalid drive key: $key\n";
1709         return undef;
1710     }
1711     my $res = eval { PVE::JSONSchema::parse_property_string($desc, $data) };
1712     return undef if !$res;
1713     $res->{interface} = $interface;
1714     $res->{index} = $index;
1715
1716     my $error = 0;
1717     foreach my $opt (qw(bps bps_rd bps_wr)) {
1718         if (my $bps = defined(delete $res->{$opt})) {
1719             if (defined($res->{"m$opt"})) {
1720                 warn "both $opt and m$opt specified\n";
1721                 ++$error;
1722                 next;
1723             }
1724             $res->{"m$opt"} = sprintf("%.3f", $bps / (1024*1024.0));
1725         }
1726     }
1727
1728     # can't use the schema's 'requires' because of the mbps* => bps* "transforming aliases"
1729     for my $requirement (
1730         [mbps_max => 'mbps'],
1731         [mbps_rd_max => 'mbps_rd'],
1732         [mbps_wr_max => 'mbps_wr'],
1733         [miops_max => 'miops'],
1734         [miops_rd_max => 'miops_rd'],
1735         [miops_wr_max => 'miops_wr'],
1736         [bps_max_length => 'mbps_max'],
1737         [bps_rd_max_length => 'mbps_rd_max'],
1738         [bps_wr_max_length => 'mbps_wr_max'],
1739         [iops_max_length => 'iops_max'],
1740         [iops_rd_max_length => 'iops_rd_max'],
1741         [iops_wr_max_length => 'iops_wr_max']) {
1742         my ($option, $requires) = @$requirement;
1743         if ($res->{$option} && !$res->{$requires}) {
1744             warn "$option requires $requires\n";
1745             ++$error;
1746         }
1747     }
1748
1749     return undef if $error;
1750
1751     return undef if $res->{mbps_rd} && $res->{mbps};
1752     return undef if $res->{mbps_wr} && $res->{mbps};
1753     return undef if $res->{iops_rd} && $res->{iops};
1754     return undef if $res->{iops_wr} && $res->{iops};
1755
1756     if ($res->{media} && ($res->{media} eq 'cdrom')) {
1757         return undef if $res->{snapshot} || $res->{trans} || $res->{format};
1758         return undef if $res->{heads} || $res->{secs} || $res->{cyls};
1759         return undef if $res->{interface} eq 'virtio';
1760     }
1761
1762     if (my $size = $res->{size}) {
1763         return undef if !defined($res->{size} = PVE::JSONSchema::parse_size($size));
1764     }
1765
1766     return $res;
1767 }
1768
1769 sub print_drive {
1770     my ($vmid, $drive) = @_;
1771     my $data = { %$drive };
1772     delete $data->{$_} for qw(index interface);
1773     return PVE::JSONSchema::print_property_string($data, $alldrive_fmt);
1774 }
1775
1776 sub scsi_inquiry {
1777     my($fh, $noerr) = @_;
1778
1779     my $SG_IO = 0x2285;
1780     my $SG_GET_VERSION_NUM = 0x2282;
1781
1782     my $versionbuf = "\x00" x 8;
1783     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1784     if (!$ret) {
1785         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1786         return undef;
1787     }
1788     my $version = unpack("I", $versionbuf);
1789     if ($version < 30000) {
1790         die "scsi generic interface too old\n"  if !$noerr;
1791         return undef;
1792     }
1793
1794     my $buf = "\x00" x 36;
1795     my $sensebuf = "\x00" x 8;
1796     my $cmd = pack("C x3 C x1", 0x12, 36);
1797
1798     # see /usr/include/scsi/sg.h
1799     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1800
1801     my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1802                       length($sensebuf), 0, length($buf), $buf,
1803                       $cmd, $sensebuf, 6000);
1804
1805     $ret = ioctl($fh, $SG_IO, $packet);
1806     if (!$ret) {
1807         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1808         return undef;
1809     }
1810
1811     my @res = unpack($sg_io_hdr_t, $packet);
1812     if ($res[17] || $res[18]) {
1813         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1814         return undef;
1815     }
1816
1817     my $res = {};
1818     (my $byte0, my $byte1, $res->{vendor},
1819      $res->{product}, $res->{revision}) = unpack("C C x6 A8 A16 A4", $buf);
1820
1821     $res->{removable} = $byte1 & 128 ? 1 : 0;
1822     $res->{type} = $byte0 & 31;
1823
1824     return $res;
1825 }
1826
1827 sub path_is_scsi {
1828     my ($path) = @_;
1829
1830     my $fh = IO::File->new("+<$path") || return undef;
1831     my $res = scsi_inquiry($fh, 1);
1832     close($fh);
1833
1834     return $res;
1835 }
1836
1837 sub machine_type_is_q35 {
1838     my ($conf) = @_;
1839
1840     return $conf->{machine} && ($conf->{machine} =~ m/q35/) ? 1 : 0;
1841 }
1842
1843 sub print_tabletdevice_full {
1844     my ($conf, $arch) = @_;
1845
1846     my $q35 = machine_type_is_q35($conf);
1847
1848     # we use uhci for old VMs because tablet driver was buggy in older qemu
1849     my $usbbus;
1850     if (machine_type_is_q35($conf) || $arch eq 'aarch64') {
1851         $usbbus = 'ehci';
1852     } else {
1853         $usbbus = 'uhci';
1854     }
1855
1856     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1857 }
1858
1859 sub print_keyboarddevice_full {
1860     my ($conf, $arch, $machine) = @_;
1861
1862     return undef if $arch ne 'aarch64';
1863
1864     return "usb-kbd,id=keyboard,bus=ehci.0,port=2";
1865 }
1866
1867 sub print_drivedevice_full {
1868     my ($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type) = @_;
1869
1870     my $device = '';
1871     my $maxdev = 0;
1872
1873     if ($drive->{interface} eq 'virtio') {
1874         my $pciaddr = print_pci_addr("$drive->{interface}$drive->{index}", $bridges, $arch, $machine_type);
1875         $device = "virtio-blk-pci,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}$pciaddr";
1876         $device .= ",iothread=iothread-$drive->{interface}$drive->{index}" if $drive->{iothread};
1877     } elsif ($drive->{interface} eq 'scsi') {
1878
1879         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1880         my $unit = $drive->{index} % $maxdev;
1881         my $devicetype = 'hd';
1882         my $path = '';
1883         if (drive_is_cdrom($drive)) {
1884             $devicetype = 'cd';
1885         } else {
1886             if ($drive->{file} =~ m|^/|) {
1887                 $path = $drive->{file};
1888                 if (my $info = path_is_scsi($path)) {
1889                     if ($info->{type} == 0 && $drive->{scsiblock}) {
1890                         $devicetype = 'block';
1891                     } elsif ($info->{type} == 1) { # tape
1892                         $devicetype = 'generic';
1893                     }
1894                 }
1895             } else {
1896                  $path = PVE::Storage::path($storecfg, $drive->{file});
1897             }
1898
1899             if($path =~ m/^iscsi\:\/\//){
1900                 $devicetype = 'generic';
1901             }
1902         }
1903
1904         if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)){
1905             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1906         } else {
1907             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0,lun=$drive->{index},drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1908         }
1909
1910         if ($drive->{ssd} && ($devicetype eq 'block' || $devicetype eq 'hd')) {
1911             $device .= ",rotation_rate=1";
1912         }
1913         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1914
1915     } elsif ($drive->{interface} eq 'ide' || $drive->{interface} eq 'sata') {
1916         my $maxdev = ($drive->{interface} eq 'sata') ? $MAX_SATA_DISKS : 2;
1917         my $controller = int($drive->{index} / $maxdev);
1918         my $unit = $drive->{index} % $maxdev;
1919         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1920
1921         $device = "ide-$devicetype";
1922         if ($drive->{interface} eq 'ide') {
1923             $device .= ",bus=ide.$controller,unit=$unit";
1924         } else {
1925             $device .= ",bus=ahci$controller.$unit";
1926         }
1927         $device .= ",drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1928
1929         if ($devicetype eq 'hd') {
1930             if (my $model = $drive->{model}) {
1931                 $model = URI::Escape::uri_unescape($model);
1932                 $device .= ",model=$model";
1933             }
1934             if ($drive->{ssd}) {
1935                 $device .= ",rotation_rate=1";
1936             }
1937         }
1938         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1939     } elsif ($drive->{interface} eq 'usb') {
1940         die "implement me";
1941         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1942     } else {
1943         die "unsupported interface type";
1944     }
1945
1946     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1947
1948     if (my $serial = $drive->{serial}) {
1949         $serial = URI::Escape::uri_unescape($serial);
1950         $device .= ",serial=$serial";
1951     }
1952
1953
1954     return $device;
1955 }
1956
1957 sub get_initiator_name {
1958     my $initiator;
1959
1960     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return undef;
1961     while (defined(my $line = <$fh>)) {
1962         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1963         $initiator = $1;
1964         last;
1965     }
1966     $fh->close();
1967
1968     return $initiator;
1969 }
1970
1971 sub print_drive_full {
1972     my ($storecfg, $vmid, $drive) = @_;
1973
1974     my $path;
1975     my $volid = $drive->{file};
1976     my $format;
1977
1978     if (drive_is_cdrom($drive)) {
1979         $path = get_iso_path($storecfg, $vmid, $volid);
1980     } else {
1981         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1982         if ($storeid) {
1983             $path = PVE::Storage::path($storecfg, $volid);
1984             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
1985             $format = qemu_img_format($scfg, $volname);
1986         } else {
1987             $path = $volid;
1988             $format = "raw";
1989         }
1990    }
1991
1992     my $opts = '';
1993     my @qemu_drive_options = qw(heads secs cyls trans media format cache rerror werror aio discard);
1994     foreach my $o (@qemu_drive_options) {
1995         $opts .= ",$o=$drive->{$o}" if defined($drive->{$o});
1996     }
1997
1998     # snapshot only accepts on|off
1999     if (defined($drive->{snapshot})) {
2000         my $v = $drive->{snapshot} ? 'on' : 'off';
2001         $opts .= ",snapshot=$v";
2002     }
2003
2004     foreach my $type (['', '-total'], [_rd => '-read'], [_wr => '-write']) {
2005         my ($dir, $qmpname) = @$type;
2006         if (my $v = $drive->{"mbps$dir"}) {
2007             $opts .= ",throttling.bps$qmpname=".int($v*1024*1024);
2008         }
2009         if (my $v = $drive->{"mbps${dir}_max"}) {
2010             $opts .= ",throttling.bps$qmpname-max=".int($v*1024*1024);
2011         }
2012         if (my $v = $drive->{"bps${dir}_max_length"}) {
2013             $opts .= ",throttling.bps$qmpname-max-length=$v";
2014         }
2015         if (my $v = $drive->{"iops${dir}"}) {
2016             $opts .= ",throttling.iops$qmpname=$v";
2017         }
2018         if (my $v = $drive->{"iops${dir}_max"}) {
2019             $opts .= ",throttling.iops$qmpname-max=$v";
2020         }
2021         if (my $v = $drive->{"iops${dir}_max_length"}) {
2022             $opts .= ",throttling.iops$qmpname-max-length=$v";
2023         }
2024     }
2025
2026     $opts .= ",format=$format" if $format && !$drive->{format};
2027
2028     my $cache_direct = 0;
2029
2030     if (my $cache = $drive->{cache}) {
2031         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
2032     } elsif (!drive_is_cdrom($drive)) {
2033         $opts .= ",cache=none";
2034         $cache_direct = 1;
2035     }
2036
2037     # aio native works only with O_DIRECT
2038     if (!$drive->{aio}) {
2039         if($cache_direct) {
2040             $opts .= ",aio=native";
2041         } else {
2042             $opts .= ",aio=threads";
2043         }
2044     }
2045
2046     if (!drive_is_cdrom($drive)) {
2047         my $detectzeroes;
2048         if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
2049             $detectzeroes = 'off';
2050         } elsif ($drive->{discard}) {
2051             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
2052         } else {
2053             # This used to be our default with discard not being specified:
2054             $detectzeroes = 'on';
2055         }
2056         $opts .= ",detect-zeroes=$detectzeroes" if $detectzeroes;
2057     }
2058
2059     my $pathinfo = $path ? "file=$path," : '';
2060
2061     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
2062 }
2063
2064 sub print_netdevice_full {
2065     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files, $arch, $machine_type) = @_;
2066
2067     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
2068
2069     my $device = $net->{model};
2070     if ($net->{model} eq 'virtio') {
2071          $device = 'virtio-net-pci';
2072      };
2073
2074     my $pciaddr = print_pci_addr("$netid", $bridges, $arch, $machine_type);
2075     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
2076     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
2077         #Consider we have N queues, the number of vectors needed is 2*N + 2 (plus one config interrupt and control vq)
2078         my $vectors = $net->{queues} * 2 + 2;
2079         $tmpstr .= ",vectors=$vectors,mq=on";
2080     }
2081     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
2082
2083     if ($use_old_bios_files) {
2084         my $romfile;
2085         if ($device eq 'virtio-net-pci') {
2086             $romfile = 'pxe-virtio.rom';
2087         } elsif ($device eq 'e1000') {
2088             $romfile = 'pxe-e1000.rom';
2089         } elsif ($device eq 'ne2k') {
2090             $romfile = 'pxe-ne2k_pci.rom';
2091         } elsif ($device eq 'pcnet') {
2092             $romfile = 'pxe-pcnet.rom';
2093         } elsif ($device eq 'rtl8139') {
2094             $romfile = 'pxe-rtl8139.rom';
2095         }
2096         $tmpstr .= ",romfile=$romfile" if $romfile;
2097     }
2098
2099     return $tmpstr;
2100 }
2101
2102 sub print_netdev_full {
2103     my ($vmid, $conf, $arch, $net, $netid, $hotplug) = @_;
2104
2105     my $i = '';
2106     if ($netid =~ m/^net(\d+)$/) {
2107         $i = int($1);
2108     }
2109
2110     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
2111
2112     my $ifname = "tap${vmid}i$i";
2113
2114     # kvm uses TUNSETIFF ioctl, and that limits ifname length
2115     die "interface name '$ifname' is too long (max 15 character)\n"
2116         if length($ifname) >= 16;
2117
2118     my $vhostparam = '';
2119     if (is_native($arch)) {
2120         $vhostparam = ',vhost=on' if kernel_has_vhost_net() && $net->{model} eq 'virtio';
2121     }
2122
2123     my $vmname = $conf->{name} || "vm$vmid";
2124
2125     my $netdev = "";
2126     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
2127
2128     if ($net->{bridge}) {
2129         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script,downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
2130     } else {
2131         $netdev = "type=user,id=$netid,hostname=$vmname";
2132     }
2133
2134     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
2135
2136     return $netdev;
2137 }
2138
2139
2140 sub print_cpu_device {
2141     my ($conf, $id) = @_;
2142
2143     my $kvm = $conf->{kvm} // 1;
2144     my $cpu = $kvm ? "kvm64" : "qemu64";
2145     if (my $cputype = $conf->{cpu}) {
2146         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
2147             or die "Cannot parse cpu description: $cputype\n";
2148         $cpu = $cpuconf->{cputype};
2149     }
2150
2151     my $cores = $conf->{cores} || 1;
2152
2153     my $current_core = ($id - 1) % $cores;
2154     my $current_socket = int(($id - 1 - $current_core)/$cores);
2155
2156     return "$cpu-x86_64-cpu,id=cpu$id,socket-id=$current_socket,core-id=$current_core,thread-id=0";
2157 }
2158
2159 my $vga_map = {
2160     'cirrus' => 'cirrus-vga',
2161     'std' => 'VGA',
2162     'vmware' => 'vmware-svga',
2163     'virtio' => 'virtio-vga',
2164 };
2165
2166 sub print_vga_device {
2167     my ($conf, $vga, $arch, $machine, $id, $qxlnum, $bridges) = @_;
2168
2169     my $type = $vga_map->{$vga->{type}};
2170     if ($arch eq 'aarch64' && defined($type) && $type eq 'virtio-vga') {
2171         $type = 'virtio-gpu';
2172     }
2173     my $vgamem_mb = $vga->{memory};
2174     if ($qxlnum) {
2175         $type = $id ? 'qxl' : 'qxl-vga';
2176     }
2177     die "no devicetype for $vga->{type}\n" if !$type;
2178
2179     my $memory = "";
2180     if ($vgamem_mb) {
2181         if ($vga->{type} eq 'virtio') {
2182             my $bytes = PVE::Tools::convert_size($vgamem_mb, "mb" => "b");
2183             $memory = ",max_hostmem=$bytes";
2184         } elsif ($qxlnum) {
2185             # from https://www.spice-space.org/multiple-monitors.html
2186             $memory = ",vgamem_mb=$vga->{memory}";
2187             my $ram = $vgamem_mb * 4;
2188             my $vram = $vgamem_mb * 2;
2189             $memory .= ",ram_size_mb=$ram,vram_size_mb=$vram";
2190         } else {
2191             $memory = ",vgamem_mb=$vga->{memory}";
2192         }
2193     } elsif ($qxlnum && $id) {
2194         $memory = ",ram_size=67108864,vram_size=33554432";
2195     }
2196
2197     my $q35 = machine_type_is_q35($conf);
2198     my $vgaid = "vga" . ($id // '');
2199     my $pciaddr;
2200
2201     if ($q35 && $vgaid eq 'vga') {
2202         # the first display uses pcie.0 bus on q35 machines
2203         $pciaddr = print_pcie_addr($vgaid, $bridges, $arch, $machine);
2204     } else {
2205         $pciaddr = print_pci_addr($vgaid, $bridges, $arch, $machine);
2206     }
2207
2208     return "$type,id=${vgaid}${memory}${pciaddr}";
2209 }
2210
2211 sub drive_is_cloudinit {
2212     my ($drive) = @_;
2213     return $drive->{file} =~ m@[:/]vm-\d+-cloudinit(?:\.$QEMU_FORMAT_RE)?$@;
2214 }
2215
2216 sub drive_is_cdrom {
2217     my ($drive, $exclude_cloudinit) = @_;
2218
2219     return 0 if $exclude_cloudinit && drive_is_cloudinit($drive);
2220
2221     return $drive && $drive->{media} && ($drive->{media} eq 'cdrom');
2222
2223 }
2224
2225 sub parse_number_sets {
2226     my ($set) = @_;
2227     my $res = [];
2228     foreach my $part (split(/;/, $set)) {
2229         if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
2230             die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
2231             push @$res, [ $1, $2 ];
2232         } else {
2233             die "invalid range: $part\n";
2234         }
2235     }
2236     return $res;
2237 }
2238
2239 sub parse_numa {
2240     my ($data) = @_;
2241
2242     my $res = PVE::JSONSchema::parse_property_string($numa_fmt, $data);
2243     $res->{cpus} = parse_number_sets($res->{cpus}) if defined($res->{cpus});
2244     $res->{hostnodes} = parse_number_sets($res->{hostnodes}) if defined($res->{hostnodes});
2245     return $res;
2246 }
2247
2248 sub parse_hostpci {
2249     my ($value) = @_;
2250
2251     return undef if !$value;
2252
2253     my $res = PVE::JSONSchema::parse_property_string($hostpci_fmt, $value);
2254
2255     my @idlist = split(/;/, $res->{host});
2256     delete $res->{host};
2257     foreach my $id (@idlist) {
2258         if ($id =~ m/\./) { # full id 00:00.1
2259             push @{$res->{pciid}}, {
2260                 id => $id,
2261             };
2262         } else { # partial id 00:00
2263             $res->{pciid} = PVE::SysFSTools::lspci($id);
2264         }
2265     }
2266     return $res;
2267 }
2268
2269 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
2270 sub parse_net {
2271     my ($data) = @_;
2272
2273     my $res = eval { PVE::JSONSchema::parse_property_string($net_fmt, $data) };
2274     if ($@) {
2275         warn $@;
2276         return undef;
2277     }
2278     if (!defined($res->{macaddr})) {
2279         my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
2280         $res->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix});
2281     }
2282     return $res;
2283 }
2284
2285 # ipconfigX ip=cidr,gw=ip,ip6=cidr,gw6=ip
2286 sub parse_ipconfig {
2287     my ($data) = @_;
2288
2289     my $res = eval { PVE::JSONSchema::parse_property_string($ipconfig_fmt, $data) };
2290     if ($@) {
2291         warn $@;
2292         return undef;
2293     }
2294
2295     if ($res->{gw} && !$res->{ip}) {
2296         warn 'gateway specified without specifying an IP address';
2297         return undef;
2298     }
2299     if ($res->{gw6} && !$res->{ip6}) {
2300         warn 'IPv6 gateway specified without specifying an IPv6 address';
2301         return undef;
2302     }
2303     if ($res->{gw} && $res->{ip} eq 'dhcp') {
2304         warn 'gateway specified together with DHCP';
2305         return undef;
2306     }
2307     if ($res->{gw6} && $res->{ip6} !~ /^$IPV6RE/) {
2308         # gw6 + auto/dhcp
2309         warn "IPv6 gateway specified together with $res->{ip6} address";
2310         return undef;
2311     }
2312
2313     if (!$res->{ip} && !$res->{ip6}) {
2314         return { ip => 'dhcp', ip6 => 'dhcp' };
2315     }
2316
2317     return $res;
2318 }
2319
2320 sub print_net {
2321     my $net = shift;
2322
2323     return PVE::JSONSchema::print_property_string($net, $net_fmt);
2324 }
2325
2326 sub add_random_macs {
2327     my ($settings) = @_;
2328
2329     foreach my $opt (keys %$settings) {
2330         next if $opt !~ m/^net(\d+)$/;
2331         my $net = parse_net($settings->{$opt});
2332         next if !$net;
2333         $settings->{$opt} = print_net($net);
2334     }
2335 }
2336
2337 sub vm_is_volid_owner {
2338     my ($storecfg, $vmid, $volid) = @_;
2339
2340     if ($volid !~  m|^/|) {
2341         my ($path, $owner);
2342         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
2343         if ($owner && ($owner == $vmid)) {
2344             return 1;
2345         }
2346     }
2347
2348     return undef;
2349 }
2350
2351 sub split_flagged_list {
2352     my $text = shift || '';
2353     $text =~ s/[,;]/ /g;
2354     $text =~ s/^\s+//;
2355     return { map { /^(!?)(.*)$/ && ($2, $1) } ($text =~ /\S+/g) };
2356 }
2357
2358 sub join_flagged_list {
2359     my ($how, $lst) = @_;
2360     join $how, map { $lst->{$_} . $_ } keys %$lst;
2361 }
2362
2363 sub vmconfig_delete_pending_option {
2364     my ($conf, $key, $force) = @_;
2365
2366     delete $conf->{pending}->{$key};
2367     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
2368     $pending_delete_hash->{$key} = $force ? '!' : '';
2369     $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
2370 }
2371
2372 sub vmconfig_undelete_pending_option {
2373     my ($conf, $key) = @_;
2374
2375     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
2376     delete $pending_delete_hash->{$key};
2377
2378     if (%$pending_delete_hash) {
2379         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
2380     } else {
2381         delete $conf->{pending}->{delete};
2382     }
2383 }
2384
2385 sub vmconfig_register_unused_drive {
2386     my ($storecfg, $vmid, $conf, $drive) = @_;
2387
2388     if (drive_is_cloudinit($drive)) {
2389         eval { PVE::Storage::vdisk_free($storecfg, $drive->{file}) };
2390         warn $@ if $@;
2391     } elsif (!drive_is_cdrom($drive)) {
2392         my $volid = $drive->{file};
2393         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
2394             PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
2395         }
2396     }
2397 }
2398
2399 sub vmconfig_cleanup_pending {
2400     my ($conf) = @_;
2401
2402     # remove pending changes when nothing changed
2403     my $changes;
2404     foreach my $opt (keys %{$conf->{pending}}) {
2405         if (defined($conf->{$opt}) && ($conf->{pending}->{$opt} eq  $conf->{$opt})) {
2406             $changes = 1;
2407             delete $conf->{pending}->{$opt};
2408         }
2409     }
2410
2411     my $current_delete_hash = split_flagged_list($conf->{pending}->{delete});
2412     my $pending_delete_hash = {};
2413     while (my ($opt, $force) = each %$current_delete_hash) {
2414         if (defined($conf->{$opt})) {
2415             $pending_delete_hash->{$opt} = $force;
2416         } else {
2417             $changes = 1;
2418         }
2419     }
2420
2421     if (%$pending_delete_hash) {
2422         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
2423     } else {
2424         delete $conf->{pending}->{delete};
2425     }
2426
2427     return $changes;
2428 }
2429
2430 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str][,base64=bool]
2431 my $smbios1_fmt = {
2432     uuid => {
2433         type => 'string',
2434         pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
2435         format_description => 'UUID',
2436         description => "Set SMBIOS1 UUID.",
2437         optional => 1,
2438     },
2439     version => {
2440         type => 'string',
2441         pattern => '[A-Za-z0-9+\/]+={0,2}',
2442         format_description => 'Base64 encoded string',
2443         description => "Set SMBIOS1 version.",
2444         optional => 1,
2445     },
2446     serial => {
2447         type => 'string',
2448         pattern => '[A-Za-z0-9+\/]+={0,2}',
2449         format_description => 'Base64 encoded string',
2450         description => "Set SMBIOS1 serial number.",
2451         optional => 1,
2452     },
2453     manufacturer => {
2454         type => 'string',
2455         pattern => '[A-Za-z0-9+\/]+={0,2}',
2456         format_description => 'Base64 encoded string',
2457         description => "Set SMBIOS1 manufacturer.",
2458         optional => 1,
2459     },
2460     product => {
2461         type => 'string',
2462         pattern => '[A-Za-z0-9+\/]+={0,2}',
2463         format_description => 'Base64 encoded string',
2464         description => "Set SMBIOS1 product ID.",
2465         optional => 1,
2466     },
2467     sku => {
2468         type => 'string',
2469         pattern => '[A-Za-z0-9+\/]+={0,2}',
2470         format_description => 'Base64 encoded string',
2471         description => "Set SMBIOS1 SKU string.",
2472         optional => 1,
2473     },
2474     family => {
2475         type => 'string',
2476         pattern => '[A-Za-z0-9+\/]+={0,2}',
2477         format_description => 'Base64 encoded string',
2478         description => "Set SMBIOS1 family string.",
2479         optional => 1,
2480     },
2481     base64 => {
2482         type => 'boolean',
2483         description => 'Flag to indicate that the SMBIOS values are base64 encoded',
2484         optional => 1,
2485     },
2486 };
2487
2488 sub parse_smbios1 {
2489     my ($data) = @_;
2490
2491     my $res = eval { PVE::JSONSchema::parse_property_string($smbios1_fmt, $data) };
2492     warn $@ if $@;
2493     return $res;
2494 }
2495
2496 sub print_smbios1 {
2497     my ($smbios1) = @_;
2498     return PVE::JSONSchema::print_property_string($smbios1, $smbios1_fmt);
2499 }
2500
2501 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_fmt);
2502
2503 PVE::JSONSchema::register_format('pve-qm-bootdisk', \&verify_bootdisk);
2504 sub verify_bootdisk {
2505     my ($value, $noerr) = @_;
2506
2507     return $value if is_valid_drivename($value);
2508
2509     return undef if $noerr;
2510
2511     die "invalid boot disk '$value'\n";
2512 }
2513
2514 sub parse_watchdog {
2515     my ($value) = @_;
2516
2517     return undef if !$value;
2518
2519     my $res = eval { PVE::JSONSchema::parse_property_string($watchdog_fmt, $value) };
2520     warn $@ if $@;
2521     return $res;
2522 }
2523
2524 sub parse_guest_agent {
2525     my ($value) = @_;
2526
2527     return {} if !defined($value->{agent});
2528
2529     my $res = eval { PVE::JSONSchema::parse_property_string($agent_fmt, $value->{agent}) };
2530     warn $@ if $@;
2531
2532     # if the agent is disabled ignore the other potentially set properties
2533     return {} if !$res->{enabled};
2534     return $res;
2535 }
2536
2537 sub parse_vga {
2538     my ($value) = @_;
2539
2540     return {} if !$value;
2541     my $res = eval { PVE::JSONSchema::parse_property_string($vga_fmt, $value) };
2542     warn $@ if $@;
2543     return $res;
2544 }
2545
2546 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
2547 sub verify_usb_device {
2548     my ($value, $noerr) = @_;
2549
2550     return $value if parse_usb_device($value);
2551
2552     return undef if $noerr;
2553
2554     die "unable to parse usb device\n";
2555 }
2556
2557 # add JSON properties for create and set function
2558 sub json_config_properties {
2559     my $prop = shift;
2560
2561     foreach my $opt (keys %$confdesc) {
2562         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate' || $opt eq 'runningmachine';
2563         $prop->{$opt} = $confdesc->{$opt};
2564     }
2565
2566     return $prop;
2567 }
2568
2569 # return copy of $confdesc_cloudinit to generate documentation
2570 sub cloudinit_config_properties {
2571
2572     return dclone($confdesc_cloudinit);
2573 }
2574
2575 sub check_type {
2576     my ($key, $value) = @_;
2577
2578     die "unknown setting '$key'\n" if !$confdesc->{$key};
2579
2580     my $type = $confdesc->{$key}->{type};
2581
2582     if (!defined($value)) {
2583         die "got undefined value\n";
2584     }
2585
2586     if ($value =~ m/[\n\r]/) {
2587         die "property contains a line feed\n";
2588     }
2589
2590     if ($type eq 'boolean') {
2591         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
2592         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
2593         die "type check ('boolean') failed - got '$value'\n";
2594     } elsif ($type eq 'integer') {
2595         return int($1) if $value =~ m/^(\d+)$/;
2596         die "type check ('integer') failed - got '$value'\n";
2597     } elsif ($type eq 'number') {
2598         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
2599         die "type check ('number') failed - got '$value'\n";
2600     } elsif ($type eq 'string') {
2601         if (my $fmt = $confdesc->{$key}->{format}) {
2602             PVE::JSONSchema::check_format($fmt, $value);
2603             return $value;
2604         }
2605         $value =~ s/^\"(.*)\"$/$1/;
2606         return $value;
2607     } else {
2608         die "internal error"
2609     }
2610 }
2611
2612 sub touch_config {
2613     my ($vmid) = @_;
2614
2615     my $conf = PVE::QemuConfig->config_file($vmid);
2616     utime undef, undef, $conf;
2617 }
2618
2619 sub destroy_vm {
2620     my ($storecfg, $vmid, $keep_empty_config, $skiplock) = @_;
2621
2622     my $conffile = PVE::QemuConfig->config_file($vmid);
2623
2624     my $conf = PVE::QemuConfig->load_config($vmid);
2625
2626     PVE::QemuConfig->check_lock($conf) if !$skiplock;
2627
2628     if ($conf->{template}) {
2629         # check if any base image is still used by a linked clone
2630         foreach_drive($conf, sub {
2631                 my ($ds, $drive) = @_;
2632
2633                 return if drive_is_cdrom($drive);
2634
2635                 my $volid = $drive->{file};
2636
2637                 return if !$volid || $volid =~ m|^/|;
2638
2639                 die "base volume '$volid' is still in use by linked cloned\n"
2640                     if PVE::Storage::volume_is_base_and_used($storecfg, $volid);
2641
2642         });
2643     }
2644
2645     # only remove disks owned by this VM
2646     foreach_drive($conf, sub {
2647         my ($ds, $drive) = @_;
2648
2649         return if drive_is_cdrom($drive, 1);
2650
2651         my $volid = $drive->{file};
2652
2653         return if !$volid || $volid =~ m|^/|;
2654
2655         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
2656         return if !$path || !$owner || ($owner != $vmid);
2657
2658         eval {
2659             PVE::Storage::vdisk_free($storecfg, $volid);
2660         };
2661         warn "Could not remove disk '$volid', check manually: $@" if $@;
2662
2663     });
2664
2665     if ($keep_empty_config) {
2666         PVE::Tools::file_set_contents($conffile, "memory: 128\n");
2667     } else {
2668         unlink $conffile;
2669     }
2670
2671     # also remove unused disk
2672     eval {
2673         my $dl = PVE::Storage::vdisk_list($storecfg, undef, $vmid);
2674
2675         eval {
2676             PVE::Storage::foreach_volid($dl, sub {
2677                 my ($volid, $sid, $volname, $d) = @_;
2678                 PVE::Storage::vdisk_free($storecfg, $volid);
2679             });
2680         };
2681         warn $@ if $@;
2682
2683     };
2684     warn $@ if $@;
2685 }
2686
2687 sub parse_vm_config {
2688     my ($filename, $raw) = @_;
2689
2690     return undef if !defined($raw);
2691
2692     my $res = {
2693         digest => Digest::SHA::sha1_hex($raw),
2694         snapshots => {},
2695         pending => {},
2696     };
2697
2698     $filename =~ m|/qemu-server/(\d+)\.conf$|
2699         || die "got strange filename '$filename'";
2700
2701     my $vmid = $1;
2702
2703     my $conf = $res;
2704     my $descr;
2705     my $section = '';
2706
2707     my @lines = split(/\n/, $raw);
2708     foreach my $line (@lines) {
2709         next if $line =~ m/^\s*$/;
2710
2711         if ($line =~ m/^\[PENDING\]\s*$/i) {
2712             $section = 'pending';
2713             if (defined($descr)) {
2714                 $descr =~ s/\s+$//;
2715                 $conf->{description} = $descr;
2716             }
2717             $descr = undef;
2718             $conf = $res->{$section} = {};
2719             next;
2720
2721         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2722             $section = $1;
2723             if (defined($descr)) {
2724                 $descr =~ s/\s+$//;
2725                 $conf->{description} = $descr;
2726             }
2727             $descr = undef;
2728             $conf = $res->{snapshots}->{$section} = {};
2729             next;
2730         }
2731
2732         if ($line =~ m/^\#(.*)\s*$/) {
2733             $descr = '' if !defined($descr);
2734             $descr .= PVE::Tools::decode_text($1) . "\n";
2735             next;
2736         }
2737
2738         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2739             $descr = '' if !defined($descr);
2740             $descr .= PVE::Tools::decode_text($2);
2741         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2742             $conf->{snapstate} = $1;
2743         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2744             my $key = $1;
2745             my $value = $2;
2746             $conf->{$key} = $value;
2747         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2748             my $value = $1;
2749             if ($section eq 'pending') {
2750                 $conf->{delete} = $value; # we parse this later
2751             } else {
2752                 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
2753             }
2754         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(.+?)\s*$/) {
2755             my $key = $1;
2756             my $value = $2;
2757             eval { $value = check_type($key, $value); };
2758             if ($@) {
2759                 warn "vm $vmid - unable to parse value of '$key' - $@";
2760             } else {
2761                 $key = 'ide2' if $key eq 'cdrom';
2762                 my $fmt = $confdesc->{$key}->{format};
2763                 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2764                     my $v = parse_drive($key, $value);
2765                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2766                         $v->{file} = $volid;
2767                         $value = print_drive($vmid, $v);
2768                     } else {
2769                         warn "vm $vmid - unable to parse value of '$key'\n";
2770                         next;
2771                     }
2772                 }
2773
2774                 $conf->{$key} = $value;
2775             }
2776         }
2777     }
2778
2779     if (defined($descr)) {
2780         $descr =~ s/\s+$//;
2781         $conf->{description} = $descr;
2782     }
2783     delete $res->{snapstate}; # just to be sure
2784
2785     return $res;
2786 }
2787
2788 sub write_vm_config {
2789     my ($filename, $conf) = @_;
2790
2791     delete $conf->{snapstate}; # just to be sure
2792
2793     if ($conf->{cdrom}) {
2794         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2795         $conf->{ide2} = $conf->{cdrom};
2796         delete $conf->{cdrom};
2797     }
2798
2799     # we do not use 'smp' any longer
2800     if ($conf->{sockets}) {
2801         delete $conf->{smp};
2802     } elsif ($conf->{smp}) {
2803         $conf->{sockets} = $conf->{smp};
2804         delete $conf->{cores};
2805         delete $conf->{smp};
2806     }
2807
2808     my $used_volids = {};
2809
2810     my $cleanup_config = sub {
2811         my ($cref, $pending, $snapname) = @_;
2812
2813         foreach my $key (keys %$cref) {
2814             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2815                 $key eq 'snapstate' || $key eq 'pending';
2816             my $value = $cref->{$key};
2817             if ($key eq 'delete') {
2818                 die "propertry 'delete' is only allowed in [PENDING]\n"
2819                     if !$pending;
2820                 # fixme: check syntax?
2821                 next;
2822             }
2823             eval { $value = check_type($key, $value); };
2824             die "unable to parse value of '$key' - $@" if $@;
2825
2826             $cref->{$key} = $value;
2827
2828             if (!$snapname && is_valid_drivename($key)) {
2829                 my $drive = parse_drive($key, $value);
2830                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2831             }
2832         }
2833     };
2834
2835     &$cleanup_config($conf);
2836
2837     &$cleanup_config($conf->{pending}, 1);
2838
2839     foreach my $snapname (keys %{$conf->{snapshots}}) {
2840         die "internal error" if $snapname eq 'pending';
2841         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2842     }
2843
2844     # remove 'unusedX' settings if we re-add a volume
2845     foreach my $key (keys %$conf) {
2846         my $value = $conf->{$key};
2847         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2848             delete $conf->{$key};
2849         }
2850     }
2851
2852     my $generate_raw_config = sub {
2853         my ($conf, $pending) = @_;
2854
2855         my $raw = '';
2856
2857         # add description as comment to top of file
2858         if (defined(my $descr = $conf->{description})) {
2859             if ($descr) {
2860                 foreach my $cl (split(/\n/, $descr)) {
2861                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2862                 }
2863             } else {
2864                 $raw .= "#\n" if $pending;
2865             }
2866         }
2867
2868         foreach my $key (sort keys %$conf) {
2869             next if $key eq 'digest' || $key eq 'description' || $key eq 'pending' || $key eq 'snapshots';
2870             $raw .= "$key: $conf->{$key}\n";
2871         }
2872         return $raw;
2873     };
2874
2875     my $raw = &$generate_raw_config($conf);
2876
2877     if (scalar(keys %{$conf->{pending}})){
2878         $raw .= "\n[PENDING]\n";
2879         $raw .= &$generate_raw_config($conf->{pending}, 1);
2880     }
2881
2882     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2883         $raw .= "\n[$snapname]\n";
2884         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2885     }
2886
2887     return $raw;
2888 }
2889
2890 sub load_defaults {
2891
2892     my $res = {};
2893
2894     # we use static defaults from our JSON schema configuration
2895     foreach my $key (keys %$confdesc) {
2896         if (defined(my $default = $confdesc->{$key}->{default})) {
2897             $res->{$key} = $default;
2898         }
2899     }
2900
2901     return $res;
2902 }
2903
2904 sub config_list {
2905     my $vmlist = PVE::Cluster::get_vmlist();
2906     my $res = {};
2907     return $res if !$vmlist || !$vmlist->{ids};
2908     my $ids = $vmlist->{ids};
2909
2910     foreach my $vmid (keys %$ids) {
2911         my $d = $ids->{$vmid};
2912         next if !$d->{node} || $d->{node} ne $nodename;
2913         next if !$d->{type} || $d->{type} ne 'qemu';
2914         $res->{$vmid}->{exists} = 1;
2915     }
2916     return $res;
2917 }
2918
2919 # test if VM uses local resources (to prevent migration)
2920 sub check_local_resources {
2921     my ($conf, $noerr) = @_;
2922
2923     my @loc_res = ();
2924
2925     push @loc_res, "hostusb" if $conf->{hostusb}; # old syntax
2926     push @loc_res, "hostpci" if $conf->{hostpci}; # old syntax
2927
2928     push @loc_res, "ivshmem" if $conf->{ivshmem};
2929
2930     foreach my $k (keys %$conf) {
2931         next if $k =~ m/^usb/ && ($conf->{$k} =~ m/^spice(?![^,])/);
2932         # sockets are safe: they will recreated be on the target side post-migrate
2933         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2934         push @loc_res, $k if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2935     }
2936
2937     die "VM uses local resources\n" if scalar @loc_res && !$noerr;
2938
2939     return \@loc_res;
2940 }
2941
2942 # check if used storages are available on all nodes (use by migrate)
2943 sub check_storage_availability {
2944     my ($storecfg, $conf, $node) = @_;
2945
2946     foreach_drive($conf, sub {
2947         my ($ds, $drive) = @_;
2948
2949         my $volid = $drive->{file};
2950         return if !$volid;
2951
2952         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2953         return if !$sid;
2954
2955         # check if storage is available on both nodes
2956         my $scfg = PVE::Storage::storage_check_node($storecfg, $sid);
2957         PVE::Storage::storage_check_node($storecfg, $sid, $node);
2958    });
2959 }
2960
2961 # list nodes where all VM images are available (used by has_feature API)
2962 sub shared_nodes {
2963     my ($conf, $storecfg) = @_;
2964
2965     my $nodelist = PVE::Cluster::get_nodelist();
2966     my $nodehash = { map { $_ => 1 } @$nodelist };
2967     my $nodename = PVE::INotify::nodename();
2968
2969     foreach_drive($conf, sub {
2970         my ($ds, $drive) = @_;
2971
2972         my $volid = $drive->{file};
2973         return if !$volid;
2974
2975         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2976         if ($storeid) {
2977             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2978             if ($scfg->{disable}) {
2979                 $nodehash = {};
2980             } elsif (my $avail = $scfg->{nodes}) {
2981                 foreach my $node (keys %$nodehash) {
2982                     delete $nodehash->{$node} if !$avail->{$node};
2983                 }
2984             } elsif (!$scfg->{shared}) {
2985                 foreach my $node (keys %$nodehash) {
2986                     delete $nodehash->{$node} if $node ne $nodename
2987                 }
2988             }
2989         }
2990     });
2991
2992     return $nodehash
2993 }
2994
2995 sub check_local_storage_availability {
2996     my ($conf, $storecfg) = @_;
2997
2998     my $nodelist = PVE::Cluster::get_nodelist();
2999     my $nodehash = { map { $_ => {} } @$nodelist };
3000
3001     foreach_drive($conf, sub {
3002         my ($ds, $drive) = @_;
3003
3004         my $volid = $drive->{file};
3005         return if !$volid;
3006
3007         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
3008         if ($storeid) {
3009             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
3010
3011             if ($scfg->{disable}) {
3012                 foreach my $node (keys %$nodehash) {
3013                     $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
3014                 }
3015             } elsif (my $avail = $scfg->{nodes}) {
3016                 foreach my $node (keys %$nodehash) {
3017                     if (!$avail->{$node}) {
3018                         $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
3019                     }
3020                 }
3021             }
3022         }
3023     });
3024
3025     foreach my $node (values %$nodehash) {
3026         if (my $unavail = $node->{unavailable_storages}) {
3027             $node->{unavailable_storages} = [ sort keys %$unavail ];
3028         }
3029     }
3030
3031     return $nodehash
3032 }
3033
3034 sub check_cmdline {
3035     my ($pidfile, $pid) = @_;
3036
3037     my $fh = IO::File->new("/proc/$pid/cmdline", "r");
3038     if (defined($fh)) {
3039         my $line = <$fh>;
3040         $fh->close;
3041         return undef if !$line;
3042         my @param = split(/\0/, $line);
3043
3044         my $cmd = $param[0];
3045         return if !$cmd || ($cmd !~ m|kvm$| && $cmd !~ m@(?:^|/)qemu-system-[^/]+$@);
3046
3047         for (my $i = 0; $i < scalar (@param); $i++) {
3048             my $p = $param[$i];
3049             next if !$p;
3050             if (($p eq '-pidfile') || ($p eq '--pidfile')) {
3051                 my $p = $param[$i+1];
3052                 return 1 if $p && ($p eq $pidfile);
3053                 return undef;
3054             }
3055         }
3056     }
3057     return undef;
3058 }
3059
3060 sub check_running {
3061     my ($vmid, $nocheck, $node) = @_;
3062
3063     my $filename = PVE::QemuConfig->config_file($vmid, $node);
3064
3065     die "unable to find configuration file for VM $vmid - no such machine\n"
3066         if !$nocheck && ! -f $filename;
3067
3068     my $pidfile = pidfile_name($vmid);
3069
3070     if (my $fd = IO::File->new("<$pidfile")) {
3071         my $st = stat($fd);
3072         my $line = <$fd>;
3073         close($fd);
3074
3075         my $mtime = $st->mtime;
3076         if ($mtime > time()) {
3077             warn "file '$filename' modified in future\n";
3078         }
3079
3080         if ($line =~ m/^(\d+)$/) {
3081             my $pid = $1;
3082             if (check_cmdline($pidfile, $pid)) {
3083                 if (my $pinfo = PVE::ProcFSTools::check_process_running($pid)) {
3084                     return $pid;
3085                 }
3086             }
3087         }
3088     }
3089
3090     return undef;
3091 }
3092
3093 sub vzlist {
3094
3095     my $vzlist = config_list();
3096
3097     my $fd = IO::Dir->new($var_run_tmpdir) || return $vzlist;
3098
3099     while (defined(my $de = $fd->read)) {
3100         next if $de !~ m/^(\d+)\.pid$/;
3101         my $vmid = $1;
3102         next if !defined($vzlist->{$vmid});
3103         if (my $pid = check_running($vmid)) {
3104             $vzlist->{$vmid}->{pid} = $pid;
3105         }
3106     }
3107
3108     return $vzlist;
3109 }
3110
3111 sub disksize {
3112     my ($storecfg, $conf) = @_;
3113
3114     my $bootdisk = $conf->{bootdisk};
3115     return undef if !$bootdisk;
3116     return undef if !is_valid_drivename($bootdisk);
3117
3118     return undef if !$conf->{$bootdisk};
3119
3120     my $drive = parse_drive($bootdisk, $conf->{$bootdisk});
3121     return undef if !defined($drive);
3122
3123     return undef if drive_is_cdrom($drive);
3124
3125     my $volid = $drive->{file};
3126     return undef if !$volid;
3127
3128     return $drive->{size};
3129 }
3130
3131 our $vmstatus_return_properties = {
3132     vmid => get_standard_option('pve-vmid'),
3133     status => {
3134         description => "Qemu process status.",
3135         type => 'string',
3136         enum => ['stopped', 'running'],
3137     },
3138     maxmem => {
3139         description => "Maximum memory in bytes.",
3140         type => 'integer',
3141         optional => 1,
3142         renderer => 'bytes',
3143     },
3144     maxdisk => {
3145         description => "Root disk size in bytes.",
3146         type => 'integer',
3147         optional => 1,
3148         renderer => 'bytes',
3149     },
3150     name => {
3151         description => "VM name.",
3152         type => 'string',
3153         optional => 1,
3154     },
3155     qmpstatus => {
3156         description => "Qemu QMP agent status.",
3157         type => 'string',
3158         optional => 1,
3159     },
3160     pid => {
3161         description => "PID of running qemu process.",
3162         type => 'integer',
3163         optional => 1,
3164     },
3165     uptime => {
3166         description => "Uptime.",
3167         type => 'integer',
3168         optional => 1,
3169         renderer => 'duration',
3170     },
3171     cpus => {
3172         description => "Maximum usable CPUs.",
3173         type => 'number',
3174         optional => 1,
3175     },
3176     lock => {
3177         description => "The current config lock, if any.",
3178         type => 'string',
3179         optional => 1,
3180     }
3181 };
3182
3183 my $last_proc_pid_stat;
3184
3185 # get VM status information
3186 # This must be fast and should not block ($full == false)
3187 # We only query KVM using QMP if $full == true (this can be slow)
3188 sub vmstatus {
3189     my ($opt_vmid, $full) = @_;
3190
3191     my $res = {};
3192
3193     my $storecfg = PVE::Storage::config();
3194
3195     my $list = vzlist();
3196     my $defaults = load_defaults();
3197
3198     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
3199
3200     my $cpucount = $cpuinfo->{cpus} || 1;
3201
3202     foreach my $vmid (keys %$list) {
3203         next if $opt_vmid && ($vmid ne $opt_vmid);
3204
3205         my $cfspath = PVE::QemuConfig->cfs_config_path($vmid);
3206         my $conf = PVE::Cluster::cfs_read_file($cfspath) || {};
3207
3208         my $d = { vmid => $vmid };
3209         $d->{pid} = $list->{$vmid}->{pid};
3210
3211         # fixme: better status?
3212         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
3213
3214         my $size = disksize($storecfg, $conf);
3215         if (defined($size)) {
3216             $d->{disk} = 0; # no info available
3217             $d->{maxdisk} = $size;
3218         } else {
3219             $d->{disk} = 0;
3220             $d->{maxdisk} = 0;
3221         }
3222
3223         $d->{cpus} = ($conf->{sockets} || $defaults->{sockets})
3224             * ($conf->{cores} || $defaults->{cores});
3225         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
3226         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
3227
3228         $d->{name} = $conf->{name} || "VM $vmid";
3229         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024)
3230             : $defaults->{memory}*(1024*1024);
3231
3232         if ($conf->{balloon}) {
3233             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
3234             $d->{shares} = defined($conf->{shares}) ? $conf->{shares}
3235                 : $defaults->{shares};
3236         }
3237
3238         $d->{uptime} = 0;
3239         $d->{cpu} = 0;
3240         $d->{mem} = 0;
3241
3242         $d->{netout} = 0;
3243         $d->{netin} = 0;
3244
3245         $d->{diskread} = 0;
3246         $d->{diskwrite} = 0;
3247
3248         $d->{template} = PVE::QemuConfig->is_template($conf);
3249
3250         $d->{serial} = 1 if conf_has_serial($conf);
3251         $d->{lock} = $conf->{lock} if $conf->{lock};
3252
3253         $res->{$vmid} = $d;
3254     }
3255
3256     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
3257     foreach my $dev (keys %$netdev) {
3258         next if $dev !~ m/^tap([1-9]\d*)i/;
3259         my $vmid = $1;
3260         my $d = $res->{$vmid};
3261         next if !$d;
3262
3263         $d->{netout} += $netdev->{$dev}->{receive};
3264         $d->{netin} += $netdev->{$dev}->{transmit};
3265
3266         if ($full) {
3267             $d->{nics}->{$dev}->{netout} = $netdev->{$dev}->{receive};
3268             $d->{nics}->{$dev}->{netin} = $netdev->{$dev}->{transmit};
3269         }
3270
3271     }
3272
3273     my $ctime = gettimeofday;
3274
3275     foreach my $vmid (keys %$list) {
3276
3277         my $d = $res->{$vmid};
3278         my $pid = $d->{pid};
3279         next if !$pid;
3280
3281         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
3282         next if !$pstat; # not running
3283
3284         my $used = $pstat->{utime} + $pstat->{stime};
3285
3286         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
3287
3288         if ($pstat->{vsize}) {
3289             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
3290         }
3291
3292         my $old = $last_proc_pid_stat->{$pid};
3293         if (!$old) {
3294             $last_proc_pid_stat->{$pid} = {
3295                 time => $ctime,
3296                 used => $used,
3297                 cpu => 0,
3298             };
3299             next;
3300         }
3301
3302         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
3303
3304         if ($dtime > 1000) {
3305             my $dutime = $used -  $old->{used};
3306
3307             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
3308             $last_proc_pid_stat->{$pid} = {
3309                 time => $ctime,
3310                 used => $used,
3311                 cpu => $d->{cpu},
3312             };
3313         } else {
3314             $d->{cpu} = $old->{cpu};
3315         }
3316     }
3317
3318     return $res if !$full;
3319
3320     my $qmpclient = PVE::QMPClient->new();
3321
3322     my $ballooncb = sub {
3323         my ($vmid, $resp) = @_;
3324
3325         my $info = $resp->{'return'};
3326         return if !$info->{max_mem};
3327
3328         my $d = $res->{$vmid};
3329
3330         # use memory assigned to VM
3331         $d->{maxmem} = $info->{max_mem};
3332         $d->{balloon} = $info->{actual};
3333
3334         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
3335             $d->{mem} = $info->{total_mem} - $info->{free_mem};
3336             $d->{freemem} = $info->{free_mem};
3337         }
3338
3339         $d->{ballooninfo} = $info;
3340     };
3341
3342     my $blockstatscb = sub {
3343         my ($vmid, $resp) = @_;
3344         my $data = $resp->{'return'} || [];
3345         my $totalrdbytes = 0;
3346         my $totalwrbytes = 0;
3347
3348         for my $blockstat (@$data) {
3349             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
3350             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
3351
3352             $blockstat->{device} =~ s/drive-//;
3353             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
3354         }
3355         $res->{$vmid}->{diskread} = $totalrdbytes;
3356         $res->{$vmid}->{diskwrite} = $totalwrbytes;
3357     };
3358
3359     my $statuscb = sub {
3360         my ($vmid, $resp) = @_;
3361
3362         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
3363         # this fails if ballon driver is not loaded, so this must be
3364         # the last commnand (following command are aborted if this fails).
3365         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
3366
3367         my $status = 'unknown';
3368         if (!defined($status = $resp->{'return'}->{status})) {
3369             warn "unable to get VM status\n";
3370             return;
3371         }
3372
3373         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
3374     };
3375
3376     foreach my $vmid (keys %$list) {
3377         next if $opt_vmid && ($vmid ne $opt_vmid);
3378         next if !$res->{$vmid}->{pid}; # not running
3379         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
3380     }
3381
3382     $qmpclient->queue_execute(undef, 2);
3383
3384     foreach my $vmid (keys %$list) {
3385         next if $opt_vmid && ($vmid ne $opt_vmid);
3386         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
3387     }
3388
3389     return $res;
3390 }
3391
3392 sub foreach_drive {
3393     my ($conf, $func, @param) = @_;
3394
3395     foreach my $ds (valid_drive_names()) {
3396         next if !defined($conf->{$ds});
3397
3398         my $drive = parse_drive($ds, $conf->{$ds});
3399         next if !$drive;
3400
3401         &$func($ds, $drive, @param);
3402     }
3403 }
3404
3405 sub foreach_volid {
3406     my ($conf, $func, @param) = @_;
3407
3408     my $volhash = {};
3409
3410     my $test_volid = sub {
3411         my ($volid, $is_cdrom, $replicate, $shared, $snapname, $size) = @_;
3412
3413         return if !$volid;
3414
3415         $volhash->{$volid}->{cdrom} //= 1;
3416         $volhash->{$volid}->{cdrom} = 0 if !$is_cdrom;
3417
3418         $volhash->{$volid}->{replicate} //= 0;
3419         $volhash->{$volid}->{replicate} = 1 if $replicate;
3420
3421         $volhash->{$volid}->{shared} //= 0;
3422         $volhash->{$volid}->{shared} = 1 if $shared;
3423
3424         $volhash->{$volid}->{referenced_in_config} //= 0;
3425         $volhash->{$volid}->{referenced_in_config} = 1 if !defined($snapname);
3426
3427         $volhash->{$volid}->{referenced_in_snapshot}->{$snapname} = 1
3428             if defined($snapname);
3429         $volhash->{$volid}->{size} = $size if $size;
3430     };
3431
3432     foreach_drive($conf, sub {
3433         my ($ds, $drive) = @_;
3434         $test_volid->($drive->{file}, drive_is_cdrom($drive), $drive->{replicate} // 1, $drive->{shared}, undef, $drive->{size});
3435     });
3436
3437     foreach my $snapname (keys %{$conf->{snapshots}}) {
3438         my $snap = $conf->{snapshots}->{$snapname};
3439         $test_volid->($snap->{vmstate}, 0, 1, $snapname);
3440         foreach_drive($snap, sub {
3441             my ($ds, $drive) = @_;
3442             $test_volid->($drive->{file}, drive_is_cdrom($drive), $drive->{replicate} // 1, $drive->{shared}, $snapname);
3443         });
3444     }
3445
3446     foreach my $volid (keys %$volhash) {
3447         &$func($volid, $volhash->{$volid}, @param);
3448     }
3449 }
3450
3451 sub conf_has_serial {
3452     my ($conf) = @_;
3453
3454     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3455         if ($conf->{"serial$i"}) {
3456             return 1;
3457         }
3458     }
3459
3460     return 0;
3461 }
3462
3463 sub conf_has_audio {
3464     my ($conf, $id) = @_;
3465
3466     $id //= 0;
3467     my $audio = $conf->{"audio$id"};
3468     return undef if !defined($audio);
3469
3470     my $audioproperties = PVE::JSONSchema::parse_property_string($audio_fmt, $audio);
3471     my $audiodriver = $audioproperties->{driver} // 'spice';
3472
3473     return {
3474         dev => $audioproperties->{device},
3475         dev_id => "audiodev$id",
3476         backend => $audiodriver,
3477         backend_id => "$audiodriver-backend${id}",
3478     };
3479 }
3480
3481 sub vga_conf_has_spice {
3482     my ($vga) = @_;
3483
3484     my $vgaconf = parse_vga($vga);
3485     my $vgatype = $vgaconf->{type};
3486     return 0 if !$vgatype || $vgatype !~ m/^qxl([234])?$/;
3487
3488     return $1 || 1;
3489 }
3490
3491 my $host_arch; # FIXME: fix PVE::Tools::get_host_arch
3492 sub get_host_arch() {
3493     $host_arch = (POSIX::uname())[4] if !$host_arch;
3494     return $host_arch;
3495 }
3496
3497 sub is_native($) {
3498     my ($arch) = @_;
3499     return get_host_arch() eq $arch;
3500 }
3501
3502 my $default_machines = {
3503     x86_64 => 'pc',
3504     aarch64 => 'virt',
3505 };
3506
3507 sub get_basic_machine_info {
3508     my ($conf, $forcemachine) = @_;
3509
3510     my $arch = $conf->{arch} // get_host_arch();
3511     my $machine = $forcemachine || $conf->{machine} || $default_machines->{$arch};
3512     return ($arch, $machine);
3513 }
3514
3515 sub get_ovmf_files($) {
3516     my ($arch) = @_;
3517
3518     my $ovmf = $OVMF->{$arch}
3519         or die "no OVMF images known for architecture '$arch'\n";
3520
3521     return @$ovmf;
3522 }
3523
3524 my $Arch2Qemu = {
3525     aarch64 => '/usr/bin/qemu-system-aarch64',
3526     x86_64 => '/usr/bin/qemu-system-x86_64',
3527 };
3528 sub get_command_for_arch($) {
3529     my ($arch) = @_;
3530     return '/usr/bin/kvm' if is_native($arch);
3531
3532     my $cmd = $Arch2Qemu->{$arch}
3533         or die "don't know how to emulate architecture '$arch'\n";
3534     return $cmd;
3535 }
3536
3537 sub get_cpu_options {
3538     my ($conf, $arch, $kvm, $machine_type, $kvm_off, $kvmver, $winversion, $gpu_passthrough) = @_;
3539
3540     my $cpuFlags = [];
3541     my $ostype = $conf->{ostype};
3542
3543     my $cpu = $kvm ? "kvm64" : "qemu64";
3544     if ($arch eq 'aarch64') {
3545         $cpu = 'cortex-a57';
3546     }
3547     my $hv_vendor_id;
3548     if (my $cputype = $conf->{cpu}) {
3549         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
3550             or die "Cannot parse cpu description: $cputype\n";
3551         $cpu = $cpuconf->{cputype};
3552         $kvm_off = 1 if $cpuconf->{hidden};
3553         $hv_vendor_id = $cpuconf->{'hv-vendor-id'};
3554
3555         if (defined(my $flags = $cpuconf->{flags})) {
3556             push @$cpuFlags, split(";", $flags);
3557         }
3558     }
3559
3560     push @$cpuFlags , '+lahf_lm' if $cpu eq 'kvm64' && $arch eq 'x86_64';
3561
3562     push @$cpuFlags , '-x2apic'
3563         if $conf->{ostype} && $conf->{ostype} eq 'solaris';
3564
3565     push @$cpuFlags, '+sep' if $cpu eq 'kvm64' || $cpu eq 'kvm32';
3566
3567     push @$cpuFlags, '-rdtscp' if $cpu =~ m/^Opteron/;
3568
3569     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3) && $arch eq 'x86_64') {
3570
3571         push @$cpuFlags , '+kvm_pv_unhalt' if $kvm;
3572         push @$cpuFlags , '+kvm_pv_eoi' if $kvm;
3573     }
3574
3575     add_hyperv_enlightenments($cpuFlags, $winversion, $machine_type, $kvmver, $conf->{bios}, $gpu_passthrough, $hv_vendor_id) if $kvm;
3576
3577     push @$cpuFlags, 'enforce' if $cpu ne 'host' && $kvm && $arch eq 'x86_64';
3578
3579     push @$cpuFlags, 'kvm=off' if $kvm_off;
3580
3581     if (my $cpu_vendor = $cpu_vendor_list->{$cpu}) {
3582         push @$cpuFlags, "vendor=${cpu_vendor}"
3583             if $cpu_vendor ne 'default';
3584     } elsif ($arch ne 'aarch64') {
3585         die "internal error"; # should not happen
3586     }
3587
3588     $cpu .= "," . join(',', @$cpuFlags) if scalar(@$cpuFlags);
3589
3590     return ('-cpu', $cpu);
3591 }
3592
3593 sub config_to_command {
3594     my ($storecfg, $vmid, $conf, $defaults, $forcemachine) = @_;
3595
3596     my $cmd = [];
3597     my $globalFlags = [];
3598     my $machineFlags = [];
3599     my $rtcFlags = [];
3600     my $devices = [];
3601     my $pciaddr = '';
3602     my $bridges = {};
3603     my $vernum = 0; # unknown
3604     my $ostype = $conf->{ostype};
3605     my $winversion = windows_version($ostype);
3606     my $kvm = $conf->{kvm};
3607
3608     my ($arch, $machine_type) = get_basic_machine_info($conf, $forcemachine);
3609     my $kvm_binary = get_command_for_arch($arch);
3610     my $kvmver = kvm_user_version($kvm_binary);
3611     $kvm //= 1 if is_native($arch);
3612
3613     if ($kvm) {
3614         die "KVM virtualisation configured, but not available. Either disable in VM configuration or enable in BIOS.\n"
3615             if !defined kvm_version();
3616     }
3617
3618     if ($kvmver =~ m/^(\d+)\.(\d+)$/) {
3619         $vernum = $1*1000000+$2*1000;
3620     } elsif ($kvmver =~ m/^(\d+)\.(\d+)\.(\d+)$/) {
3621         $vernum = $1*1000000+$2*1000+$3;
3622     }
3623
3624     die "detected old qemu-kvm binary ($kvmver)\n" if $vernum < 15000;
3625
3626     my $q35 = machine_type_is_q35($conf);
3627     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
3628     my $use_old_bios_files = undef;
3629     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3630
3631     my $cpuunits = defined($conf->{cpuunits}) ?
3632             $conf->{cpuunits} : $defaults->{cpuunits};
3633
3634     push @$cmd, $kvm_binary;
3635
3636     push @$cmd, '-id', $vmid;
3637
3638     my $vmname = $conf->{name} || "vm$vmid";
3639
3640     push @$cmd, '-name', $vmname;
3641
3642     my $use_virtio = 0;
3643
3644     my $qmpsocket = qmp_socket($vmid);
3645     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server,nowait";
3646     push @$cmd, '-mon', "chardev=qmp,mode=control";
3647
3648     if (qemu_machine_feature_enabled($machine_type, $kvmver, 2, 12)) {
3649         push @$cmd, '-chardev', "socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5";
3650         push @$cmd, '-mon', "chardev=qmp-event,mode=control";
3651     }
3652
3653     push @$cmd, '-pidfile' , pidfile_name($vmid);
3654
3655     push @$cmd, '-daemonize';
3656
3657     if ($conf->{smbios1}) {
3658         my $smbios_conf = parse_smbios1($conf->{smbios1});
3659         if ($smbios_conf->{base64}) {
3660             # Do not pass base64 flag to qemu
3661             delete $smbios_conf->{base64};
3662             my $smbios_string = "";
3663             foreach my $key (keys %$smbios_conf) {
3664                 my $value;
3665                 if ($key eq "uuid") {
3666                     $value = $smbios_conf->{uuid}
3667                 } else {
3668                     $value = decode_base64($smbios_conf->{$key});
3669                 }
3670                 # qemu accepts any binary data, only commas need escaping by double comma
3671                 $value =~ s/,/,,/g;
3672                 $smbios_string .= "," . $key . "=" . $value if $value;
3673             }
3674             push @$cmd, '-smbios', "type=1" . $smbios_string;
3675         } else {
3676             push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
3677         }
3678     }
3679
3680     if ($conf->{vmgenid}) {
3681         push @$devices, '-device', 'vmgenid,guid='.$conf->{vmgenid};
3682     }
3683
3684     my ($ovmf_code, $ovmf_vars) = get_ovmf_files($arch);
3685     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3686         die "uefi base image not found\n" if ! -f $ovmf_code;
3687
3688         my $path;
3689         my $format;
3690         if (my $efidisk = $conf->{efidisk0}) {
3691             my $d = PVE::JSONSchema::parse_property_string($efidisk_fmt, $efidisk);
3692             my ($storeid, $volname) = PVE::Storage::parse_volume_id($d->{file}, 1);
3693             $format = $d->{format};
3694             if ($storeid) {
3695                 $path = PVE::Storage::path($storecfg, $d->{file});
3696                 if (!defined($format)) {
3697                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
3698                     $format = qemu_img_format($scfg, $volname);
3699                 }
3700             } else {
3701                 $path = $d->{file};
3702                 die "efidisk format must be specified\n"
3703                     if !defined($format);
3704             }
3705         } else {
3706             warn "no efidisk configured! Using temporary efivars disk.\n";
3707             $path = "/tmp/$vmid-ovmf.fd";
3708             PVE::Tools::file_copy($ovmf_vars, $path, -s $ovmf_vars);
3709             $format = 'raw';
3710         }
3711
3712         push @$cmd, '-drive', "if=pflash,unit=0,format=raw,readonly,file=$ovmf_code";
3713         push @$cmd, '-drive', "if=pflash,unit=1,format=$format,id=drive-efidisk0,file=$path";
3714     }
3715
3716     # load q35 config
3717     if ($q35) {
3718         # we use different pcie-port hardware for qemu >= 4.0 for passthrough
3719         if (qemu_machine_feature_enabled($machine_type, $kvmver, 4, 0)) {
3720             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35-4.0.cfg';
3721         } else {
3722             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35.cfg';
3723         }
3724     }
3725
3726     # add usb controllers
3727     my @usbcontrollers = PVE::QemuServer::USB::get_usb_controllers($conf, $bridges, $arch, $machine_type, $usbdesc->{format}, $MAX_USB_DEVICES);
3728     push @$devices, @usbcontrollers if @usbcontrollers;
3729     my $vga = parse_vga($conf->{vga});
3730
3731     my $qxlnum = vga_conf_has_spice($conf->{vga});
3732     $vga->{type} = 'qxl' if $qxlnum;
3733
3734     if (!$vga->{type}) {
3735         if ($arch eq 'aarch64') {
3736             $vga->{type} = 'virtio';
3737         } elsif (qemu_machine_feature_enabled($machine_type, $kvmver, 2, 9)) {
3738             $vga->{type} = (!$winversion || $winversion >= 6) ? 'std' : 'cirrus';
3739         } else {
3740             $vga->{type} = ($winversion >= 6) ? 'std' : 'cirrus';
3741         }
3742     }
3743
3744     # enable absolute mouse coordinates (needed by vnc)
3745     my $tablet;
3746     if (defined($conf->{tablet})) {
3747         $tablet = $conf->{tablet};
3748     } else {
3749         $tablet = $defaults->{tablet};
3750         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
3751         $tablet = 0 if $vga->{type} =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
3752     }
3753
3754     if ($tablet) {
3755         push @$devices, '-device', print_tabletdevice_full($conf, $arch) if $tablet;
3756         my $kbd = print_keyboarddevice_full($conf, $arch);
3757         push @$devices, '-device', $kbd if defined($kbd);
3758     }
3759
3760     my $kvm_off = 0;
3761     my $gpu_passthrough;
3762
3763     # host pci devices
3764     for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
3765         my $id = "hostpci$i";
3766         my $d = parse_hostpci($conf->{$id});
3767         next if !$d;
3768
3769         if (my $pcie = $d->{pcie}) {
3770             die "q35 machine model is not enabled" if !$q35;
3771             # win7 wants to have the pcie devices directly on the pcie bus
3772             # instead of in the root port
3773             if ($winversion == 7) {
3774                 $pciaddr = print_pcie_addr("${id}bus0");
3775             } else {
3776                 # add more root ports if needed, 4 are present by default
3777                 # by pve-q35 cfgs, rest added here on demand.
3778                 if ($i > 3) {
3779                     push @$devices, '-device', print_pcie_root_port($i);
3780                 }
3781                 $pciaddr = print_pcie_addr($id);
3782             }
3783         } else {
3784             $pciaddr = print_pci_addr($id, $bridges, $arch, $machine_type);
3785         }
3786
3787         my $xvga = '';
3788         if ($d->{'x-vga'}) {
3789             $xvga = ',x-vga=on' if !($conf->{bios} && $conf->{bios} eq 'ovmf');
3790             $kvm_off = 1;
3791             $vga->{type} = 'none' if !defined($conf->{vga});
3792             $gpu_passthrough = 1;
3793         }
3794
3795         my $pcidevices = $d->{pciid};
3796         my $multifunction = 1 if @$pcidevices > 1;
3797
3798         my $sysfspath;
3799         if ($d->{mdev} && scalar(@$pcidevices) == 1) {
3800             my $pci_id = $pcidevices->[0]->{id};
3801             my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $i);
3802             $sysfspath = "/sys/bus/pci/devices/0000:$pci_id/$uuid";
3803         } elsif ($d->{mdev}) {
3804             warn "ignoring mediated device '$id' with multifunction device\n";
3805         }
3806
3807         my $j=0;
3808         foreach my $pcidevice (@$pcidevices) {
3809             my $devicestr = "vfio-pci";
3810
3811             if ($sysfspath) {
3812                 $devicestr .= ",sysfsdev=$sysfspath";
3813             } else {
3814                 $devicestr .= ",host=$pcidevice->{id}";
3815             }
3816
3817             my $mf_addr = $multifunction ? ".$j" : '';
3818             $devicestr .= ",id=${id}${mf_addr}${pciaddr}${mf_addr}";
3819
3820             if ($j == 0) {
3821                 $devicestr .= ',rombar=0' if defined($d->{rombar}) && !$d->{rombar};
3822                 $devicestr .= "$xvga";
3823                 $devicestr .= ",multifunction=on" if $multifunction;
3824                 $devicestr .= ",romfile=/usr/share/kvm/$d->{romfile}" if $d->{romfile};
3825             }
3826
3827             push @$devices, '-device', $devicestr;
3828             $j++;
3829         }
3830     }
3831
3832     # usb devices
3833     my $usb_dev_features = {};
3834     $usb_dev_features->{spice_usb3} = 1 if qemu_machine_feature_enabled($machine_type, $kvmver, 4, 0);
3835
3836     my @usbdevices = PVE::QemuServer::USB::get_usb_devices($conf, $usbdesc->{format}, $MAX_USB_DEVICES, $usb_dev_features);
3837     push @$devices, @usbdevices if @usbdevices;
3838     # serial devices
3839     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3840         if (my $path = $conf->{"serial$i"}) {
3841             if ($path eq 'socket') {
3842                 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
3843                 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server,nowait";
3844                 # On aarch64, serial0 is the UART device. Qemu only allows
3845                 # connecting UART devices via the '-serial' command line, as
3846                 # the device has a fixed slot on the hardware...
3847                 if ($arch eq 'aarch64' && $i == 0) {
3848                     push @$devices, '-serial', "chardev:serial$i";
3849                 } else {
3850                     push @$devices, '-device', "isa-serial,chardev=serial$i";
3851                 }
3852             } else {
3853                 die "no such serial device\n" if ! -c $path;
3854                 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
3855                 push @$devices, '-device', "isa-serial,chardev=serial$i";
3856             }
3857         }
3858     }
3859
3860     # parallel devices
3861     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
3862         if (my $path = $conf->{"parallel$i"}) {
3863             die "no such parallel device\n" if ! -c $path;
3864             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
3865             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
3866             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
3867         }
3868     }
3869
3870     if (my $audio = conf_has_audio($conf)) {
3871
3872         my $audiopciaddr = print_pci_addr("audio0", $bridges, $arch, $machine_type);
3873
3874         my $id = $audio->{dev_id};
3875         if ($audio->{dev} eq 'AC97') {
3876             push @$devices, '-device', "AC97,id=${id}${audiopciaddr}";
3877         } elsif ($audio->{dev} =~ /intel\-hda$/) {
3878             push @$devices, '-device', "$audio->{dev},id=${id}${audiopciaddr}";
3879             push @$devices, '-device', "hda-micro,id=${id}-codec0,bus=${id}.0,cad=0";
3880             push @$devices, '-device', "hda-duplex,id=${id}-codec1,bus=${id}.0,cad=1";
3881         } else {
3882             die "unkown audio device '$audio->{dev}', implement me!";
3883         }
3884
3885         push @$devices, '-audiodev', "$audio->{backend},id=$audio->{backend_id}";
3886     }
3887
3888     my $sockets = 1;
3889     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3890     $sockets = $conf->{sockets} if  $conf->{sockets};
3891
3892     my $cores = $conf->{cores} || 1;
3893
3894     my $maxcpus = $sockets * $cores;
3895
3896     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
3897
3898     my $allowed_vcpus = $cpuinfo->{cpus};
3899
3900     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n"
3901         if ($allowed_vcpus < $maxcpus);
3902
3903     if($hotplug_features->{cpu} && qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 7)) {
3904
3905         push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3906         for (my $i = 2; $i <= $vcpus; $i++)  {
3907             my $cpustr = print_cpu_device($conf,$i);
3908             push @$cmd, '-device', $cpustr;
3909         }
3910
3911     } else {
3912
3913         push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3914     }
3915     push @$cmd, '-nodefaults';
3916
3917     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
3918
3919     my $bootindex_hash = {};
3920     my $i = 1;
3921     foreach my $o (split(//, $bootorder)) {
3922         $bootindex_hash->{$o} = $i*100;
3923         $i++;
3924     }
3925
3926     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
3927
3928     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
3929
3930     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
3931
3932     if ($vga->{type} && $vga->{type} !~ m/^serial\d+$/ && $vga->{type} ne 'none'){
3933         push @$devices, '-device', print_vga_device($conf, $vga, $arch, $machine_type, undef, $qxlnum, $bridges);
3934         my $socket = vnc_socket($vmid);
3935         push @$cmd,  '-vnc', "unix:$socket,password";
3936     } else {
3937         push @$cmd, '-vga', 'none' if $vga->{type} eq 'none';
3938         push @$cmd, '-nographic';
3939     }
3940
3941     # time drift fix
3942     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
3943
3944     my $useLocaltime = $conf->{localtime};
3945
3946     if ($winversion >= 5) { # windows
3947         $useLocaltime = 1 if !defined($conf->{localtime});
3948
3949         # use time drift fix when acpi is enabled
3950         if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
3951             $tdf = 1 if !defined($conf->{tdf});
3952         }
3953     }
3954
3955     if ($winversion >= 6) {
3956         push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3957         push @$cmd, '-no-hpet';
3958     }
3959
3960     push @$rtcFlags, 'driftfix=slew' if $tdf;
3961
3962     if (!$kvm) {
3963         push @$machineFlags, 'accel=tcg';
3964     }
3965
3966     if ($machine_type) {
3967         push @$machineFlags, "type=${machine_type}";
3968     }
3969
3970     if (($conf->{startdate}) && ($conf->{startdate} ne 'now')) {
3971         push @$rtcFlags, "base=$conf->{startdate}";
3972     } elsif ($useLocaltime) {
3973         push @$rtcFlags, 'base=localtime';
3974     }
3975
3976     push @$cmd, get_cpu_options($conf, $arch, $kvm, $machine_type, $kvm_off, $kvmver, $winversion, $gpu_passthrough);
3977
3978     PVE::QemuServer::Memory::config($conf, $vmid, $sockets, $cores, $defaults, $hotplug_features, $cmd);
3979
3980     push @$cmd, '-S' if $conf->{freeze};
3981
3982     push @$cmd, '-k', $conf->{keyboard} if defined($conf->{keyboard});
3983
3984     if (parse_guest_agent($conf)->{enabled}) {
3985         my $qgasocket = qmp_socket($vmid, 1);
3986         my $pciaddr = print_pci_addr("qga0", $bridges, $arch, $machine_type);
3987         push @$devices, '-chardev', "socket,path=$qgasocket,server,nowait,id=qga0";
3988         push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3989         push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3990     }
3991
3992     my $spice_port;
3993
3994     if ($qxlnum) {
3995         if ($qxlnum > 1) {
3996             if ($winversion){
3997                 for(my $i = 1; $i < $qxlnum; $i++){
3998                     push @$devices, '-device', print_vga_device($conf, $vga, $arch, $machine_type, $i, $qxlnum, $bridges);
3999                 }
4000             } else {
4001                 # assume other OS works like Linux
4002                 my ($ram, $vram) = ("134217728", "67108864");
4003                 if ($vga->{memory}) {
4004                     $ram = PVE::Tools::convert_size($qxlnum*4*$vga->{memory}, 'mb' => 'b');
4005                     $vram = PVE::Tools::convert_size($qxlnum*2*$vga->{memory}, 'mb' => 'b');
4006                 }
4007                 push @$cmd, '-global', "qxl-vga.ram_size=$ram";
4008                 push @$cmd, '-global', "qxl-vga.vram_size=$vram";
4009             }
4010         }
4011
4012         my $pciaddr = print_pci_addr("spice", $bridges, $arch, $machine_type);
4013
4014         my $nodename = PVE::INotify::nodename();
4015         my $pfamily = PVE::Tools::get_host_address_family($nodename);
4016         my @nodeaddrs = PVE::Tools::getaddrinfo_all('localhost', family => $pfamily);
4017         die "failed to get an ip address of type $pfamily for 'localhost'\n" if !@nodeaddrs;
4018
4019         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
4020         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
4021         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
4022
4023         my $localhost = PVE::Network::addr_to_ip($nodeaddrs[0]->{addr});
4024         $spice_port = PVE::Tools::next_spice_port($pfamily, $localhost);
4025
4026         my $spice_enhancement = PVE::JSONSchema::parse_property_string($spice_enhancements_fmt, $conf->{spice_enhancements} // '');
4027         if ($spice_enhancement->{foldersharing}) {
4028             push @$devices, '-chardev', "spiceport,id=foldershare,name=org.spice-space.webdav.0";
4029             push @$devices, '-device', "virtserialport,chardev=foldershare,name=org.spice-space.webdav.0";
4030         }
4031
4032         my $spice_opts = "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";
4033         $spice_opts .= ",streaming-video=$spice_enhancement->{videostreaming}" if $spice_enhancement->{videostreaming};
4034         push @$devices, '-spice', "$spice_opts";
4035     }
4036
4037     # enable balloon by default, unless explicitly disabled
4038     if (!defined($conf->{balloon}) || $conf->{balloon}) {
4039         $pciaddr = print_pci_addr("balloon0", $bridges, $arch, $machine_type);
4040         push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
4041     }
4042
4043     if ($conf->{watchdog}) {
4044         my $wdopts = parse_watchdog($conf->{watchdog});
4045         $pciaddr = print_pci_addr("watchdog", $bridges, $arch, $machine_type);
4046         my $watchdog = $wdopts->{model} || 'i6300esb';
4047         push @$devices, '-device', "$watchdog$pciaddr";
4048         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
4049     }
4050
4051     my $vollist = [];
4052     my $scsicontroller = {};
4053     my $ahcicontroller = {};
4054     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
4055
4056     # Add iscsi initiator name if available
4057     if (my $initiator = get_initiator_name()) {
4058         push @$devices, '-iscsi', "initiator-name=$initiator";
4059     }
4060
4061     foreach_drive($conf, sub {
4062         my ($ds, $drive) = @_;
4063
4064         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
4065             push @$vollist, $drive->{file};
4066         }
4067
4068         # ignore efidisk here, already added in bios/fw handling code above
4069         return if $drive->{interface} eq 'efidisk';
4070
4071         $use_virtio = 1 if $ds =~ m/^virtio/;
4072
4073         if (drive_is_cdrom ($drive)) {
4074             if ($bootindex_hash->{d}) {
4075                 $drive->{bootindex} = $bootindex_hash->{d};
4076                 $bootindex_hash->{d} += 1;
4077             }
4078         } else {
4079             if ($bootindex_hash->{c}) {
4080                 $drive->{bootindex} = $bootindex_hash->{c} if $conf->{bootdisk} && ($conf->{bootdisk} eq $ds);
4081                 $bootindex_hash->{c} += 1;
4082             }
4083         }
4084
4085         if($drive->{interface} eq 'virtio'){
4086            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
4087         }
4088
4089         if ($drive->{interface} eq 'scsi') {
4090
4091             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
4092
4093             $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges, $arch, $machine_type);
4094             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
4095
4096             my $iothread = '';
4097             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
4098                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
4099                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
4100             } elsif ($drive->{iothread}) {
4101                 warn "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n";
4102             }
4103
4104             my $queues = '';
4105             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
4106                 $queues = ",num_queues=$drive->{queues}";
4107             }
4108
4109             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues" if !$scsicontroller->{$controller};
4110             $scsicontroller->{$controller}=1;
4111         }
4112
4113         if ($drive->{interface} eq 'sata') {
4114            my $controller = int($drive->{index} / $MAX_SATA_DISKS);
4115            $pciaddr = print_pci_addr("ahci$controller", $bridges, $arch, $machine_type);
4116            push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr" if !$ahcicontroller->{$controller};
4117            $ahcicontroller->{$controller}=1;
4118         }
4119
4120         my $drive_cmd = print_drive_full($storecfg, $vmid, $drive);
4121         push @$devices, '-drive',$drive_cmd;
4122         push @$devices, '-device', print_drivedevice_full($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type);
4123     });
4124
4125     for (my $i = 0; $i < $MAX_NETS; $i++) {
4126          next if !$conf->{"net$i"};
4127          my $d = parse_net($conf->{"net$i"});
4128          next if !$d;
4129
4130          $use_virtio = 1 if $d->{model} eq 'virtio';
4131
4132          if ($bootindex_hash->{n}) {
4133             $d->{bootindex} = $bootindex_hash->{n};
4134             $bootindex_hash->{n} += 1;
4135          }
4136
4137          my $netdevfull = print_netdev_full($vmid, $conf, $arch, $d, "net$i");
4138          push @$devices, '-netdev', $netdevfull;
4139
4140          my $netdevicefull = print_netdevice_full($vmid, $conf, $d, "net$i", $bridges, $use_old_bios_files, $arch, $machine_type);
4141          push @$devices, '-device', $netdevicefull;
4142     }
4143
4144     if ($conf->{ivshmem}) {
4145         my $ivshmem = PVE::JSONSchema::parse_property_string($ivshmem_fmt, $conf->{ivshmem});
4146
4147         my $bus;
4148         if ($q35) {
4149             $bus = print_pcie_addr("ivshmem");
4150         } else {
4151             $bus = print_pci_addr("ivshmem", $bridges, $arch, $machine_type);
4152         }
4153
4154         my $ivshmem_name = $ivshmem->{name} // $vmid;
4155         my $path = '/dev/shm/pve-shm-' . $ivshmem_name;
4156
4157         push @$devices, '-device', "ivshmem-plain,memdev=ivshmem$bus,";
4158         push @$devices, '-object', "memory-backend-file,id=ivshmem,share=on,mem-path=$path,size=$ivshmem->{size}M";
4159     }
4160
4161     if (!$q35) {
4162         # add pci bridges
4163         if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
4164            $bridges->{1} = 1;
4165            $bridges->{2} = 1;
4166         }
4167
4168         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
4169
4170         for my $k (sort {$b cmp $a} keys %$bridges) {
4171             $pciaddr = print_pci_addr("pci.$k", undef, $arch, $machine_type);
4172             unshift @$devices, '-device', "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr" if $k > 0;
4173         }
4174     }
4175
4176     push @$cmd, @$devices;
4177     push @$cmd, '-rtc', join(',', @$rtcFlags)
4178         if scalar(@$rtcFlags);
4179     push @$cmd, '-machine', join(',', @$machineFlags)
4180         if scalar(@$machineFlags);
4181     push @$cmd, '-global', join(',', @$globalFlags)
4182         if scalar(@$globalFlags);
4183
4184     if (my $vmstate = $conf->{vmstate}) {
4185         my $statepath = PVE::Storage::path($storecfg, $vmstate);
4186         PVE::Storage::activate_volumes($storecfg, [$vmstate]);
4187         push @$cmd, '-loadstate', $statepath;
4188     }
4189
4190     # add custom args
4191     if ($conf->{args}) {
4192         my $aa = PVE::Tools::split_args($conf->{args});
4193         push @$cmd, @$aa;
4194     }
4195
4196     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
4197 }
4198
4199 sub vnc_socket {
4200     my ($vmid) = @_;
4201     return "${var_run_tmpdir}/$vmid.vnc";
4202 }
4203
4204 sub spice_port {
4205     my ($vmid) = @_;
4206
4207     my $res = vm_mon_cmd($vmid, 'query-spice');
4208
4209     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
4210 }
4211
4212 sub qmp_socket {
4213     my ($vmid, $qga, $name) = @_;
4214     my $sockettype = $qga ? 'qga' : 'qmp';
4215     my $ext = $name ? '-'.$name : '';
4216     return "${var_run_tmpdir}/$vmid$ext.$sockettype";
4217 }
4218
4219 sub pidfile_name {
4220     my ($vmid) = @_;
4221     return "${var_run_tmpdir}/$vmid.pid";
4222 }
4223
4224 sub vm_devices_list {
4225     my ($vmid) = @_;
4226
4227     my $res = vm_mon_cmd($vmid, 'query-pci');
4228     my $devices_to_check = [];
4229     my $devices = {};
4230     foreach my $pcibus (@$res) {
4231         push @$devices_to_check, @{$pcibus->{devices}},
4232     }
4233
4234     while (@$devices_to_check) {
4235         my $to_check = [];
4236         for my $d (@$devices_to_check) {
4237             $devices->{$d->{'qdev_id'}} = 1 if $d->{'qdev_id'};
4238             next if !$d->{'pci_bridge'};
4239
4240             $devices->{$d->{'qdev_id'}} += scalar(@{$d->{'pci_bridge'}->{devices}});
4241             push @$to_check, @{$d->{'pci_bridge'}->{devices}};
4242         }
4243         $devices_to_check = $to_check;
4244     }
4245
4246     my $resblock = vm_mon_cmd($vmid, 'query-block');
4247     foreach my $block (@$resblock) {
4248         if($block->{device} =~ m/^drive-(\S+)/){
4249                 $devices->{$1} = 1;
4250         }
4251     }
4252
4253     my $resmice = vm_mon_cmd($vmid, 'query-mice');
4254     foreach my $mice (@$resmice) {
4255         if ($mice->{name} eq 'QEMU HID Tablet') {
4256             $devices->{tablet} = 1;
4257             last;
4258         }
4259     }
4260
4261     # for usb devices there is no query-usb
4262     # but we can iterate over the entries in
4263     # qom-list path=/machine/peripheral
4264     my $resperipheral = vm_mon_cmd($vmid, 'qom-list', path => '/machine/peripheral');
4265     foreach my $per (@$resperipheral) {
4266         if ($per->{name} =~ m/^usb\d+$/) {
4267             $devices->{$per->{name}} = 1;
4268         }
4269     }
4270
4271     return $devices;
4272 }
4273
4274 sub vm_deviceplug {
4275     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4276
4277     my $q35 = machine_type_is_q35($conf);
4278
4279     my $devices_list = vm_devices_list($vmid);
4280     return 1 if defined($devices_list->{$deviceid});
4281
4282     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid, $arch, $machine_type); # add PCI bridge if we need it for the device
4283
4284     if ($deviceid eq 'tablet') {
4285
4286         qemu_deviceadd($vmid, print_tabletdevice_full($conf, $arch));
4287
4288     } elsif ($deviceid eq 'keyboard') {
4289
4290         qemu_deviceadd($vmid, print_keyboarddevice_full($conf, $arch));
4291
4292     } elsif ($deviceid =~ m/^usb(\d+)$/) {
4293
4294         die "usb hotplug currently not reliable\n";
4295         # since we can't reliably hot unplug all added usb devices
4296         # and usb passthrough disables live migration
4297         # we disable usb hotplugging for now
4298         qemu_deviceadd($vmid, PVE::QemuServer::USB::print_usbdevice_full($conf, $deviceid, $device));
4299
4300     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4301
4302         qemu_iothread_add($vmid, $deviceid, $device);
4303
4304         qemu_driveadd($storecfg, $vmid, $device);
4305         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, $arch, $machine_type);
4306
4307         qemu_deviceadd($vmid, $devicefull);
4308         eval { qemu_deviceaddverify($vmid, $deviceid); };
4309         if (my $err = $@) {
4310             eval { qemu_drivedel($vmid, $deviceid); };
4311             warn $@ if $@;
4312             die $err;
4313         }
4314
4315     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4316
4317
4318         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
4319         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
4320         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
4321
4322         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
4323
4324         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
4325             qemu_iothread_add($vmid, $deviceid, $device);
4326             $devicefull .= ",iothread=iothread-$deviceid";
4327         }
4328
4329         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
4330             $devicefull .= ",num_queues=$device->{queues}";
4331         }
4332
4333         qemu_deviceadd($vmid, $devicefull);
4334         qemu_deviceaddverify($vmid, $deviceid);
4335
4336     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4337
4338         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device, $arch, $machine_type);
4339         qemu_driveadd($storecfg, $vmid, $device);
4340
4341         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, $arch, $machine_type);
4342         eval { qemu_deviceadd($vmid, $devicefull); };
4343         if (my $err = $@) {
4344             eval { qemu_drivedel($vmid, $deviceid); };
4345             warn $@ if $@;
4346             die $err;
4347         }
4348
4349     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4350
4351         return undef if !qemu_netdevadd($vmid, $conf, $arch, $device, $deviceid);
4352
4353         my $machine_type = PVE::QemuServer::qemu_machine_pxe($vmid, $conf);
4354         my $use_old_bios_files = undef;
4355         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
4356
4357         my $netdevicefull = print_netdevice_full($vmid, $conf, $device, $deviceid, undef, $use_old_bios_files, $arch, $machine_type);
4358         qemu_deviceadd($vmid, $netdevicefull);
4359         eval {
4360             qemu_deviceaddverify($vmid, $deviceid);
4361             qemu_set_link_status($vmid, $deviceid, !$device->{link_down});
4362         };
4363         if (my $err = $@) {
4364             eval { qemu_netdevdel($vmid, $deviceid); };
4365             warn $@ if $@;
4366             die $err;
4367         }
4368
4369     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
4370
4371         my $bridgeid = $2;
4372         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
4373         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
4374
4375         qemu_deviceadd($vmid, $devicefull);
4376         qemu_deviceaddverify($vmid, $deviceid);
4377
4378     } else {
4379         die "can't hotplug device '$deviceid'\n";
4380     }
4381
4382     return 1;
4383 }
4384
4385 # fixme: this should raise exceptions on error!
4386 sub vm_deviceunplug {
4387     my ($vmid, $conf, $deviceid) = @_;
4388
4389     my $devices_list = vm_devices_list($vmid);
4390     return 1 if !defined($devices_list->{$deviceid});
4391
4392     die "can't unplug bootdisk" if $conf->{bootdisk} && $conf->{bootdisk} eq $deviceid;
4393
4394     if ($deviceid eq 'tablet' || $deviceid eq 'keyboard') {
4395
4396         qemu_devicedel($vmid, $deviceid);
4397
4398     } elsif ($deviceid =~ m/^usb\d+$/) {
4399
4400         die "usb hotplug currently not reliable\n";
4401         # when unplugging usb devices this way,
4402         # there may be remaining usb controllers/hubs
4403         # so we disable it for now
4404         qemu_devicedel($vmid, $deviceid);
4405         qemu_devicedelverify($vmid, $deviceid);
4406
4407     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4408
4409         qemu_devicedel($vmid, $deviceid);
4410         qemu_devicedelverify($vmid, $deviceid);
4411         qemu_drivedel($vmid, $deviceid);
4412         qemu_iothread_del($conf, $vmid, $deviceid);
4413
4414     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4415
4416         qemu_devicedel($vmid, $deviceid);
4417         qemu_devicedelverify($vmid, $deviceid);
4418         qemu_iothread_del($conf, $vmid, $deviceid);
4419
4420     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4421
4422         qemu_devicedel($vmid, $deviceid);
4423         qemu_drivedel($vmid, $deviceid);
4424         qemu_deletescsihw($conf, $vmid, $deviceid);
4425
4426     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4427
4428         qemu_devicedel($vmid, $deviceid);
4429         qemu_devicedelverify($vmid, $deviceid);
4430         qemu_netdevdel($vmid, $deviceid);
4431
4432     } else {
4433         die "can't unplug device '$deviceid'\n";
4434     }
4435
4436     return 1;
4437 }
4438
4439 sub qemu_deviceadd {
4440     my ($vmid, $devicefull) = @_;
4441
4442     $devicefull = "driver=".$devicefull;
4443     my %options =  split(/[=,]/, $devicefull);
4444
4445     vm_mon_cmd($vmid, "device_add" , %options);
4446 }
4447
4448 sub qemu_devicedel {
4449     my ($vmid, $deviceid) = @_;
4450
4451     my $ret = vm_mon_cmd($vmid, "device_del", id => $deviceid);
4452 }
4453
4454 sub qemu_iothread_add {
4455     my($vmid, $deviceid, $device) = @_;
4456
4457     if ($device->{iothread}) {
4458         my $iothreads = vm_iothreads_list($vmid);
4459         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
4460     }
4461 }
4462
4463 sub qemu_iothread_del {
4464     my($conf, $vmid, $deviceid) = @_;
4465
4466     my $confid = $deviceid;
4467     if ($deviceid =~ m/^(?:virtioscsi|scsihw)(\d+)$/) {
4468         $confid = 'scsi' . $1;
4469     }
4470     my $device = parse_drive($confid, $conf->{$confid});
4471     if ($device->{iothread}) {
4472         my $iothreads = vm_iothreads_list($vmid);
4473         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
4474     }
4475 }
4476
4477 sub qemu_objectadd {
4478     my($vmid, $objectid, $qomtype) = @_;
4479
4480     vm_mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
4481
4482     return 1;
4483 }
4484
4485 sub qemu_objectdel {
4486     my($vmid, $objectid) = @_;
4487
4488     vm_mon_cmd($vmid, "object-del", id => $objectid);
4489
4490     return 1;
4491 }
4492
4493 sub qemu_driveadd {
4494     my ($storecfg, $vmid, $device) = @_;
4495
4496     my $drive = print_drive_full($storecfg, $vmid, $device);
4497     $drive =~ s/\\/\\\\/g;
4498     my $ret = vm_human_monitor_command($vmid, "drive_add auto \"$drive\"");
4499
4500     # If the command succeeds qemu prints: "OK"
4501     return 1 if $ret =~ m/OK/s;
4502
4503     die "adding drive failed: $ret\n";
4504 }
4505
4506 sub qemu_drivedel {
4507     my($vmid, $deviceid) = @_;
4508
4509     my $ret = vm_human_monitor_command($vmid, "drive_del drive-$deviceid");
4510     $ret =~ s/^\s+//;
4511
4512     return 1 if $ret eq "";
4513
4514     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
4515     return 1 if $ret =~ m/Device \'.*?\' not found/s;
4516
4517     die "deleting drive $deviceid failed : $ret\n";
4518 }
4519
4520 sub qemu_deviceaddverify {
4521     my ($vmid, $deviceid) = @_;
4522
4523     for (my $i = 0; $i <= 5; $i++) {
4524          my $devices_list = vm_devices_list($vmid);
4525          return 1 if defined($devices_list->{$deviceid});
4526          sleep 1;
4527     }
4528
4529     die "error on hotplug device '$deviceid'\n";
4530 }
4531
4532
4533 sub qemu_devicedelverify {
4534     my ($vmid, $deviceid) = @_;
4535
4536     # need to verify that the device is correctly removed as device_del
4537     # is async and empty return is not reliable
4538
4539     for (my $i = 0; $i <= 5; $i++) {
4540          my $devices_list = vm_devices_list($vmid);
4541          return 1 if !defined($devices_list->{$deviceid});
4542          sleep 1;
4543     }
4544
4545     die "error on hot-unplugging device '$deviceid'\n";
4546 }
4547
4548 sub qemu_findorcreatescsihw {
4549     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4550
4551     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4552
4553     my $scsihwid="$controller_prefix$controller";
4554     my $devices_list = vm_devices_list($vmid);
4555
4556     if(!defined($devices_list->{$scsihwid})) {
4557         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device, $arch, $machine_type);
4558     }
4559
4560     return 1;
4561 }
4562
4563 sub qemu_deletescsihw {
4564     my ($conf, $vmid, $opt) = @_;
4565
4566     my $device = parse_drive($opt, $conf->{$opt});
4567
4568     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
4569         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
4570         return 1;
4571     }
4572
4573     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4574
4575     my $devices_list = vm_devices_list($vmid);
4576     foreach my $opt (keys %{$devices_list}) {
4577         if (PVE::QemuServer::is_valid_drivename($opt)) {
4578             my $drive = PVE::QemuServer::parse_drive($opt, $conf->{$opt});
4579             if($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
4580                 return 1;
4581             }
4582         }
4583     }
4584
4585     my $scsihwid="scsihw$controller";
4586
4587     vm_deviceunplug($vmid, $conf, $scsihwid);
4588
4589     return 1;
4590 }
4591
4592 sub qemu_add_pci_bridge {
4593     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4594
4595     my $bridges = {};
4596
4597     my $bridgeid;
4598
4599     print_pci_addr($device, $bridges, $arch, $machine_type);
4600
4601     while (my ($k, $v) = each %$bridges) {
4602         $bridgeid = $k;
4603     }
4604     return 1 if !defined($bridgeid) || $bridgeid < 1;
4605
4606     my $bridge = "pci.$bridgeid";
4607     my $devices_list = vm_devices_list($vmid);
4608
4609     if (!defined($devices_list->{$bridge})) {
4610         vm_deviceplug($storecfg, $conf, $vmid, $bridge, $arch, $machine_type);
4611     }
4612
4613     return 1;
4614 }
4615
4616 sub qemu_set_link_status {
4617     my ($vmid, $device, $up) = @_;
4618
4619     vm_mon_cmd($vmid, "set_link", name => $device,
4620                up => $up ? JSON::true : JSON::false);
4621 }
4622
4623 sub qemu_netdevadd {
4624     my ($vmid, $conf, $arch, $device, $deviceid) = @_;
4625
4626     my $netdev = print_netdev_full($vmid, $conf, $arch, $device, $deviceid, 1);
4627     my %options =  split(/[=,]/, $netdev);
4628
4629     vm_mon_cmd($vmid, "netdev_add",  %options);
4630     return 1;
4631 }
4632
4633 sub qemu_netdevdel {
4634     my ($vmid, $deviceid) = @_;
4635
4636     vm_mon_cmd($vmid, "netdev_del", id => $deviceid);
4637 }
4638
4639 sub qemu_usb_hotplug {
4640     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4641
4642     return if !$device;
4643
4644     # remove the old one first
4645     vm_deviceunplug($vmid, $conf, $deviceid);
4646
4647     # check if xhci controller is necessary and available
4648     if ($device->{usb3}) {
4649
4650         my $devicelist = vm_devices_list($vmid);
4651
4652         if (!$devicelist->{xhci}) {
4653             my $pciaddr = print_pci_addr("xhci", undef, $arch, $machine_type);
4654             qemu_deviceadd($vmid, "nec-usb-xhci,id=xhci$pciaddr");
4655         }
4656     }
4657     my $d = parse_usb_device($device->{host});
4658     $d->{usb3} = $device->{usb3};
4659
4660     # add the new one
4661     vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $d, $arch, $machine_type);
4662 }
4663
4664 sub qemu_cpu_hotplug {
4665     my ($vmid, $conf, $vcpus) = @_;
4666
4667     my $machine_type = PVE::QemuServer::get_current_qemu_machine($vmid);
4668
4669     my $sockets = 1;
4670     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
4671     $sockets = $conf->{sockets} if  $conf->{sockets};
4672     my $cores = $conf->{cores} || 1;
4673     my $maxcpus = $sockets * $cores;
4674
4675     $vcpus = $maxcpus if !$vcpus;
4676
4677     die "you can't add more vcpus than maxcpus\n"
4678         if $vcpus > $maxcpus;
4679
4680     my $currentvcpus = $conf->{vcpus} || $maxcpus;
4681
4682     if ($vcpus < $currentvcpus) {
4683
4684         if (qemu_machine_feature_enabled ($machine_type, undef, 2, 7)) {
4685
4686             for (my $i = $currentvcpus; $i > $vcpus; $i--) {
4687                 qemu_devicedel($vmid, "cpu$i");
4688                 my $retry = 0;
4689                 my $currentrunningvcpus = undef;
4690                 while (1) {
4691                     $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
4692                     last if scalar(@{$currentrunningvcpus}) == $i-1;
4693                     raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
4694                     $retry++;
4695                     sleep 1;
4696                 }
4697                 #update conf after each succesfull cpu unplug
4698                 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4699                 PVE::QemuConfig->write_config($vmid, $conf);
4700             }
4701         } else {
4702             die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
4703         }
4704
4705         return;
4706     }
4707
4708     my $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
4709     die "vcpus in running vm does not match its configuration\n"
4710         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
4711
4712     if (qemu_machine_feature_enabled ($machine_type, undef, 2, 7)) {
4713
4714         for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
4715             my $cpustr = print_cpu_device($conf, $i);
4716             qemu_deviceadd($vmid, $cpustr);
4717
4718             my $retry = 0;
4719             my $currentrunningvcpus = undef;
4720             while (1) {
4721                 $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
4722                 last if scalar(@{$currentrunningvcpus}) == $i;
4723                 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
4724                 sleep 1;
4725                 $retry++;
4726             }
4727             #update conf after each succesfull cpu hotplug
4728             $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4729             PVE::QemuConfig->write_config($vmid, $conf);
4730         }
4731     } else {
4732
4733         for (my $i = $currentvcpus; $i < $vcpus; $i++) {
4734             vm_mon_cmd($vmid, "cpu-add", id => int($i));
4735         }
4736     }
4737 }
4738
4739 sub qemu_block_set_io_throttle {
4740     my ($vmid, $deviceid,
4741         $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
4742         $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max,
4743         $bps_max_length, $bps_rd_max_length, $bps_wr_max_length,
4744         $iops_max_length, $iops_rd_max_length, $iops_wr_max_length) = @_;
4745
4746     return if !check_running($vmid) ;
4747
4748     vm_mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
4749         bps => int($bps),
4750         bps_rd => int($bps_rd),
4751         bps_wr => int($bps_wr),
4752         iops => int($iops),
4753         iops_rd => int($iops_rd),
4754         iops_wr => int($iops_wr),
4755         bps_max => int($bps_max),
4756         bps_rd_max => int($bps_rd_max),
4757         bps_wr_max => int($bps_wr_max),
4758         iops_max => int($iops_max),
4759         iops_rd_max => int($iops_rd_max),
4760         iops_wr_max => int($iops_wr_max),
4761         bps_max_length => int($bps_max_length),
4762         bps_rd_max_length => int($bps_rd_max_length),
4763         bps_wr_max_length => int($bps_wr_max_length),
4764         iops_max_length => int($iops_max_length),
4765         iops_rd_max_length => int($iops_rd_max_length),
4766         iops_wr_max_length => int($iops_wr_max_length),
4767     );
4768
4769 }
4770
4771 # old code, only used to shutdown old VM after update
4772 sub __read_avail {
4773     my ($fh, $timeout) = @_;
4774
4775     my $sel = new IO::Select;
4776     $sel->add($fh);
4777
4778     my $res = '';
4779     my $buf;
4780
4781     my @ready;
4782     while (scalar (@ready = $sel->can_read($timeout))) {
4783         my $count;
4784         if ($count = $fh->sysread($buf, 8192)) {
4785             if ($buf =~ /^(.*)\(qemu\) $/s) {
4786                 $res .= $1;
4787                 last;
4788             } else {
4789                 $res .= $buf;
4790             }
4791         } else {
4792             if (!defined($count)) {
4793                 die "$!\n";
4794             }
4795             last;
4796         }
4797     }
4798
4799     die "monitor read timeout\n" if !scalar(@ready);
4800
4801     return $res;
4802 }
4803
4804 sub qemu_block_resize {
4805     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4806
4807     my $running = check_running($vmid);
4808
4809     $size = 0 if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4810
4811     return if !$running;
4812
4813     vm_mon_cmd($vmid, "block_resize", device => $deviceid, size => int($size));
4814
4815 }
4816
4817 sub qemu_volume_snapshot {
4818     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4819
4820     my $running = check_running($vmid);
4821
4822     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4823         vm_mon_cmd($vmid, 'blockdev-snapshot-internal-sync', device => $deviceid, name => $snap);
4824     } else {
4825         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4826     }
4827 }
4828
4829 sub qemu_volume_snapshot_delete {
4830     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4831
4832     my $running = check_running($vmid);
4833
4834     if($running) {
4835
4836         $running = undef;
4837         my $conf = PVE::QemuConfig->load_config($vmid);
4838         foreach_drive($conf, sub {
4839             my ($ds, $drive) = @_;
4840             $running = 1 if $drive->{file} eq $volid;
4841         });
4842     }
4843
4844     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4845         vm_mon_cmd($vmid, 'blockdev-snapshot-delete-internal-sync', device => $deviceid, name => $snap);
4846     } else {
4847         PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4848     }
4849 }
4850
4851 sub set_migration_caps {
4852     my ($vmid) = @_;
4853
4854     my $cap_ref = [];
4855
4856     my $enabled_cap = {
4857         "auto-converge" => 1,
4858         "xbzrle" => 1,
4859         "x-rdma-pin-all" => 0,
4860         "zero-blocks" => 0,
4861         "compress" => 0
4862     };
4863
4864     my $supported_capabilities = vm_mon_cmd_nocheck($vmid, "query-migrate-capabilities");
4865
4866     for my $supported_capability (@$supported_capabilities) {
4867         push @$cap_ref, {
4868             capability => $supported_capability->{capability},
4869             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4870         };
4871     }
4872
4873     vm_mon_cmd_nocheck($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4874 }
4875
4876 my $fast_plug_option = {
4877     'lock' => 1,
4878     'name' => 1,
4879     'onboot' => 1,
4880     'shares' => 1,
4881     'startup' => 1,
4882     'description' => 1,
4883     'protection' => 1,
4884     'vmstatestorage' => 1,
4885     'hookscript' => 1,
4886 };
4887
4888 # hotplug changes in [PENDING]
4889 # $selection hash can be used to only apply specified options, for
4890 # example: { cores => 1 } (only apply changed 'cores')
4891 # $errors ref is used to return error messages
4892 sub vmconfig_hotplug_pending {
4893     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4894
4895     my $defaults = load_defaults();
4896     my ($arch, $machine_type) = get_basic_machine_info($conf, undef);
4897
4898     # commit values which do not have any impact on running VM first
4899     # Note: those option cannot raise errors, we we do not care about
4900     # $selection and always apply them.
4901
4902     my $add_error = sub {
4903         my ($opt, $msg) = @_;
4904         $errors->{$opt} = "hotplug problem - $msg";
4905     };
4906
4907     my $changes = 0;
4908     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4909         if ($fast_plug_option->{$opt}) {
4910             $conf->{$opt} = $conf->{pending}->{$opt};
4911             delete $conf->{pending}->{$opt};
4912             $changes = 1;
4913         }
4914     }
4915
4916     if ($changes) {
4917         PVE::QemuConfig->write_config($vmid, $conf);
4918         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4919     }
4920
4921     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4922
4923     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4924     while (my ($opt, $force) = each %$pending_delete_hash) {
4925         next if $selection && !$selection->{$opt};
4926         eval {
4927             if ($opt eq 'hotplug') {
4928                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
4929             } elsif ($opt eq 'tablet') {
4930                 die "skip\n" if !$hotplug_features->{usb};
4931                 if ($defaults->{tablet}) {
4932                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4933                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4934                         if $arch eq 'aarch64';
4935                 } else {
4936                     vm_deviceunplug($vmid, $conf, 'tablet');
4937                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4938                 }
4939             } elsif ($opt =~ m/^usb\d+/) {
4940                 die "skip\n";
4941                 # since we cannot reliably hot unplug usb devices
4942                 # we are disabling it
4943                 die "skip\n" if !$hotplug_features->{usb} || $conf->{$opt} =~ m/spice/i;
4944                 vm_deviceunplug($vmid, $conf, $opt);
4945             } elsif ($opt eq 'vcpus') {
4946                 die "skip\n" if !$hotplug_features->{cpu};
4947                 qemu_cpu_hotplug($vmid, $conf, undef);
4948             } elsif ($opt eq 'balloon') {
4949                 # enable balloon device is not hotpluggable
4950                 die "skip\n" if defined($conf->{balloon}) && $conf->{balloon} == 0;
4951                 # here we reset the ballooning value to memory
4952                 my $balloon = $conf->{memory} || $defaults->{memory};
4953                 vm_mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4954             } elsif ($fast_plug_option->{$opt}) {
4955                 # do nothing
4956             } elsif ($opt =~ m/^net(\d+)$/) {
4957                 die "skip\n" if !$hotplug_features->{network};
4958                 vm_deviceunplug($vmid, $conf, $opt);
4959             } elsif (is_valid_drivename($opt)) {
4960                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4961                 vm_deviceunplug($vmid, $conf, $opt);
4962                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4963             } elsif ($opt =~ m/^memory$/) {
4964                 die "skip\n" if !$hotplug_features->{memory};
4965                 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
4966             } elsif ($opt eq 'cpuunits') {
4967                 cgroups_write("cpu", $vmid, "cpu.shares", $defaults->{cpuunits});
4968             } elsif ($opt eq 'cpulimit') {
4969                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", -1);
4970             } else {
4971                 die "skip\n";
4972             }
4973         };
4974         if (my $err = $@) {
4975             &$add_error($opt, $err) if $err ne "skip\n";
4976         } else {
4977             # save new config if hotplug was successful
4978             delete $conf->{$opt};
4979             vmconfig_undelete_pending_option($conf, $opt);
4980             PVE::QemuConfig->write_config($vmid, $conf);
4981             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4982         }
4983     }
4984
4985     my $apply_pending_cloudinit;
4986     $apply_pending_cloudinit = sub {
4987         my ($key, $value) = @_;
4988         $apply_pending_cloudinit = sub {}; # once is enough
4989
4990         my @cloudinit_opts = keys %$confdesc_cloudinit;
4991         foreach my $opt (keys %{$conf->{pending}}) {
4992             next if !grep { $_ eq $opt } @cloudinit_opts;
4993             $conf->{$opt} = delete $conf->{pending}->{$opt};
4994         }
4995
4996         my $new_conf = { %$conf };
4997         $new_conf->{$key} = $value;
4998         PVE::QemuServer::Cloudinit::generate_cloudinitconfig($new_conf, $vmid);
4999     };
5000
5001     foreach my $opt (keys %{$conf->{pending}}) {
5002         next if $selection && !$selection->{$opt};
5003         my $value = $conf->{pending}->{$opt};
5004         eval {
5005             if ($opt eq 'hotplug') {
5006                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
5007             } elsif ($opt eq 'tablet') {
5008                 die "skip\n" if !$hotplug_features->{usb};
5009                 if ($value == 1) {
5010                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
5011                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
5012                         if $arch eq 'aarch64';
5013                 } elsif ($value == 0) {
5014                     vm_deviceunplug($vmid, $conf, 'tablet');
5015                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
5016                 }
5017             } elsif ($opt =~ m/^usb\d+$/) {
5018                 die "skip\n";
5019                 # since we cannot reliably hot unplug usb devices
5020                 # we are disabling it
5021                 die "skip\n" if !$hotplug_features->{usb} || $value =~ m/spice/i;
5022                 my $d = eval { PVE::JSONSchema::parse_property_string($usbdesc->{format}, $value) };
5023                 die "skip\n" if !$d;
5024                 qemu_usb_hotplug($storecfg, $conf, $vmid, $opt, $d, $arch, $machine_type);
5025             } elsif ($opt eq 'vcpus') {
5026                 die "skip\n" if !$hotplug_features->{cpu};
5027                 qemu_cpu_hotplug($vmid, $conf, $value);
5028             } elsif ($opt eq 'balloon') {
5029                 # enable/disable balloning device is not hotpluggable
5030                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
5031                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
5032                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
5033
5034                 # allow manual ballooning if shares is set to zero
5035                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
5036                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
5037                     vm_mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
5038                 }
5039             } elsif ($opt =~ m/^net(\d+)$/) {
5040                 # some changes can be done without hotplug
5041                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
5042                                     $vmid, $opt, $value, $arch, $machine_type);
5043             } elsif (is_valid_drivename($opt)) {
5044                 # some changes can be done without hotplug
5045                 my $drive = parse_drive($opt, $value);
5046                 if (drive_is_cloudinit($drive)) {
5047                     &$apply_pending_cloudinit($opt, $value);
5048                 }
5049                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
5050                                      $vmid, $opt, $value, 1, $arch, $machine_type);
5051             } elsif ($opt =~ m/^memory$/) { #dimms
5052                 die "skip\n" if !$hotplug_features->{memory};
5053                 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
5054             } elsif ($opt eq 'cpuunits') {
5055                 cgroups_write("cpu", $vmid, "cpu.shares", $conf->{pending}->{$opt});
5056             } elsif ($opt eq 'cpulimit') {
5057                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
5058                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", $cpulimit);
5059             } else {
5060                 die "skip\n";  # skip non-hot-pluggable options
5061             }
5062         };
5063         if (my $err = $@) {
5064             &$add_error($opt, $err) if $err ne "skip\n";
5065         } else {
5066             # save new config if hotplug was successful
5067             $conf->{$opt} = $value;
5068             delete $conf->{pending}->{$opt};
5069             PVE::QemuConfig->write_config($vmid, $conf);
5070             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5071         }
5072     }
5073 }
5074
5075 sub try_deallocate_drive {
5076     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
5077
5078     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
5079         my $volid = $drive->{file};
5080         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
5081             my $sid = PVE::Storage::parse_volume_id($volid);
5082             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
5083
5084             # check if the disk is really unused
5085             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
5086                 if is_volume_in_use($storecfg, $conf, $key, $volid);
5087             PVE::Storage::vdisk_free($storecfg, $volid);
5088             return 1;
5089         } else {
5090             # If vm is not owner of this disk remove from config
5091             return 1;
5092         }
5093     }
5094
5095     return undef;
5096 }
5097
5098 sub vmconfig_delete_or_detach_drive {
5099     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
5100
5101     my $drive = parse_drive($opt, $conf->{$opt});
5102
5103     my $rpcenv = PVE::RPCEnvironment::get();
5104     my $authuser = $rpcenv->get_user();
5105
5106     if ($force) {
5107         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
5108         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
5109     } else {
5110         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
5111     }
5112 }
5113
5114 sub vmconfig_apply_pending {
5115     my ($vmid, $conf, $storecfg) = @_;
5116
5117     # cold plug
5118
5119     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
5120     while (my ($opt, $force) = each %$pending_delete_hash) {
5121         die "internal error" if $opt =~ m/^unused/;
5122         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5123         if (!defined($conf->{$opt})) {
5124             vmconfig_undelete_pending_option($conf, $opt);
5125             PVE::QemuConfig->write_config($vmid, $conf);
5126         } elsif (is_valid_drivename($opt)) {
5127             vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
5128             vmconfig_undelete_pending_option($conf, $opt);
5129             delete $conf->{$opt};
5130             PVE::QemuConfig->write_config($vmid, $conf);
5131         } else {
5132             vmconfig_undelete_pending_option($conf, $opt);
5133             delete $conf->{$opt};
5134             PVE::QemuConfig->write_config($vmid, $conf);
5135         }
5136     }
5137
5138     $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5139
5140     foreach my $opt (keys %{$conf->{pending}}) { # add/change
5141         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5142
5143         if (defined($conf->{$opt}) && ($conf->{$opt} eq $conf->{pending}->{$opt})) {
5144             # skip if nothing changed
5145         } elsif (is_valid_drivename($opt)) {
5146             vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
5147                 if defined($conf->{$opt});
5148             $conf->{$opt} = $conf->{pending}->{$opt};
5149         } else {
5150             $conf->{$opt} = $conf->{pending}->{$opt};
5151         }
5152
5153         delete $conf->{pending}->{$opt};
5154         PVE::QemuConfig->write_config($vmid, $conf);
5155     }
5156 }
5157
5158 my $safe_num_ne = sub {
5159     my ($a, $b) = @_;
5160
5161     return 0 if !defined($a) && !defined($b);
5162     return 1 if !defined($a);
5163     return 1 if !defined($b);
5164
5165     return $a != $b;
5166 };
5167
5168 my $safe_string_ne = sub {
5169     my ($a, $b) = @_;
5170
5171     return 0 if !defined($a) && !defined($b);
5172     return 1 if !defined($a);
5173     return 1 if !defined($b);
5174
5175     return $a ne $b;
5176 };
5177
5178 sub vmconfig_update_net {
5179     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
5180
5181     my $newnet = parse_net($value);
5182
5183     if ($conf->{$opt}) {
5184         my $oldnet = parse_net($conf->{$opt});
5185
5186         if (&$safe_string_ne($oldnet->{model}, $newnet->{model}) ||
5187             &$safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
5188             &$safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
5189             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
5190
5191             # for non online change, we try to hot-unplug
5192             die "skip\n" if !$hotplug;
5193             vm_deviceunplug($vmid, $conf, $opt);
5194         } else {
5195
5196             die "internal error" if $opt !~ m/net(\d+)/;
5197             my $iface = "tap${vmid}i$1";
5198
5199             if (&$safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
5200                 &$safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
5201                 &$safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
5202                 &$safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
5203                 PVE::Network::tap_unplug($iface);
5204                 PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
5205             } elsif (&$safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
5206                 # Rate can be applied on its own but any change above needs to
5207                 # include the rate in tap_plug since OVS resets everything.
5208                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
5209             }
5210
5211             if (&$safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
5212                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
5213             }
5214
5215             return 1;
5216         }
5217     }
5218
5219     if ($hotplug) {
5220         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet, $arch, $machine_type);
5221     } else {
5222         die "skip\n";
5223     }
5224 }
5225
5226 sub vmconfig_update_disk {
5227     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $force, $arch, $machine_type) = @_;
5228
5229     # fixme: do we need force?
5230
5231     my $drive = parse_drive($opt, $value);
5232
5233     if ($conf->{$opt}) {
5234
5235         if (my $old_drive = parse_drive($opt, $conf->{$opt}))  {
5236
5237             my $media = $drive->{media} || 'disk';
5238             my $oldmedia = $old_drive->{media} || 'disk';
5239             die "unable to change media type\n" if $media ne $oldmedia;
5240
5241             if (!drive_is_cdrom($old_drive)) {
5242
5243                 if ($drive->{file} ne $old_drive->{file}) {
5244
5245                     die "skip\n" if !$hotplug;
5246
5247                     # unplug and register as unused
5248                     vm_deviceunplug($vmid, $conf, $opt);
5249                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
5250
5251                 } else {
5252                     # update existing disk
5253
5254                     # skip non hotpluggable value
5255                     if (&$safe_string_ne($drive->{discard}, $old_drive->{discard}) ||
5256                         &$safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
5257                         &$safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
5258                         &$safe_string_ne($drive->{cache}, $old_drive->{cache})) {
5259                         die "skip\n";
5260                     }
5261
5262                     # apply throttle
5263                     if (&$safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
5264                         &$safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
5265                         &$safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
5266                         &$safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
5267                         &$safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
5268                         &$safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
5269                         &$safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
5270                         &$safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
5271                         &$safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
5272                         &$safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
5273                         &$safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
5274                         &$safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max}) ||
5275                         &$safe_num_ne($drive->{bps_max_length}, $old_drive->{bps_max_length}) ||
5276                         &$safe_num_ne($drive->{bps_rd_max_length}, $old_drive->{bps_rd_max_length}) ||
5277                         &$safe_num_ne($drive->{bps_wr_max_length}, $old_drive->{bps_wr_max_length}) ||
5278                         &$safe_num_ne($drive->{iops_max_length}, $old_drive->{iops_max_length}) ||
5279                         &$safe_num_ne($drive->{iops_rd_max_length}, $old_drive->{iops_rd_max_length}) ||
5280                         &$safe_num_ne($drive->{iops_wr_max_length}, $old_drive->{iops_wr_max_length})) {
5281
5282                         qemu_block_set_io_throttle($vmid,"drive-$opt",
5283                                                    ($drive->{mbps} || 0)*1024*1024,
5284                                                    ($drive->{mbps_rd} || 0)*1024*1024,
5285                                                    ($drive->{mbps_wr} || 0)*1024*1024,
5286                                                    $drive->{iops} || 0,
5287                                                    $drive->{iops_rd} || 0,
5288                                                    $drive->{iops_wr} || 0,
5289                                                    ($drive->{mbps_max} || 0)*1024*1024,
5290                                                    ($drive->{mbps_rd_max} || 0)*1024*1024,
5291                                                    ($drive->{mbps_wr_max} || 0)*1024*1024,
5292                                                    $drive->{iops_max} || 0,
5293                                                    $drive->{iops_rd_max} || 0,
5294                                                    $drive->{iops_wr_max} || 0,
5295                                                    $drive->{bps_max_length} || 1,
5296                                                    $drive->{bps_rd_max_length} || 1,
5297                                                    $drive->{bps_wr_max_length} || 1,
5298                                                    $drive->{iops_max_length} || 1,
5299                                                    $drive->{iops_rd_max_length} || 1,
5300                                                    $drive->{iops_wr_max_length} || 1);
5301
5302                     }
5303
5304                     return 1;
5305                 }
5306
5307             } else { # cdrom
5308
5309                 if ($drive->{file} eq 'none') {
5310                     vm_mon_cmd($vmid, "eject",force => JSON::true,device => "drive-$opt");
5311                     if (drive_is_cloudinit($old_drive)) {
5312                         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive);
5313                     }
5314                 } else {
5315                     my $path = get_iso_path($storecfg, $vmid, $drive->{file});
5316                     vm_mon_cmd($vmid, "eject", force => JSON::true,device => "drive-$opt"); # force eject if locked
5317                     vm_mon_cmd($vmid, "change", device => "drive-$opt",target => "$path") if $path;
5318                 }
5319
5320                 return 1;
5321             }
5322         }
5323     }
5324
5325     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
5326     # hotplug new disks
5327     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
5328     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive, $arch, $machine_type);
5329 }
5330
5331 sub vm_start {
5332     my ($storecfg, $vmid, $statefile, $skiplock, $migratedfrom, $paused,
5333         $forcemachine, $spice_ticket, $migration_network, $migration_type, $targetstorage) = @_;
5334
5335     PVE::QemuConfig->lock_config($vmid, sub {
5336         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
5337
5338         die "you can't start a vm if it's a template\n" if PVE::QemuConfig->is_template($conf);
5339
5340         my $is_suspended = PVE::QemuConfig->has_lock($conf, 'suspended');
5341
5342         PVE::QemuConfig->check_lock($conf)
5343             if !($skiplock || $is_suspended);
5344
5345         die "VM $vmid already running\n" if check_running($vmid, undef, $migratedfrom);
5346
5347         # clean up leftover reboot request files
5348         eval { clear_reboot_request($vmid); };
5349         warn $@ if $@;
5350
5351         if (!$statefile && scalar(keys %{$conf->{pending}})) {
5352             vmconfig_apply_pending($vmid, $conf, $storecfg);
5353             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5354         }
5355
5356         PVE::QemuServer::Cloudinit::generate_cloudinitconfig($conf, $vmid);
5357
5358         my $defaults = load_defaults();
5359
5360         # set environment variable useful inside network script
5361         $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
5362
5363         my $local_volumes = {};
5364
5365         if ($targetstorage) {
5366             foreach_drive($conf, sub {
5367                 my ($ds, $drive) = @_;
5368
5369                 return if drive_is_cdrom($drive);
5370
5371                 my $volid = $drive->{file};
5372
5373                 return if !$volid;
5374
5375                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
5376
5377                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5378                 return if $scfg->{shared};
5379                 $local_volumes->{$ds} = [$volid, $storeid, $volname];
5380             });
5381
5382             my $format = undef;
5383
5384             foreach my $opt (sort keys %$local_volumes) {
5385
5386                 my ($volid, $storeid, $volname) = @{$local_volumes->{$opt}};
5387                 my $drive = parse_drive($opt, $conf->{$opt});
5388
5389                 #if remote storage is specified, use default format
5390                 if ($targetstorage && $targetstorage ne "1") {
5391                     $storeid = $targetstorage;
5392                     my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
5393                     $format = $defFormat;
5394                 } else {
5395                     #else we use same format than original
5396                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5397                     $format = qemu_img_format($scfg, $volid);
5398                 }
5399
5400                 my $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $format, undef, ($drive->{size}/1024));
5401                 my $newdrive = $drive;
5402                 $newdrive->{format} = $format;
5403                 $newdrive->{file} = $newvolid;
5404                 my $drivestr = PVE::QemuServer::print_drive($vmid, $newdrive);
5405                 $local_volumes->{$opt} = $drivestr;
5406                 #pass drive to conf for command line
5407                 $conf->{$opt} = $drivestr;
5408             }
5409         }
5410
5411         PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-start', 1);
5412
5413         if ($is_suspended) {
5414             # enforce machine type on suspended vm to ensure HW compatibility
5415             $forcemachine = $conf->{runningmachine};
5416             print "Resuming suspended VM\n";
5417         }
5418
5419         my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine);
5420
5421         my $migrate_port = 0;
5422         my $migrate_uri;
5423         if ($statefile) {
5424             if ($statefile eq 'tcp') {
5425                 my $localip = "localhost";
5426                 my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
5427                 my $nodename = PVE::INotify::nodename();
5428
5429                 if (!defined($migration_type)) {
5430                     if (defined($datacenterconf->{migration}->{type})) {
5431                         $migration_type = $datacenterconf->{migration}->{type};
5432                     } else {
5433                         $migration_type = 'secure';
5434                     }
5435                 }
5436
5437                 if ($migration_type eq 'insecure') {
5438                     my $migrate_network_addr = PVE::Cluster::get_local_migration_ip($migration_network);
5439                     if ($migrate_network_addr) {
5440                         $localip = $migrate_network_addr;
5441                     } else {
5442                         $localip = PVE::Cluster::remote_node_ip($nodename, 1);
5443                     }
5444
5445                     $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5446                 }
5447
5448                 my $pfamily = PVE::Tools::get_host_address_family($nodename);
5449                 $migrate_port = PVE::Tools::next_migrate_port($pfamily);
5450                 $migrate_uri = "tcp:${localip}:${migrate_port}";
5451                 push @$cmd, '-incoming', $migrate_uri;
5452                 push @$cmd, '-S';
5453
5454             } elsif ($statefile eq 'unix') {
5455                 # should be default for secure migrations as a ssh TCP forward
5456                 # tunnel is not deterministic reliable ready and fails regurarly
5457                 # to set up in time, so use UNIX socket forwards
5458                 my $socket_addr = "/run/qemu-server/$vmid.migrate";
5459                 unlink $socket_addr;
5460
5461                 $migrate_uri = "unix:$socket_addr";
5462
5463                 push @$cmd, '-incoming', $migrate_uri;
5464                 push @$cmd, '-S';
5465
5466             } else {
5467                 push @$cmd, '-loadstate', $statefile;
5468             }
5469         } elsif ($paused) {
5470             push @$cmd, '-S';
5471         }
5472
5473         # host pci devices
5474         for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
5475           my $d = parse_hostpci($conf->{"hostpci$i"});
5476           next if !$d;
5477           my $pcidevices = $d->{pciid};
5478           foreach my $pcidevice (@$pcidevices) {
5479                 my $pciid = $pcidevice->{id};
5480
5481                 my $info = PVE::SysFSTools::pci_device_info("0000:$pciid");
5482                 die "IOMMU not present\n" if !PVE::SysFSTools::check_iommu_support();
5483                 die "no pci device info for device '$pciid'\n" if !$info;
5484
5485                 if ($d->{mdev}) {
5486                     my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $i);
5487                     PVE::SysFSTools::pci_create_mdev_device($pciid, $uuid, $d->{mdev});
5488                 } else {
5489                     die "can't unbind/bind pci group to vfio '$pciid'\n"
5490                         if !PVE::SysFSTools::pci_dev_group_bind_to_vfio($pciid);
5491                     die "can't reset pci device '$pciid'\n"
5492                         if $info->{has_fl_reset} and !PVE::SysFSTools::pci_dev_reset($info);
5493                 }
5494           }
5495         }
5496
5497         PVE::Storage::activate_volumes($storecfg, $vollist);
5498
5499         eval {
5500             run_command(['/bin/systemctl', 'stop', "$vmid.scope"],
5501                 outfunc => sub {}, errfunc => sub {});
5502         };
5503         # Issues with the above 'stop' not being fully completed are extremely rare, a very low
5504         # timeout should be more than enough here...
5505         PVE::Systemd::wait_for_unit_removed("$vmid.scope", 5);
5506
5507         my $cpuunits = defined($conf->{cpuunits}) ? $conf->{cpuunits}
5508                                                   : $defaults->{cpuunits};
5509
5510         my $start_timeout = ($conf->{hugepages} || $is_suspended) ? 300 : 30;
5511         my %run_params = (timeout => $statefile ? undef : $start_timeout, umask => 0077);
5512
5513         my %properties = (
5514             Slice => 'qemu.slice',
5515             KillMode => 'none',
5516             CPUShares => $cpuunits
5517         );
5518
5519         if (my $cpulimit = $conf->{cpulimit}) {
5520             $properties{CPUQuota} = int($cpulimit * 100);
5521         }
5522         $properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
5523
5524         my $run_qemu = sub {
5525             PVE::Tools::run_fork sub {
5526                 PVE::Systemd::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
5527                 run_command($cmd, %run_params);
5528             };
5529         };
5530
5531         if ($conf->{hugepages}) {
5532
5533             my $code = sub {
5534                 my $hugepages_topology = PVE::QemuServer::Memory::hugepages_topology($conf);
5535                 my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
5536
5537                 PVE::QemuServer::Memory::hugepages_mount();
5538                 PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
5539
5540                 eval { $run_qemu->() };
5541                 if (my $err = $@) {
5542                     PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology);
5543                     die $err;
5544                 }
5545
5546                 PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology);
5547             };
5548             eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
5549
5550         } else {
5551             eval { $run_qemu->() };
5552         }
5553
5554         if (my $err = $@) {
5555             # deactivate volumes if start fails
5556             eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
5557             die "start failed: $err";
5558         }
5559
5560         print "migration listens on $migrate_uri\n" if $migrate_uri;
5561
5562         if ($statefile && $statefile ne 'tcp' && $statefile ne 'unix')  {
5563             eval { vm_mon_cmd_nocheck($vmid, "cont"); };
5564             warn $@ if $@;
5565         }
5566
5567         #start nbd server for storage migration
5568         if ($targetstorage) {
5569             my $nodename = PVE::INotify::nodename();
5570             my $migrate_network_addr = PVE::Cluster::get_local_migration_ip($migration_network);
5571             my $localip = $migrate_network_addr ? $migrate_network_addr : PVE::Cluster::remote_node_ip($nodename, 1);
5572             my $pfamily = PVE::Tools::get_host_address_family($nodename);
5573             $migrate_port = PVE::Tools::next_migrate_port($pfamily);
5574
5575             vm_mon_cmd_nocheck($vmid, "nbd-server-start", addr => { type => 'inet', data => { host => "${localip}", port => "${migrate_port}" } } );
5576
5577             $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5578
5579             foreach my $opt (sort keys %$local_volumes) {
5580                 my $volid = $local_volumes->{$opt};
5581                 vm_mon_cmd_nocheck($vmid, "nbd-server-add", device => "drive-$opt", writable => JSON::true );
5582                 my $migrate_storage_uri = "nbd:${localip}:${migrate_port}:exportname=drive-$opt";
5583                 print "storage migration listens on $migrate_storage_uri volume:$volid\n";
5584             }
5585         }
5586
5587         if ($migratedfrom) {
5588             eval {
5589                 set_migration_caps($vmid);
5590             };
5591             warn $@ if $@;
5592
5593             if ($spice_port) {
5594                 print "spice listens on port $spice_port\n";
5595                 if ($spice_ticket) {
5596                     vm_mon_cmd_nocheck($vmid, "set_password", protocol => 'spice', password => $spice_ticket);
5597                     vm_mon_cmd_nocheck($vmid, "expire_password", protocol => 'spice', time => "+30");
5598                 }
5599             }
5600
5601         } else {
5602             vm_mon_cmd_nocheck($vmid, "balloon", value => $conf->{balloon}*1024*1024)
5603                 if !$statefile && $conf->{balloon};
5604
5605             foreach my $opt (keys %$conf) {
5606                 next if $opt !~  m/^net\d+$/;
5607                 my $nicconf = parse_net($conf->{$opt});
5608                 qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
5609             }
5610         }
5611
5612         vm_mon_cmd_nocheck($vmid, 'qom-set',
5613                     path => "machine/peripheral/balloon0",
5614                     property => "guest-stats-polling-interval",
5615                     value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
5616
5617         if ($is_suspended && (my $vmstate = $conf->{vmstate})) {
5618             print "Resumed VM, removing state\n";
5619             delete $conf->@{qw(lock vmstate runningmachine)};
5620             PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
5621             PVE::Storage::vdisk_free($storecfg, $vmstate);
5622             PVE::QemuConfig->write_config($vmid, $conf);
5623         }
5624
5625         PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'post-start');
5626     });
5627 }
5628
5629 sub vm_mon_cmd {
5630     my ($vmid, $execute, %params) = @_;
5631
5632     my $cmd = { execute => $execute, arguments => \%params };
5633     vm_qmp_command($vmid, $cmd);
5634 }
5635
5636 sub vm_mon_cmd_nocheck {
5637     my ($vmid, $execute, %params) = @_;
5638
5639     my $cmd = { execute => $execute, arguments => \%params };
5640     vm_qmp_command($vmid, $cmd, 1);
5641 }
5642
5643 sub vm_qmp_command {
5644     my ($vmid, $cmd, $nocheck) = @_;
5645
5646     my $res;
5647
5648     my $timeout;
5649     if ($cmd->{arguments}) {
5650         $timeout = delete $cmd->{arguments}->{timeout};
5651     }
5652
5653     eval {
5654         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
5655         my $sname = qmp_socket($vmid);
5656         if (-e $sname) { # test if VM is reasonambe new and supports qmp/qga
5657             my $qmpclient = PVE::QMPClient->new();
5658
5659             $res = $qmpclient->cmd($vmid, $cmd, $timeout);
5660         } else {
5661             die "unable to open monitor socket\n";
5662         }
5663     };
5664     if (my $err = $@) {
5665         syslog("err", "VM $vmid qmp command failed - $err");
5666         die $err;
5667     }
5668
5669     return $res;
5670 }
5671
5672 sub vm_human_monitor_command {
5673     my ($vmid, $cmdline) = @_;
5674
5675     my $cmd = {
5676         execute => 'human-monitor-command',
5677         arguments => { 'command-line' => $cmdline},
5678     };
5679
5680     return vm_qmp_command($vmid, $cmd);
5681 }
5682
5683 sub vm_commandline {
5684     my ($storecfg, $vmid, $snapname) = @_;
5685
5686     my $conf = PVE::QemuConfig->load_config($vmid);
5687
5688     if ($snapname) {
5689         my $snapshot = $conf->{snapshots}->{$snapname};
5690         die "snapshot '$snapname' does not exist\n" if !defined($snapshot);
5691
5692         $snapshot->{digest} = $conf->{digest}; # keep file digest for API
5693
5694         $conf = $snapshot;
5695     }
5696
5697     my $defaults = load_defaults();
5698
5699     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults);
5700
5701     return PVE::Tools::cmd2string($cmd);
5702 }
5703
5704 sub vm_reset {
5705     my ($vmid, $skiplock) = @_;
5706
5707     PVE::QemuConfig->lock_config($vmid, sub {
5708
5709         my $conf = PVE::QemuConfig->load_config($vmid);
5710
5711         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5712
5713         vm_mon_cmd($vmid, "system_reset");
5714     });
5715 }
5716
5717 sub get_vm_volumes {
5718     my ($conf) = @_;
5719
5720     my $vollist = [];
5721     foreach_volid($conf, sub {
5722         my ($volid, $attr) = @_;
5723
5724         return if $volid =~ m|^/|;
5725
5726         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5727         return if !$sid;
5728
5729         push @$vollist, $volid;
5730     });
5731
5732     return $vollist;
5733 }
5734
5735 sub vm_stop_cleanup {
5736     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
5737
5738     eval {
5739
5740         if (!$keepActive) {
5741             my $vollist = get_vm_volumes($conf);
5742             PVE::Storage::deactivate_volumes($storecfg, $vollist);
5743         }
5744
5745         foreach my $ext (qw(mon qmp pid vnc qga)) {
5746             unlink "/var/run/qemu-server/${vmid}.$ext";
5747         }
5748
5749         if ($conf->{ivshmem}) {
5750             my $ivshmem = PVE::JSONSchema::parse_property_string($ivshmem_fmt, $conf->{ivshmem});
5751             # just delete it for now, VMs which have this already open do not
5752             # are affected, but new VMs will get a separated one. If this
5753             # becomes an issue we either add some sort of ref-counting or just
5754             # add a "don't delete on stop" flag to the ivshmem format.
5755             unlink '/dev/shm/pve-shm-' . ($ivshmem->{name} // $vmid);
5756         }
5757
5758         foreach my $key (keys %$conf) {
5759             next if $key !~ m/^hostpci(\d+)$/;
5760             my $hostpciindex = $1;
5761             my $d = parse_hostpci($conf->{$key});
5762             my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $hostpciindex);
5763
5764             foreach my $pci (@{$d->{pciid}}) {
5765                 my $pciid = $pci->{id};
5766                 PVE::SysFSTools::pci_cleanup_mdev_device($pciid, $uuid);
5767             }
5768         }
5769
5770         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
5771     };
5772     warn $@ if $@; # avoid errors - just warn
5773 }
5774
5775 # call only in locked context
5776 sub _do_vm_stop {
5777     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive) = @_;
5778
5779     my $pid = check_running($vmid, $nocheck);
5780     return if !$pid;
5781
5782     my $conf;
5783     if (!$nocheck) {
5784         $conf = PVE::QemuConfig->load_config($vmid);
5785         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5786         if (!defined($timeout) && $shutdown && $conf->{startup}) {
5787             my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
5788             $timeout = $opts->{down} if $opts->{down};
5789         }
5790         PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-stop');
5791     }
5792
5793     eval {
5794         if ($shutdown) {
5795             if (defined($conf) && parse_guest_agent($conf)->{enabled}) {
5796                 vm_qmp_command($vmid, {
5797                         execute => "guest-shutdown",
5798                         arguments => { timeout => $timeout }
5799                     }, $nocheck);
5800             } else {
5801                 vm_qmp_command($vmid, { execute => "system_powerdown" }, $nocheck);
5802             }
5803         } else {
5804             vm_qmp_command($vmid, { execute => "quit" }, $nocheck);
5805         }
5806     };
5807     my $err = $@;
5808
5809     if (!$err) {
5810         $timeout = 60 if !defined($timeout);
5811
5812         my $count = 0;
5813         while (($count < $timeout) && check_running($vmid, $nocheck)) {
5814             $count++;
5815             sleep 1;
5816         }
5817
5818         if ($count >= $timeout) {
5819             if ($force) {
5820                 warn "VM still running - terminating now with SIGTERM\n";
5821                 kill 15, $pid;
5822             } else {
5823                 die "VM quit/powerdown failed - got timeout\n";
5824             }
5825         } else {
5826             vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5827             return;
5828         }
5829     } else {
5830         if ($force) {
5831             warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
5832             kill 15, $pid;
5833         } else {
5834             die "VM quit/powerdown failed\n";
5835         }
5836     }
5837
5838     # wait again
5839     $timeout = 10;
5840
5841     my $count = 0;
5842     while (($count < $timeout) && check_running($vmid, $nocheck)) {
5843         $count++;
5844         sleep 1;
5845     }
5846
5847     if ($count >= $timeout) {
5848         warn "VM still running - terminating now with SIGKILL\n";
5849         kill 9, $pid;
5850         sleep 1;
5851     }
5852
5853     vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5854 }
5855
5856 # Note: use $nocheck to skip tests if VM configuration file exists.
5857 # We need that when migration VMs to other nodes (files already moved)
5858 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
5859 sub vm_stop {
5860     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
5861
5862     $force = 1 if !defined($force) && !$shutdown;
5863
5864     if ($migratedfrom){
5865         my $pid = check_running($vmid, $nocheck, $migratedfrom);
5866         kill 15, $pid if $pid;
5867         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
5868         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
5869         return;
5870     }
5871
5872     PVE::QemuConfig->lock_config($vmid, sub {
5873         _do_vm_stop($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive);
5874    });
5875 }
5876
5877 sub vm_reboot {
5878     my ($vmid, $timeout) = @_;
5879
5880     PVE::QemuConfig->lock_config($vmid, sub {
5881
5882         # only reboot if running, as qmeventd starts it again on a stop event
5883         return if !check_running($vmid);
5884
5885         create_reboot_request($vmid);
5886
5887         my $storecfg = PVE::Storage::config();
5888         _do_vm_stop($storecfg, $vmid, undef, undef, $timeout, 1);
5889
5890    });
5891 }
5892
5893 sub vm_suspend {
5894     my ($vmid, $skiplock, $includestate, $statestorage) = @_;
5895
5896     my $conf;
5897     my $path;
5898     my $storecfg;
5899     my $vmstate;
5900
5901     PVE::QemuConfig->lock_config($vmid, sub {
5902
5903         $conf = PVE::QemuConfig->load_config($vmid);
5904
5905         my $is_backing_up = PVE::QemuConfig->has_lock($conf, 'backup');
5906         PVE::QemuConfig->check_lock($conf)
5907             if !($skiplock || $is_backing_up);
5908
5909         die "cannot suspend to disk during backup\n"
5910             if $is_backing_up && $includestate;
5911
5912         if ($includestate) {
5913             $conf->{lock} = 'suspending';
5914             my $date = strftime("%Y-%m-%d", localtime(time()));
5915             $storecfg = PVE::Storage::config();
5916             $vmstate = PVE::QemuConfig->__snapshot_save_vmstate($vmid, $conf, "suspend-$date", $storecfg, $statestorage, 1);
5917             $path = PVE::Storage::path($storecfg, $vmstate);
5918             PVE::QemuConfig->write_config($vmid, $conf);
5919         } else {
5920             vm_mon_cmd($vmid, "stop");
5921         }
5922     });
5923
5924     if ($includestate) {
5925         # save vm state
5926         PVE::Storage::activate_volumes($storecfg, [$vmstate]);
5927
5928         eval {
5929             vm_mon_cmd($vmid, "savevm-start", statefile => $path);
5930             for(;;) {
5931                 my $state = vm_mon_cmd_nocheck($vmid, "query-savevm");
5932                 if (!$state->{status}) {
5933                     die "savevm not active\n";
5934                 } elsif ($state->{status} eq 'active') {
5935                     sleep(1);
5936                     next;
5937                 } elsif ($state->{status} eq 'completed') {
5938                     print "State saved, quitting\n";
5939                     last;
5940                 } elsif ($state->{status} eq 'failed' && $state->{error}) {
5941                     die "query-savevm failed with error '$state->{error}'\n"
5942                 } else {
5943                     die "query-savevm returned status '$state->{status}'\n";
5944                 }
5945             }
5946         };
5947         my $err = $@;
5948
5949         PVE::QemuConfig->lock_config($vmid, sub {
5950             $conf = PVE::QemuConfig->load_config($vmid);
5951             if ($err) {
5952                 # cleanup, but leave suspending lock, to indicate something went wrong
5953                 eval {
5954                     vm_mon_cmd($vmid, "savevm-end");
5955                     PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
5956                     PVE::Storage::vdisk_free($storecfg, $vmstate);
5957                     delete $conf->@{qw(vmstate runningmachine)};
5958                     PVE::QemuConfig->write_config($vmid, $conf);
5959                 };
5960                 warn $@ if $@;
5961                 die $err;
5962             }
5963
5964             die "lock changed unexpectedly\n"
5965                 if !PVE::QemuConfig->has_lock($conf, 'suspending');
5966
5967             vm_qmp_command($vmid, { execute => "quit" });
5968             $conf->{lock} = 'suspended';
5969             PVE::QemuConfig->write_config($vmid, $conf);
5970         });
5971     }
5972 }
5973
5974 sub vm_resume {
5975     my ($vmid, $skiplock, $nocheck) = @_;
5976
5977     PVE::QemuConfig->lock_config($vmid, sub {
5978         my $vm_mon_cmd = $nocheck ? \&vm_mon_cmd_nocheck : \&vm_mon_cmd;
5979         my $res = $vm_mon_cmd->($vmid, 'query-status');
5980         my $resume_cmd = 'cont';
5981
5982         if ($res->{status} && $res->{status} eq 'suspended') {
5983             $resume_cmd = 'system_wakeup';
5984         }
5985
5986         if (!$nocheck) {
5987
5988             my $conf = PVE::QemuConfig->load_config($vmid);
5989
5990             PVE::QemuConfig->check_lock($conf)
5991                 if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
5992         }
5993
5994         $vm_mon_cmd->($vmid, $resume_cmd);
5995     });
5996 }
5997
5998 sub vm_sendkey {
5999     my ($vmid, $skiplock, $key) = @_;
6000
6001     PVE::QemuConfig->lock_config($vmid, sub {
6002
6003         my $conf = PVE::QemuConfig->load_config($vmid);
6004
6005         # there is no qmp command, so we use the human monitor command
6006         my $res = vm_human_monitor_command($vmid, "sendkey $key");
6007         die $res if $res ne '';
6008     });
6009 }
6010
6011 sub vm_destroy {
6012     my ($storecfg, $vmid, $skiplock) = @_;
6013
6014     PVE::QemuConfig->lock_config($vmid, sub {
6015
6016         my $conf = PVE::QemuConfig->load_config($vmid);
6017
6018         if (!check_running($vmid)) {
6019             destroy_vm($storecfg, $vmid, undef, $skiplock);
6020         } else {
6021             die "VM $vmid is running - destroy failed\n";
6022         }
6023     });
6024 }
6025
6026 # vzdump restore implementaion
6027
6028 sub tar_archive_read_firstfile {
6029     my $archive = shift;
6030
6031     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
6032
6033     # try to detect archive type first
6034     my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
6035         die "unable to open file '$archive'\n";
6036     my $firstfile = <$fh>;
6037     kill 15, $pid;
6038     close $fh;
6039
6040     die "ERROR: archive contaions no data\n" if !$firstfile;
6041     chomp $firstfile;
6042
6043     return $firstfile;
6044 }
6045
6046 sub tar_restore_cleanup {
6047     my ($storecfg, $statfile) = @_;
6048
6049     print STDERR "starting cleanup\n";
6050
6051     if (my $fd = IO::File->new($statfile, "r")) {
6052         while (defined(my $line = <$fd>)) {
6053             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
6054                 my $volid = $2;
6055                 eval {
6056                     if ($volid =~ m|^/|) {
6057                         unlink $volid || die 'unlink failed\n';
6058                     } else {
6059                         PVE::Storage::vdisk_free($storecfg, $volid);
6060                     }
6061                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
6062                 };
6063                 print STDERR "unable to cleanup '$volid' - $@" if $@;
6064             } else {
6065                 print STDERR "unable to parse line in statfile - $line";
6066             }
6067         }
6068         $fd->close();
6069     }
6070 }
6071
6072 sub restore_archive {
6073     my ($archive, $vmid, $user, $opts) = @_;
6074
6075     my $format = $opts->{format};
6076     my $comp;
6077
6078     if ($archive =~ m/\.tgz$/ || $archive =~ m/\.tar\.gz$/) {
6079         $format = 'tar' if !$format;
6080         $comp = 'gzip';
6081     } elsif ($archive =~ m/\.tar$/) {
6082         $format = 'tar' if !$format;
6083     } elsif ($archive =~ m/.tar.lzo$/) {
6084         $format = 'tar' if !$format;
6085         $comp = 'lzop';
6086     } elsif ($archive =~ m/\.vma$/) {
6087         $format = 'vma' if !$format;
6088     } elsif ($archive =~ m/\.vma\.gz$/) {
6089         $format = 'vma' if !$format;
6090         $comp = 'gzip';
6091     } elsif ($archive =~ m/\.vma\.lzo$/) {
6092         $format = 'vma' if !$format;
6093         $comp = 'lzop';
6094     } else {
6095         $format = 'vma' if !$format; # default
6096     }
6097
6098     # try to detect archive format
6099     if ($format eq 'tar') {
6100         return restore_tar_archive($archive, $vmid, $user, $opts);
6101     } else {
6102         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
6103     }
6104 }
6105
6106 sub restore_update_config_line {
6107     my ($outfd, $cookie, $vmid, $map, $line, $unique) = @_;
6108
6109     return if $line =~ m/^\#qmdump\#/;
6110     return if $line =~ m/^\#vzdump\#/;
6111     return if $line =~ m/^lock:/;
6112     return if $line =~ m/^unused\d+:/;
6113     return if $line =~ m/^parent:/;
6114
6115     my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
6116     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
6117         # try to convert old 1.X settings
6118         my ($id, $ind, $ethcfg) = ($1, $2, $3);
6119         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
6120             my ($model, $macaddr) = split(/\=/, $devconfig);
6121             $macaddr = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if !$macaddr || $unique;
6122             my $net = {
6123                 model => $model,
6124                 bridge => "vmbr$ind",
6125                 macaddr => $macaddr,
6126             };
6127             my $netstr = print_net($net);
6128
6129             print $outfd "net$cookie->{netcount}: $netstr\n";
6130             $cookie->{netcount}++;
6131         }
6132     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
6133         my ($id, $netstr) = ($1, $2);
6134         my $net = parse_net($netstr);
6135         $net->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if $net->{macaddr};
6136         $netstr = print_net($net);
6137         print $outfd "$id: $netstr\n";
6138     } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk)\d+):\s*(\S+)\s*$/) {
6139         my $virtdev = $1;
6140         my $value = $3;
6141         my $di = parse_drive($virtdev, $value);
6142         if (defined($di->{backup}) && !$di->{backup}) {
6143             print $outfd "#$line";
6144         } elsif ($map->{$virtdev}) {
6145             delete $di->{format}; # format can change on restore
6146             $di->{file} = $map->{$virtdev};
6147             $value = print_drive($vmid, $di);
6148             print $outfd "$virtdev: $value\n";
6149         } else {
6150             print $outfd $line;
6151         }
6152     } elsif (($line =~ m/^vmgenid: (.*)/)) {
6153         my $vmgenid = $1;
6154         if ($vmgenid ne '0') {
6155             # always generate a new vmgenid if there was a valid one setup
6156             $vmgenid = generate_uuid();
6157         }
6158         print $outfd "vmgenid: $vmgenid\n";
6159     } elsif (($line =~ m/^(smbios1: )(.*)/) && $unique) {
6160         my ($uuid, $uuid_str);
6161         UUID::generate($uuid);
6162         UUID::unparse($uuid, $uuid_str);
6163         my $smbios1 = parse_smbios1($2);
6164         $smbios1->{uuid} = $uuid_str;
6165         print $outfd $1.print_smbios1($smbios1)."\n";
6166     } else {
6167         print $outfd $line;
6168     }
6169 }
6170
6171 sub scan_volids {
6172     my ($cfg, $vmid) = @_;
6173
6174     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid);
6175
6176     my $volid_hash = {};
6177     foreach my $storeid (keys %$info) {
6178         foreach my $item (@{$info->{$storeid}}) {
6179             next if !($item->{volid} && $item->{size});
6180             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
6181             $volid_hash->{$item->{volid}} = $item;
6182         }
6183     }
6184
6185     return $volid_hash;
6186 }
6187
6188 sub is_volume_in_use {
6189     my ($storecfg, $conf, $skip_drive, $volid) = @_;
6190
6191     my $path = PVE::Storage::path($storecfg, $volid);
6192
6193     my $scan_config = sub {
6194         my ($cref, $snapname) = @_;
6195
6196         foreach my $key (keys %$cref) {
6197             my $value = $cref->{$key};
6198             if (is_valid_drivename($key)) {
6199                 next if $skip_drive && $key eq $skip_drive;
6200                 my $drive = parse_drive($key, $value);
6201                 next if !$drive || !$drive->{file} || drive_is_cdrom($drive);
6202                 return 1 if $volid eq $drive->{file};
6203                 if ($drive->{file} =~ m!^/!) {
6204                     return 1 if $drive->{file} eq $path;
6205                 } else {
6206                     my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file}, 1);
6207                     next if !$storeid;
6208                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid, 1);
6209                     next if !$scfg;
6210                     return 1 if $path eq PVE::Storage::path($storecfg, $drive->{file}, $snapname);
6211                 }
6212             }
6213         }
6214
6215         return 0;
6216     };
6217
6218     return 1 if &$scan_config($conf);
6219
6220     undef $skip_drive;
6221
6222     foreach my $snapname (keys %{$conf->{snapshots}}) {
6223         return 1 if &$scan_config($conf->{snapshots}->{$snapname}, $snapname);
6224     }
6225
6226     return 0;
6227 }
6228
6229 sub update_disksize {
6230     my ($vmid, $conf, $volid_hash) = @_;
6231
6232     my $changes;
6233     my $prefix = "VM $vmid:";
6234
6235     # used and unused disks
6236     my $referenced = {};
6237
6238     # Note: it is allowed to define multiple storages with same path (alias), so
6239     # we need to check both 'volid' and real 'path' (two different volid can point
6240     # to the same path).
6241
6242     my $referencedpath = {};
6243
6244     # update size info
6245     foreach my $opt (keys %$conf) {
6246         if (is_valid_drivename($opt)) {
6247             my $drive = parse_drive($opt, $conf->{$opt});
6248             my $volid = $drive->{file};
6249             next if !$volid;
6250
6251             $referenced->{$volid} = 1;
6252             if ($volid_hash->{$volid} &&
6253                 (my $path = $volid_hash->{$volid}->{path})) {
6254                 $referencedpath->{$path} = 1;
6255             }
6256
6257             next if drive_is_cdrom($drive);
6258             next if !$volid_hash->{$volid};
6259
6260             $drive->{size} = $volid_hash->{$volid}->{size};
6261             my $new = print_drive($vmid, $drive);
6262             if ($new ne $conf->{$opt}) {
6263                 $changes = 1;
6264                 $conf->{$opt} = $new;
6265                 print "$prefix update disk '$opt' information.\n";
6266             }
6267         }
6268     }
6269
6270     # remove 'unusedX' entry if volume is used
6271     foreach my $opt (keys %$conf) {
6272         next if $opt !~ m/^unused\d+$/;
6273         my $volid = $conf->{$opt};
6274         my $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
6275         if ($referenced->{$volid} || ($path && $referencedpath->{$path})) {
6276             print "$prefix remove entry '$opt', its volume '$volid' is in use.\n";
6277             $changes = 1;
6278             delete $conf->{$opt};
6279         }
6280
6281         $referenced->{$volid} = 1;
6282         $referencedpath->{$path} = 1 if $path;
6283     }
6284
6285     foreach my $volid (sort keys %$volid_hash) {
6286         next if $volid =~ m/vm-$vmid-state-/;
6287         next if $referenced->{$volid};
6288         my $path = $volid_hash->{$volid}->{path};
6289         next if !$path; # just to be sure
6290         next if $referencedpath->{$path};
6291         $changes = 1;
6292         my $key = PVE::QemuConfig->add_unused_volume($conf, $volid);
6293         print "$prefix add unreferenced volume '$volid' as '$key' to config.\n";
6294         $referencedpath->{$path} = 1; # avoid to add more than once (aliases)
6295     }
6296
6297     return $changes;
6298 }
6299
6300 sub rescan {
6301     my ($vmid, $nolock, $dryrun) = @_;
6302
6303     my $cfg = PVE::Storage::config();
6304
6305     # FIXME: Remove once our RBD plugin can handle CT and VM on a single storage
6306     # see: https://pve.proxmox.com/pipermail/pve-devel/2018-July/032900.html
6307     foreach my $stor (keys %{$cfg->{ids}}) {
6308         delete($cfg->{ids}->{$stor}) if ! $cfg->{ids}->{$stor}->{content}->{images};
6309     }
6310
6311     print "rescan volumes...\n";
6312     my $volid_hash = scan_volids($cfg, $vmid);
6313
6314     my $updatefn =  sub {
6315         my ($vmid) = @_;
6316
6317         my $conf = PVE::QemuConfig->load_config($vmid);
6318
6319         PVE::QemuConfig->check_lock($conf);
6320
6321         my $vm_volids = {};
6322         foreach my $volid (keys %$volid_hash) {
6323             my $info = $volid_hash->{$volid};
6324             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
6325         }
6326
6327         my $changes = update_disksize($vmid, $conf, $vm_volids);
6328
6329         PVE::QemuConfig->write_config($vmid, $conf) if $changes && !$dryrun;
6330     };
6331
6332     if (defined($vmid)) {
6333         if ($nolock) {
6334             &$updatefn($vmid);
6335         } else {
6336             PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6337         }
6338     } else {
6339         my $vmlist = config_list();
6340         foreach my $vmid (keys %$vmlist) {
6341             if ($nolock) {
6342                 &$updatefn($vmid);
6343             } else {
6344                 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6345             }
6346         }
6347     }
6348 }
6349
6350 sub restore_vma_archive {
6351     my ($archive, $vmid, $user, $opts, $comp) = @_;
6352
6353     my $readfrom = $archive;
6354
6355     my $cfg = PVE::Storage::config();
6356     my $commands = [];
6357     my $bwlimit = $opts->{bwlimit};
6358
6359     my $dbg_cmdstring = '';
6360     my $add_pipe = sub {
6361         my ($cmd) = @_;
6362         push @$commands, $cmd;
6363         $dbg_cmdstring .= ' | ' if length($dbg_cmdstring);
6364         $dbg_cmdstring .= PVE::Tools::cmd2string($cmd);
6365         $readfrom = '-';
6366     };
6367
6368     my $input = undef;
6369     if ($archive eq '-') {
6370         $input = '<&STDIN';
6371     } else {
6372         # If we use a backup from a PVE defined storage we also consider that
6373         # storage's rate limit:
6374         my (undef, $volid) = PVE::Storage::path_to_volume_id($cfg, $archive);
6375         if (defined($volid)) {
6376             my ($sid, undef) = PVE::Storage::parse_volume_id($volid);
6377             my $readlimit = PVE::Storage::get_bandwidth_limit('restore', [$sid], $bwlimit);
6378             if ($readlimit) {
6379                 print STDERR "applying read rate limit: $readlimit\n";
6380                 my $cstream = ['cstream', '-t', $readlimit*1024, '--', $readfrom];
6381                 $add_pipe->($cstream);
6382             }
6383         }
6384     }
6385
6386     if ($comp) {
6387         my $cmd;
6388         if ($comp eq 'gzip') {
6389             $cmd = ['zcat', $readfrom];
6390         } elsif ($comp eq 'lzop') {
6391             $cmd = ['lzop', '-d', '-c', $readfrom];
6392         } else {
6393             die "unknown compression method '$comp'\n";
6394         }
6395         $add_pipe->($cmd);
6396     }
6397
6398     my $tmpdir = "/var/tmp/vzdumptmp$$";
6399     rmtree $tmpdir;
6400
6401     # disable interrupts (always do cleanups)
6402     local $SIG{INT} =
6403         local $SIG{TERM} =
6404         local $SIG{QUIT} =
6405         local $SIG{HUP} = sub { warn "got interrupt - ignored\n"; };
6406
6407     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
6408     POSIX::mkfifo($mapfifo, 0600);
6409     my $fifofh;
6410
6411     my $openfifo = sub {
6412         open($fifofh, '>', $mapfifo) || die $!;
6413     };
6414
6415     $add_pipe->(['vma', 'extract', '-v', '-r', $mapfifo, $readfrom, $tmpdir]);
6416
6417     my $oldtimeout;
6418     my $timeout = 5;
6419
6420     my $devinfo = {};
6421
6422     my $rpcenv = PVE::RPCEnvironment::get();
6423
6424     my $conffile = PVE::QemuConfig->config_file($vmid);
6425     my $tmpfn = "$conffile.$$.tmp";
6426
6427     # Note: $oldconf is undef if VM does not exists
6428     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
6429     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
6430
6431     my %storage_limits;
6432
6433     my $print_devmap = sub {
6434         my $virtdev_hash = {};
6435
6436         my $cfgfn = "$tmpdir/qemu-server.conf";
6437
6438         # we can read the config - that is already extracted
6439         my $fh = IO::File->new($cfgfn, "r") ||
6440             "unable to read qemu-server.conf - $!\n";
6441
6442         my $fwcfgfn = "$tmpdir/qemu-server.fw";
6443         if (-f $fwcfgfn) {
6444             my $pve_firewall_dir = '/etc/pve/firewall';
6445             mkdir $pve_firewall_dir; # make sure the dir exists
6446             PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
6447         }
6448
6449         while (defined(my $line = <$fh>)) {
6450             if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
6451                 my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
6452                 die "archive does not contain data for drive '$virtdev'\n"
6453                     if !$devinfo->{$devname};
6454                 if (defined($opts->{storage})) {
6455                     $storeid = $opts->{storage} || 'local';
6456                 } elsif (!$storeid) {
6457                     $storeid = 'local';
6458                 }
6459                 $format = 'raw' if !$format;
6460                 $devinfo->{$devname}->{devname} = $devname;
6461                 $devinfo->{$devname}->{virtdev} = $virtdev;
6462                 $devinfo->{$devname}->{format} = $format;
6463                 $devinfo->{$devname}->{storeid} = $storeid;
6464
6465                 # check permission on storage
6466                 my $pool = $opts->{pool}; # todo: do we need that?
6467                 if ($user ne 'root@pam') {
6468                     $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
6469                 }
6470
6471                 $storage_limits{$storeid} = $bwlimit;
6472
6473                 $virtdev_hash->{$virtdev} = $devinfo->{$devname};
6474             } elsif ($line =~ m/^((?:ide|sata|scsi)\d+):\s*(.*)\s*$/) {
6475                 my $virtdev = $1;
6476                 my $drive = parse_drive($virtdev, $2);
6477                 if (drive_is_cloudinit($drive)) {
6478                     my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6479                     my $scfg = PVE::Storage::storage_config($cfg, $storeid);
6480                     my $format = qemu_img_format($scfg, $volname); # has 'raw' fallback
6481
6482                     my $d = {
6483                         format => $format,
6484                         storeid => $opts->{storage} // $storeid,
6485                         size => PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE,
6486                         file => $drive->{file}, # to make drive_is_cloudinit check possible
6487                         name => "vm-$vmid-cloudinit",
6488                         is_cloudinit => 1,
6489                     };
6490                     $virtdev_hash->{$virtdev} = $d;
6491                 }
6492             }
6493         }
6494
6495         foreach my $key (keys %storage_limits) {
6496             my $limit = PVE::Storage::get_bandwidth_limit('restore', [$key], $bwlimit);
6497             next if !$limit;
6498             print STDERR "rate limit for storage $key: $limit KiB/s\n";
6499             $storage_limits{$key} = $limit * 1024;
6500         }
6501
6502         foreach my $devname (keys %$devinfo) {
6503             die "found no device mapping information for device '$devname'\n"
6504                 if !$devinfo->{$devname}->{virtdev};
6505         }
6506
6507         # create empty/temp config
6508         if ($oldconf) {
6509             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
6510             foreach_drive($oldconf, sub {
6511                 my ($ds, $drive) = @_;
6512
6513                 return if drive_is_cdrom($drive, 1);
6514
6515                 my $volid = $drive->{file};
6516                 return if !$volid || $volid =~ m|^/|;
6517
6518                 my ($path, $owner) = PVE::Storage::path($cfg, $volid);
6519                 return if !$path || !$owner || ($owner != $vmid);
6520
6521                 # Note: only delete disk we want to restore
6522                 # other volumes will become unused
6523                 if ($virtdev_hash->{$ds}) {
6524                     eval { PVE::Storage::vdisk_free($cfg, $volid); };
6525                     if (my $err = $@) {
6526                         warn $err;
6527                     }
6528                 }
6529             });
6530
6531             # delete vmstate files, after the restore we have no snapshots anymore
6532             foreach my $snapname (keys %{$oldconf->{snapshots}}) {
6533                 my $snap = $oldconf->{snapshots}->{$snapname};
6534                 if ($snap->{vmstate}) {
6535                     eval { PVE::Storage::vdisk_free($cfg, $snap->{vmstate}); };
6536                     if (my $err = $@) {
6537                         warn $err;
6538                     }
6539                 }
6540             }
6541         }
6542
6543         my $map = {};
6544         foreach my $virtdev (sort keys %$virtdev_hash) {
6545             my $d = $virtdev_hash->{$virtdev};
6546             my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
6547             my $storeid = $d->{storeid};
6548             my $scfg = PVE::Storage::storage_config($cfg, $storeid);
6549
6550             my $map_opts = '';
6551             if (my $limit = $storage_limits{$storeid}) {
6552                 $map_opts .= "throttling.bps=$limit:throttling.group=$storeid:";
6553             }
6554
6555             # test if requested format is supported
6556             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($cfg, $storeid);
6557             my $supported = grep { $_ eq $d->{format} } @$validFormats;
6558             $d->{format} = $defFormat if !$supported;
6559
6560             my $name;
6561             if ($d->{is_cloudinit}) {
6562                 $name = $d->{name};
6563                 $name .= ".$d->{format}" if $d->{format} ne 'raw';
6564             }
6565
6566             my $volid = PVE::Storage::vdisk_alloc($cfg, $storeid, $vmid, $d->{format}, $name, $alloc_size);
6567             print STDERR "new volume ID is '$volid'\n";
6568             $d->{volid} = $volid;
6569
6570             PVE::Storage::activate_volumes($cfg, [$volid]);
6571
6572             my $write_zeros = 1;
6573             if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
6574                 $write_zeros = 0;
6575             }
6576
6577             if (!$d->{is_cloudinit}) {
6578                 my $path = PVE::Storage::path($cfg, $volid);
6579
6580                 print $fifofh "${map_opts}format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
6581
6582                 print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
6583             }
6584             $map->{$virtdev} = $volid;
6585         }
6586
6587         $fh->seek(0, 0) || die "seek failed - $!\n";
6588
6589         my $outfd = new IO::File ($tmpfn, "w") ||
6590             die "unable to write config for VM $vmid\n";
6591
6592         my $cookie = { netcount => 0 };
6593         while (defined(my $line = <$fh>)) {
6594             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
6595         }
6596
6597         $fh->close();
6598         $outfd->close();
6599     };
6600
6601     eval {
6602         # enable interrupts
6603         local $SIG{INT} =
6604             local $SIG{TERM} =
6605             local $SIG{QUIT} =
6606             local $SIG{HUP} =
6607             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6608         local $SIG{ALRM} = sub { die "got timeout\n"; };
6609
6610         $oldtimeout = alarm($timeout);
6611
6612         my $parser = sub {
6613             my $line = shift;
6614
6615             print "$line\n";
6616
6617             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
6618                 my ($dev_id, $size, $devname) = ($1, $2, $3);
6619                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
6620             } elsif ($line =~ m/^CTIME: /) {
6621                 # we correctly received the vma config, so we can disable
6622                 # the timeout now for disk allocation (set to 10 minutes, so
6623                 # that we always timeout if something goes wrong)
6624                 alarm(600);
6625                 &$print_devmap();
6626                 print $fifofh "done\n";
6627                 my $tmp = $oldtimeout || 0;
6628                 $oldtimeout = undef;
6629                 alarm($tmp);
6630                 close($fifofh);
6631             }
6632         };
6633
6634         print "restore vma archive: $dbg_cmdstring\n";
6635         run_command($commands, input => $input, outfunc => $parser, afterfork => $openfifo);
6636     };
6637     my $err = $@;
6638
6639     alarm($oldtimeout) if $oldtimeout;
6640
6641     my $vollist = [];
6642     foreach my $devname (keys %$devinfo) {
6643         my $volid = $devinfo->{$devname}->{volid};
6644         push @$vollist, $volid if $volid;
6645     }
6646
6647     PVE::Storage::deactivate_volumes($cfg, $vollist);
6648
6649     unlink $mapfifo;
6650
6651     if ($err) {
6652         rmtree $tmpdir;
6653         unlink $tmpfn;
6654
6655         foreach my $devname (keys %$devinfo) {
6656             my $volid = $devinfo->{$devname}->{volid};
6657             next if !$volid;
6658             eval {
6659                 if ($volid =~ m|^/|) {
6660                     unlink $volid || die 'unlink failed\n';
6661                 } else {
6662                     PVE::Storage::vdisk_free($cfg, $volid);
6663                 }
6664                 print STDERR "temporary volume '$volid' sucessfuly removed\n";
6665             };
6666             print STDERR "unable to cleanup '$volid' - $@" if $@;
6667         }
6668         die $err;
6669     }
6670
6671     rmtree $tmpdir;
6672
6673     rename($tmpfn, $conffile) ||
6674         die "unable to commit configuration file '$conffile'\n";
6675
6676     PVE::Cluster::cfs_update(); # make sure we read new file
6677
6678     eval { rescan($vmid, 1); };
6679     warn $@ if $@;
6680 }
6681
6682 sub restore_tar_archive {
6683     my ($archive, $vmid, $user, $opts) = @_;
6684
6685     if ($archive ne '-') {
6686         my $firstfile = tar_archive_read_firstfile($archive);
6687         die "ERROR: file '$archive' dos not lock like a QemuServer vzdump backup\n"
6688             if $firstfile ne 'qemu-server.conf';
6689     }
6690
6691     my $storecfg = PVE::Storage::config();
6692
6693     # destroy existing data - keep empty config
6694     my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
6695     destroy_vm($storecfg, $vmid, 1) if -f $vmcfgfn;
6696
6697     my $tocmd = "/usr/lib/qemu-server/qmextract";
6698
6699     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
6700     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
6701     $tocmd .= ' --prealloc' if $opts->{prealloc};
6702     $tocmd .= ' --info' if $opts->{info};
6703
6704     # tar option "xf" does not autodetect compression when read from STDIN,
6705     # so we pipe to zcat
6706     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
6707         PVE::Tools::shellquote("--to-command=$tocmd");
6708
6709     my $tmpdir = "/var/tmp/vzdumptmp$$";
6710     mkpath $tmpdir;
6711
6712     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
6713     local $ENV{VZDUMP_VMID} = $vmid;
6714     local $ENV{VZDUMP_USER} = $user;
6715
6716     my $conffile = PVE::QemuConfig->config_file($vmid);
6717     my $tmpfn = "$conffile.$$.tmp";
6718
6719     # disable interrupts (always do cleanups)
6720     local $SIG{INT} =
6721         local $SIG{TERM} =
6722         local $SIG{QUIT} =
6723         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
6724
6725     eval {
6726         # enable interrupts
6727         local $SIG{INT} =
6728             local $SIG{TERM} =
6729             local $SIG{QUIT} =
6730             local $SIG{HUP} =
6731             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6732
6733         if ($archive eq '-') {
6734             print "extracting archive from STDIN\n";
6735             run_command($cmd, input => "<&STDIN");
6736         } else {
6737             print "extracting archive '$archive'\n";
6738             run_command($cmd);
6739         }
6740
6741         return if $opts->{info};
6742
6743         # read new mapping
6744         my $map = {};
6745         my $statfile = "$tmpdir/qmrestore.stat";
6746         if (my $fd = IO::File->new($statfile, "r")) {
6747             while (defined (my $line = <$fd>)) {
6748                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
6749                     $map->{$1} = $2 if $1;
6750                 } else {
6751                     print STDERR "unable to parse line in statfile - $line\n";
6752                 }
6753             }
6754             $fd->close();
6755         }
6756
6757         my $confsrc = "$tmpdir/qemu-server.conf";
6758
6759         my $srcfd = new IO::File($confsrc, "r") ||
6760             die "unable to open file '$confsrc'\n";
6761
6762         my $outfd = new IO::File ($tmpfn, "w") ||
6763             die "unable to write config for VM $vmid\n";
6764
6765         my $cookie = { netcount => 0 };
6766         while (defined (my $line = <$srcfd>)) {
6767             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
6768         }
6769
6770         $srcfd->close();
6771         $outfd->close();
6772     };
6773     my $err = $@;
6774
6775     if ($err) {
6776
6777         unlink $tmpfn;
6778
6779         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
6780
6781         die $err;
6782     }
6783
6784     rmtree $tmpdir;
6785
6786     rename $tmpfn, $conffile ||
6787         die "unable to commit configuration file '$conffile'\n";
6788
6789     PVE::Cluster::cfs_update(); # make sure we read new file
6790
6791     eval { rescan($vmid, 1); };
6792     warn $@ if $@;
6793 };
6794
6795 sub foreach_storage_used_by_vm {
6796     my ($conf, $func) = @_;
6797
6798     my $sidhash = {};
6799
6800     foreach_drive($conf, sub {
6801         my ($ds, $drive) = @_;
6802         return if drive_is_cdrom($drive);
6803
6804         my $volid = $drive->{file};
6805
6806         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
6807         $sidhash->{$sid} = $sid if $sid;
6808     });
6809
6810     foreach my $sid (sort keys %$sidhash) {
6811         &$func($sid);
6812     }
6813 }
6814
6815 sub do_snapshots_with_qemu {
6816     my ($storecfg, $volid) = @_;
6817
6818     my $storage_name = PVE::Storage::parse_volume_id($volid);
6819     my $scfg = $storecfg->{ids}->{$storage_name};
6820
6821     if ($qemu_snap_storage->{$scfg->{type}} && !$scfg->{krbd}){
6822         return 1;
6823     }
6824
6825     if ($volid =~ m/\.(qcow2|qed)$/){
6826         return 1;
6827     }
6828
6829     return undef;
6830 }
6831
6832 sub qga_check_running {
6833     my ($vmid, $nowarn) = @_;
6834
6835     eval { vm_mon_cmd($vmid, "guest-ping", timeout => 3); };
6836     if ($@) {
6837         warn "Qemu Guest Agent is not running - $@" if !$nowarn;
6838         return 0;
6839     }
6840     return 1;
6841 }
6842
6843 sub template_create {
6844     my ($vmid, $conf, $disk) = @_;
6845
6846     my $storecfg = PVE::Storage::config();
6847
6848     foreach_drive($conf, sub {
6849         my ($ds, $drive) = @_;
6850
6851         return if drive_is_cdrom($drive);
6852         return if $disk && $ds ne $disk;
6853
6854         my $volid = $drive->{file};
6855         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
6856
6857         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
6858         $drive->{file} = $voliddst;
6859         $conf->{$ds} = print_drive($vmid, $drive);
6860         PVE::QemuConfig->write_config($vmid, $conf);
6861     });
6862 }
6863
6864 sub convert_iscsi_path {
6865     my ($path) = @_;
6866
6867     if ($path =~ m|^iscsi://([^/]+)/([^/]+)/(.+)$|) {
6868         my $portal = $1;
6869         my $target = $2;
6870         my $lun = $3;
6871
6872         my $initiator_name = get_initiator_name();
6873
6874         return "file.driver=iscsi,file.transport=tcp,file.initiator-name=$initiator_name,".
6875                "file.portal=$portal,file.target=$target,file.lun=$lun,driver=raw";
6876     }
6877
6878     die "cannot convert iscsi path '$path', unkown format\n";
6879 }
6880
6881 sub qemu_img_convert {
6882     my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized) = @_;
6883
6884     my $storecfg = PVE::Storage::config();
6885     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
6886     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
6887
6888     if ($src_storeid && $dst_storeid) {
6889
6890         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
6891
6892         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
6893         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6894
6895         my $src_format = qemu_img_format($src_scfg, $src_volname);
6896         my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
6897
6898         my $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
6899         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6900
6901         my $src_is_iscsi = ($src_path =~ m|^iscsi://|);
6902         my $dst_is_iscsi = ($dst_path =~ m|^iscsi://|);
6903
6904         my $cmd = [];
6905         push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
6906         push @$cmd, '-l', "snapshot.name=$snapname" if($snapname && $src_format eq "qcow2");
6907         push @$cmd, '-t', 'none' if $dst_scfg->{type} eq 'zfspool';
6908         push @$cmd, '-T', 'none' if $src_scfg->{type} eq 'zfspool';
6909
6910         if ($src_is_iscsi) {
6911             push @$cmd, '--image-opts';
6912             $src_path = convert_iscsi_path($src_path);
6913         } else {
6914             push @$cmd, '-f', $src_format;
6915         }
6916
6917         if ($dst_is_iscsi) {
6918             push @$cmd, '--target-image-opts';
6919             $dst_path = convert_iscsi_path($dst_path);
6920         } else {
6921             push @$cmd, '-O', $dst_format;
6922         }
6923
6924         push @$cmd, $src_path;
6925
6926         if (!$dst_is_iscsi && $is_zero_initialized) {
6927             push @$cmd, "zeroinit:$dst_path";
6928         } else {
6929             push @$cmd, $dst_path;
6930         }
6931
6932         my $parser = sub {
6933             my $line = shift;
6934             if($line =~ m/\((\S+)\/100\%\)/){
6935                 my $percent = $1;
6936                 my $transferred = int($size * $percent / 100);
6937                 my $remaining = $size - $transferred;
6938
6939                 print "transferred: $transferred bytes remaining: $remaining bytes total: $size bytes progression: $percent %\n";
6940             }
6941
6942         };
6943
6944         eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
6945         my $err = $@;
6946         die "copy failed: $err" if $err;
6947     }
6948 }
6949
6950 sub qemu_img_format {
6951     my ($scfg, $volname) = @_;
6952
6953     if ($scfg->{path} && $volname =~ m/\.($QEMU_FORMAT_RE)$/) {
6954         return $1;
6955     } else {
6956         return "raw";
6957     }
6958 }
6959
6960 sub qemu_drive_mirror {
6961     my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $skipcomplete, $qga, $bwlimit) = @_;
6962
6963     $jobs = {} if !$jobs;
6964
6965     my $qemu_target;
6966     my $format;
6967     $jobs->{"drive-$drive"} = {};
6968
6969     if ($dst_volid =~ /^nbd:/) {
6970         $qemu_target = $dst_volid;
6971         $format = "nbd";
6972     } else {
6973         my $storecfg = PVE::Storage::config();
6974         my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
6975
6976         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6977
6978         $format = qemu_img_format($dst_scfg, $dst_volname);
6979
6980         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6981
6982         $qemu_target = $is_zero_initialized ? "zeroinit:$dst_path" : $dst_path;
6983     }
6984
6985     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $qemu_target };
6986     $opts->{format} = $format if $format;
6987
6988     if (defined($bwlimit)) {
6989         $opts->{speed} = $bwlimit * 1024;
6990         print "drive mirror is starting for drive-$drive with bandwidth limit: ${bwlimit} KB/s\n";
6991     } else {
6992         print "drive mirror is starting for drive-$drive\n";
6993     }
6994
6995     # if a job already runs for this device we get an error, catch it for cleanup
6996     eval { vm_mon_cmd($vmid, "drive-mirror", %$opts); };
6997     if (my $err = $@) {
6998         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6999         warn "$@\n" if $@;
7000         die "mirroring error: $err\n";
7001     }
7002
7003     qemu_drive_mirror_monitor ($vmid, $vmiddst, $jobs, $skipcomplete, $qga);
7004 }
7005
7006 sub qemu_drive_mirror_monitor {
7007     my ($vmid, $vmiddst, $jobs, $skipcomplete, $qga) = @_;
7008
7009     eval {
7010         my $err_complete = 0;
7011
7012         while (1) {
7013             die "storage migration timed out\n" if $err_complete > 300;
7014
7015             my $stats = vm_mon_cmd($vmid, "query-block-jobs");
7016
7017             my $running_mirror_jobs = {};
7018             foreach my $stat (@$stats) {
7019                 next if $stat->{type} ne 'mirror';
7020                 $running_mirror_jobs->{$stat->{device}} = $stat;
7021             }
7022
7023             my $readycounter = 0;
7024
7025             foreach my $job (keys %$jobs) {
7026
7027                 if(defined($jobs->{$job}->{complete}) && !defined($running_mirror_jobs->{$job})) {
7028                     print "$job : finished\n";
7029                     delete $jobs->{$job};
7030                     next;
7031                 }
7032
7033                 die "$job: mirroring has been cancelled\n" if !defined($running_mirror_jobs->{$job});
7034
7035                 my $busy = $running_mirror_jobs->{$job}->{busy};
7036                 my $ready = $running_mirror_jobs->{$job}->{ready};
7037                 if (my $total = $running_mirror_jobs->{$job}->{len}) {
7038                     my $transferred = $running_mirror_jobs->{$job}->{offset} || 0;
7039                     my $remaining = $total - $transferred;
7040                     my $percent = sprintf "%.2f", ($transferred * 100 / $total);
7041
7042                     print "$job: transferred: $transferred bytes remaining: $remaining bytes total: $total bytes progression: $percent % busy: $busy ready: $ready \n";
7043                 }
7044
7045                 $readycounter++ if $running_mirror_jobs->{$job}->{ready};
7046             }
7047
7048             last if scalar(keys %$jobs) == 0;
7049
7050             if ($readycounter == scalar(keys %$jobs)) {
7051                 print "all mirroring jobs are ready \n";
7052                 last if $skipcomplete; #do the complete later
7053
7054                 if ($vmiddst && $vmiddst != $vmid) {
7055                     my $agent_running = $qga && qga_check_running($vmid);
7056                     if ($agent_running) {
7057                         print "freeze filesystem\n";
7058                         eval { PVE::QemuServer::vm_mon_cmd($vmid, "guest-fsfreeze-freeze"); };
7059                     } else {
7060                         print "suspend vm\n";
7061                         eval { PVE::QemuServer::vm_suspend($vmid, 1); };
7062                     }
7063
7064                     # if we clone a disk for a new target vm, we don't switch the disk
7065                     PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs);
7066
7067                     if ($agent_running) {
7068                         print "unfreeze filesystem\n";
7069                         eval { PVE::QemuServer::vm_mon_cmd($vmid, "guest-fsfreeze-thaw"); };
7070                     } else {
7071                         print "resume vm\n";
7072                         eval {  PVE::QemuServer::vm_resume($vmid, 1, 1); };
7073                     }
7074
7075                     last;
7076                 } else {
7077
7078                     foreach my $job (keys %$jobs) {
7079                         # try to switch the disk if source and destination are on the same guest
7080                         print "$job: Completing block job...\n";
7081
7082                         eval { vm_mon_cmd($vmid, "block-job-complete", device => $job) };
7083                         if ($@ =~ m/cannot be completed/) {
7084                             print "$job: Block job cannot be completed, try again.\n";
7085                             $err_complete++;
7086                         }else {
7087                             print "$job: Completed successfully.\n";
7088                             $jobs->{$job}->{complete} = 1;
7089                         }
7090                     }
7091                 }
7092             }
7093             sleep 1;
7094         }
7095     };
7096     my $err = $@;
7097
7098     if ($err) {
7099         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
7100         die "mirroring error: $err";
7101     }
7102
7103 }
7104
7105 sub qemu_blockjobs_cancel {
7106     my ($vmid, $jobs) = @_;
7107
7108     foreach my $job (keys %$jobs) {
7109         print "$job: Cancelling block job\n";
7110         eval { vm_mon_cmd($vmid, "block-job-cancel", device => $job); };
7111         $jobs->{$job}->{cancel} = 1;
7112     }
7113
7114     while (1) {
7115         my $stats = vm_mon_cmd($vmid, "query-block-jobs");
7116
7117         my $running_jobs = {};
7118         foreach my $stat (@$stats) {
7119             $running_jobs->{$stat->{device}} = $stat;
7120         }
7121
7122         foreach my $job (keys %$jobs) {
7123
7124             if (defined($jobs->{$job}->{cancel}) && !defined($running_jobs->{$job})) {
7125                 print "$job: Done.\n";
7126                 delete $jobs->{$job};
7127             }
7128         }
7129
7130         last if scalar(keys %$jobs) == 0;
7131
7132         sleep 1;
7133     }
7134 }
7135
7136 sub clone_disk {
7137     my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
7138         $newvmid, $storage, $format, $full, $newvollist, $jobs, $skipcomplete, $qga, $bwlimit) = @_;
7139
7140     my $newvolid;
7141
7142     if (!$full) {
7143         print "create linked clone of drive $drivename ($drive->{file})\n";
7144         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
7145         push @$newvollist, $newvolid;
7146     } else {
7147
7148         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
7149         $storeid = $storage if $storage;
7150
7151         my $dst_format = resolve_dst_disk_format($storecfg, $storeid, $volname, $format);
7152         my ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 3);
7153
7154         print "create full clone of drive $drivename ($drive->{file})\n";
7155         my $name = undef;
7156         $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $newvmid, $dst_format, $name, ($size/1024));
7157         push @$newvollist, $newvolid;
7158
7159         PVE::Storage::activate_volumes($storecfg, [$newvolid]);
7160
7161         my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
7162         if (!$running || $snapname) {
7163             # TODO: handle bwlimits
7164             qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit);
7165         } else {
7166
7167             my $kvmver = get_running_qemu_version ($vmid);
7168             if (!qemu_machine_feature_enabled (undef, $kvmver, 2, 7)) {
7169                 die "drive-mirror with iothread requires qemu version 2.7 or higher\n"
7170                     if $drive->{iothread};
7171             }
7172
7173             qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid, $sparseinit, $jobs, $skipcomplete, $qga, $bwlimit);
7174         }
7175     }
7176
7177     my ($size) = PVE::Storage::volume_size_info($storecfg, $newvolid, 3);
7178
7179     my $disk = $drive;
7180     $disk->{format} = undef;
7181     $disk->{file} = $newvolid;
7182     $disk->{size} = $size;
7183
7184     return $disk;
7185 }
7186
7187 # this only works if VM is running
7188 sub get_current_qemu_machine {
7189     my ($vmid) = @_;
7190
7191     my $cmd = { execute => 'query-machines', arguments => {} };
7192     my $res = vm_qmp_command($vmid, $cmd);
7193
7194     my ($current, $default);
7195     foreach my $e (@$res) {
7196         $default = $e->{name} if $e->{'is-default'};
7197         $current = $e->{name} if $e->{'is-current'};
7198     }
7199
7200     # fallback to the default machine if current is not supported by qemu
7201     return $current || $default || 'pc';
7202 }
7203
7204 sub get_running_qemu_version {
7205     my ($vmid) = @_;
7206     my $cmd = { execute => 'query-version', arguments => {} };
7207     my $res = vm_qmp_command($vmid, $cmd);
7208     return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
7209 }
7210
7211 sub qemu_machine_feature_enabled {
7212     my ($machine, $kvmver, $version_major, $version_minor) = @_;
7213
7214     my $current_major;
7215     my $current_minor;
7216
7217     if ($machine && $machine =~ m/^((?:pc(-i440fx|-q35)?|virt)-(\d+)\.(\d+))/) {
7218
7219         $current_major = $3;
7220         $current_minor = $4;
7221
7222     } elsif ($kvmver =~ m/^(\d+)\.(\d+)/) {
7223
7224         $current_major = $1;
7225         $current_minor = $2;
7226     }
7227
7228     return 1 if $current_major > $version_major ||
7229                 ($current_major == $version_major &&
7230                  $current_minor >= $version_minor);
7231 }
7232
7233 sub qemu_machine_pxe {
7234     my ($vmid, $conf) = @_;
7235
7236     my $machine =  PVE::QemuServer::get_current_qemu_machine($vmid);
7237
7238     if ($conf->{machine} && $conf->{machine} =~ m/\.pxe$/) {
7239         $machine .= '.pxe';
7240     }
7241
7242     return $machine;
7243 }
7244
7245 sub qemu_use_old_bios_files {
7246     my ($machine_type) = @_;
7247
7248     return if !$machine_type;
7249
7250     my $use_old_bios_files = undef;
7251
7252     if ($machine_type =~ m/^(\S+)\.pxe$/) {
7253         $machine_type = $1;
7254         $use_old_bios_files = 1;
7255     } else {
7256         my $kvmver = kvm_user_version();
7257         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
7258         # load new efi bios files on migration. So this hack is required to allow
7259         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
7260         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
7261         $use_old_bios_files = !qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 4);
7262     }
7263
7264     return ($use_old_bios_files, $machine_type);
7265 }
7266
7267 sub create_efidisk($$$$$) {
7268     my ($storecfg, $storeid, $vmid, $fmt, $arch) = @_;
7269
7270     my (undef, $ovmf_vars) = get_ovmf_files($arch);
7271     die "EFI vars default image not found\n" if ! -f $ovmf_vars;
7272
7273     my $vars_size = PVE::Tools::convert_size(-s $ovmf_vars, 'b' => 'kb');
7274     my $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $fmt, undef, $vars_size);
7275     PVE::Storage::activate_volumes($storecfg, [$volid]);
7276
7277     my $path = PVE::Storage::path($storecfg, $volid);
7278     eval {
7279         run_command(['/usr/bin/qemu-img', 'convert', '-n', '-f', 'raw', '-O', $fmt, $ovmf_vars, $path]);
7280     };
7281     die "Copying EFI vars image failed: $@" if $@;
7282
7283     return ($volid, $vars_size);
7284 }
7285
7286 sub vm_iothreads_list {
7287     my ($vmid) = @_;
7288
7289     my $res = vm_mon_cmd($vmid, 'query-iothreads');
7290
7291     my $iothreads = {};
7292     foreach my $iothread (@$res) {
7293         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
7294     }
7295
7296     return $iothreads;
7297 }
7298
7299 sub scsihw_infos {
7300     my ($conf, $drive) = @_;
7301
7302     my $maxdev = 0;
7303
7304     if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)) {
7305         $maxdev = 7;
7306     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
7307         $maxdev = 1;
7308     } else {
7309         $maxdev = 256;
7310     }
7311
7312     my $controller = int($drive->{index} / $maxdev);
7313     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single') ? "virtioscsi" : "scsihw";
7314
7315     return ($maxdev, $controller, $controller_prefix);
7316 }
7317
7318 sub add_hyperv_enlightenments {
7319     my ($cpuFlags, $winversion, $machine_type, $kvmver, $bios, $gpu_passthrough, $hv_vendor_id) = @_;
7320
7321     return if $winversion < 6;
7322     return if $bios && $bios eq 'ovmf' && $winversion < 8;
7323
7324     if ($gpu_passthrough || defined($hv_vendor_id)) {
7325         $hv_vendor_id //= 'proxmox';
7326         push @$cpuFlags , "hv_vendor_id=$hv_vendor_id";
7327     }
7328
7329     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
7330         push @$cpuFlags , 'hv_spinlocks=0x1fff';
7331         push @$cpuFlags , 'hv_vapic';
7332         push @$cpuFlags , 'hv_time';
7333     } else {
7334         push @$cpuFlags , 'hv_spinlocks=0xffff';
7335     }
7336
7337     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 6)) {
7338         push @$cpuFlags , 'hv_reset';
7339         push @$cpuFlags , 'hv_vpindex';
7340         push @$cpuFlags , 'hv_runtime';
7341     }
7342
7343     if ($winversion >= 7) {
7344         push @$cpuFlags , 'hv_relaxed';
7345
7346         if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 12)) {
7347             push @$cpuFlags , 'hv_synic';
7348             push @$cpuFlags , 'hv_stimer';
7349         }
7350
7351         if (qemu_machine_feature_enabled ($machine_type, $kvmver, 3, 1)) {
7352             push @$cpuFlags , 'hv_ipi';
7353         }
7354     }
7355 }
7356
7357 sub windows_version {
7358     my ($ostype) = @_;
7359
7360     return 0 if !$ostype;
7361
7362     my $winversion = 0;
7363
7364     if($ostype eq 'wxp' || $ostype eq 'w2k3' || $ostype eq 'w2k') {
7365         $winversion = 5;
7366     } elsif($ostype eq 'w2k8' || $ostype eq 'wvista') {
7367         $winversion = 6;
7368     } elsif ($ostype =~ m/^win(\d+)$/) {
7369         $winversion = $1;
7370     }
7371
7372     return $winversion;
7373 }
7374
7375 sub resolve_dst_disk_format {
7376         my ($storecfg, $storeid, $src_volname, $format) = @_;
7377         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
7378
7379         if (!$format) {
7380             # if no target format is specified, use the source disk format as hint
7381             if ($src_volname) {
7382                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
7383                 $format = qemu_img_format($scfg, $src_volname);
7384             } else {
7385                 return $defFormat;
7386             }
7387         }
7388
7389         # test if requested format is supported - else use default
7390         my $supported = grep { $_ eq $format } @$validFormats;
7391         $format = $defFormat if !$supported;
7392         return $format;
7393 }
7394
7395 sub resolve_first_disk {
7396     my $conf = shift;
7397     my @disks = PVE::QemuServer::valid_drive_names();
7398     my $firstdisk;
7399     foreach my $ds (reverse @disks) {
7400         next if !$conf->{$ds};
7401         my $disk = PVE::QemuServer::parse_drive($ds, $conf->{$ds});
7402         next if PVE::QemuServer::drive_is_cdrom($disk);
7403         $firstdisk = $ds;
7404     }
7405     return $firstdisk;
7406 }
7407
7408 sub generate_uuid {
7409     my ($uuid, $uuid_str);
7410     UUID::generate($uuid);
7411     UUID::unparse($uuid, $uuid_str);
7412     return $uuid_str;
7413 }
7414
7415 sub generate_smbios1_uuid {
7416     return "uuid=".generate_uuid();
7417 }
7418
7419 sub nbd_stop {
7420     my ($vmid) = @_;
7421
7422     vm_mon_cmd($vmid, 'nbd-server-stop');
7423 }
7424
7425 sub create_reboot_request {
7426     my ($vmid) = @_;
7427     open(my $fh, '>', "/run/qemu-server/$vmid.reboot")
7428         or die "failed to create reboot trigger file: $!\n";
7429     close($fh);
7430 }
7431
7432 sub clear_reboot_request {
7433     my ($vmid) = @_;
7434     my $path = "/run/qemu-server/$vmid.reboot";
7435     my $res = 0;
7436
7437     $res = unlink($path);
7438     die "could not remove reboot request for $vmid: $!"
7439         if !$res && $! != POSIX::ENOENT;
7440
7441     return $res;
7442 }
7443
7444 # bash completion helper
7445
7446 sub complete_backup_archives {
7447     my ($cmdname, $pname, $cvalue) = @_;
7448
7449     my $cfg = PVE::Storage::config();
7450
7451     my $storeid;
7452
7453     if ($cvalue =~ m/^([^:]+):/) {
7454         $storeid = $1;
7455     }
7456
7457     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
7458
7459     my $res = [];
7460     foreach my $id (keys %$data) {
7461         foreach my $item (@{$data->{$id}}) {
7462             next if $item->{format} !~ m/^vma\.(gz|lzo)$/;
7463             push @$res, $item->{volid} if defined($item->{volid});
7464         }
7465     }
7466
7467     return $res;
7468 }
7469
7470 my $complete_vmid_full = sub {
7471     my ($running) = @_;
7472
7473     my $idlist = vmstatus();
7474
7475     my $res = [];
7476
7477     foreach my $id (keys %$idlist) {
7478         my $d = $idlist->{$id};
7479         if (defined($running)) {
7480             next if $d->{template};
7481             next if $running && $d->{status} ne 'running';
7482             next if !$running && $d->{status} eq 'running';
7483         }
7484         push @$res, $id;
7485
7486     }
7487     return $res;
7488 };
7489
7490 sub complete_vmid {
7491     return &$complete_vmid_full();
7492 }
7493
7494 sub complete_vmid_stopped {
7495     return &$complete_vmid_full(0);
7496 }
7497
7498 sub complete_vmid_running {
7499     return &$complete_vmid_full(1);
7500 }
7501
7502 sub complete_storage {
7503
7504     my $cfg = PVE::Storage::config();
7505     my $ids = $cfg->{ids};
7506
7507     my $res = [];
7508     foreach my $sid (keys %$ids) {
7509         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
7510         next if !$ids->{$sid}->{content}->{images};
7511         push @$res, $sid;
7512     }
7513
7514     return $res;
7515 }
7516
7517 1;