PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5
   6 use POSIX;
   7 use IO::Handle;
   8 use IO::Select;
   9 use IO::File;
  10 use IO::Dir;
  11 use IO::Socket::UNIX;
  12 use File::Basename;
  13 use File::Path;
  14 use File::stat;
  15 use Getopt::Long;
  16 use Digest::SHA;
  17 use Fcntl ':flock';
  18 use Cwd 'abs_path';
  19 use IPC::Open3;
  20 use JSON;
  21 use Fcntl;
  22 use PVE::SafeSyslog;
  23 use Storable qw(dclone);
  24 use MIME::Base64;
  25 use PVE::Exception qw(raise raise_param_exc);
  26 use PVE::Storage;
  27 use PVE::Tools qw(run_command lock_file lock_file_full file_read_firstline dir_glob_foreach $IPV6RE);
  28 use PVE::JSONSchema qw(get_standard_option);
  29 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file cfs_lock_file);
  30 use PVE::INotify;
  31 use PVE::ProcFSTools;
  32 use PVE::QemuConfig;
  33 use PVE::QMPClient;
  34 use PVE::RPCEnvironment;
  35 use PVE::GuestHelpers;
  36 use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr print_pcie_root_port);
  37 use PVE::QemuServer::Memory;
  38 use PVE::QemuServer::USB qw(parse_usb_device);
  39 use PVE::QemuServer::Cloudinit;
  40 use PVE::SysFSTools;
  41 use PVE::Systemd;
  42 use Time::HiRes qw(gettimeofday);
  43 use File::Copy qw(copy);
  44 use URI::Escape;
  45
  46 my $EDK2_FW_BASE = '/usr/share/pve-edk2-firmware/';
  47 my $OVMF = {
  48     x86_64 => [
  49         "$EDK2_FW_BASE/OVMF_CODE.fd",
  50         "$EDK2_FW_BASE/OVMF_VARS.fd"
  51     ],
  52     aarch64 => [
  53         "$EDK2_FW_BASE/AAVMF_CODE.fd",
  54         "$EDK2_FW_BASE/AAVMF_VARS.fd"
  55     ],
  56 };
  57
  58 my $qemu_snap_storage = { rbd => 1 };
  59
  60 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  61
  62 my $QEMU_FORMAT_RE = qr/raw|cow|qcow|qcow2|qed|vmdk|cloop/;
  63
  64 # Note about locking: we use flock on the config file protect
  65 # against concurent actions.
  66 # Aditionaly, we have a 'lock' setting in the config file. This
  67 # can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
  68 # allowed when such lock is set. But you can ignore this kind of
  69 # lock with the --skiplock flag.
  70
  71 cfs_register_file('/qemu-server/',
  72                   \&parse_vm_config,
  73                   \&write_vm_config);
  74
  75 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
  76     description => "Some command save/restore state from this location.",
  77     type => 'string',
  78     maxLength => 128,
  79     optional => 1,
  80 });
  81
  82 PVE::JSONSchema::register_standard_option('pve-qm-image-format', {
  83     type => 'string',
  84     enum => [qw(raw cow qcow qed qcow2 vmdk cloop)],
  85     description => "The drive's backing file's data format.",
  86     optional => 1,
  87 });
  88
  89 PVE::JSONSchema::register_standard_option('pve-qemu-machine', {
  90         description => "Specifies the Qemu machine type.",
  91         type => 'string',
  92         pattern => '(pc|pc(-i440fx)?-\d+\.\d+(\.pxe)?|q35|pc-q35-\d+\.\d+(\.pxe)?|virt(?:-\d+\.\d+)?)',
  93         maxLength => 40,
  94         optional => 1,
  95 });
  96
  97 #no warnings 'redefine';
  98
  99 sub cgroups_write {
 100    my ($controller, $vmid, $option, $value) = @_;
 101
 102    my $path = "/sys/fs/cgroup/$controller/qemu.slice/$vmid.scope/$option";
 103    PVE::ProcFSTools::write_proc_entry($path, $value);
 104
 105 }
 106
 107 my $nodename = PVE::INotify::nodename();
 108
 109 mkdir "/etc/pve/nodes/$nodename";
 110 my $confdir = "/etc/pve/nodes/$nodename/qemu-server";
 111 mkdir $confdir;
 112
 113 my $var_run_tmpdir = "/var/run/qemu-server";
 114 mkdir $var_run_tmpdir;
 115
 116 my $lock_dir = "/var/lock/qemu-server";
 117 mkdir $lock_dir;
 118
 119 my $cpu_vendor_list = {
 120     # Intel CPUs
 121     486 => 'GenuineIntel',
 122     pentium => 'GenuineIntel',
 123     pentium2  => 'GenuineIntel',
 124     pentium3  => 'GenuineIntel',
 125     coreduo => 'GenuineIntel',
 126     core2duo => 'GenuineIntel',
 127     Conroe  => 'GenuineIntel',
 128     Penryn  => 'GenuineIntel',
 129     Nehalem  => 'GenuineIntel',
 130     'Nehalem-IBRS'  => 'GenuineIntel',
 131     Westmere => 'GenuineIntel',
 132     'Westmere-IBRS' => 'GenuineIntel',
 133     SandyBridge => 'GenuineIntel',
 134     'SandyBridge-IBRS' => 'GenuineIntel',
 135     IvyBridge => 'GenuineIntel',
 136     'IvyBridge-IBRS' => 'GenuineIntel',
 137     Haswell => 'GenuineIntel',
 138     'Haswell-IBRS' => 'GenuineIntel',
 139     'Haswell-noTSX' => 'GenuineIntel',
 140     'Haswell-noTSX-IBRS' => 'GenuineIntel',
 141     Broadwell => 'GenuineIntel',
 142     'Broadwell-IBRS' => 'GenuineIntel',
 143     'Broadwell-noTSX' => 'GenuineIntel',
 144     'Broadwell-noTSX-IBRS' => 'GenuineIntel',
 145     'Skylake-Client' => 'GenuineIntel',
 146     'Skylake-Client-IBRS' => 'GenuineIntel',
 147     'Skylake-Server' => 'GenuineIntel',
 148     'Skylake-Server-IBRS' => 'GenuineIntel',
 149     'Cascadelake-Server' => 'GenuineIntel',
 150     KnightsMill => 'GenuineIntel',
 151
 152
 153     # AMD CPUs
 154     athlon => 'AuthenticAMD',
 155     phenom  => 'AuthenticAMD',
 156     Opteron_G1  => 'AuthenticAMD',
 157     Opteron_G2  => 'AuthenticAMD',
 158     Opteron_G3  => 'AuthenticAMD',
 159     Opteron_G4  => 'AuthenticAMD',
 160     Opteron_G5  => 'AuthenticAMD',
 161     EPYC => 'AuthenticAMD',
 162     'EPYC-IBPB' => 'AuthenticAMD',
 163
 164     # generic types, use vendor from host node
 165     host => 'default',
 166     kvm32 => 'default',
 167     kvm64 => 'default',
 168     qemu32 => 'default',
 169     qemu64 => 'default',
 170     max => 'default',
 171 };
 172
 173 my @supported_cpu_flags = (
 174     'pcid',
 175     'spec-ctrl',
 176     'ibpb',
 177     'ssbd',
 178     'virt-ssbd',
 179     'amd-ssbd',
 180     'amd-no-ssb',
 181     'pdpe1gb',
 182     'md-clear',
 183     'hv-tlbflush',
 184     'hv-evmcs',
 185     'aes'
 186 );
 187 my $cpu_flag = qr/[+-](@{[join('|', @supported_cpu_flags)]})/;
 188
 189 my $cpu_fmt = {
 190     cputype => {
 191         description => "Emulated CPU type.",
 192         type => 'string',
 193         enum => [ sort { "\L$a" cmp "\L$b" } keys %$cpu_vendor_list ],
 194         default => 'kvm64',
 195         default_key => 1,
 196     },
 197     hidden => {
 198         description => "Do not identify as a KVM virtual machine.",
 199         type => 'boolean',
 200         optional => 1,
 201         default => 0
 202     },
 203     'hv-vendor-id' => {
 204         type => 'string',
 205         pattern => qr/[a-zA-Z0-9]{1,12}/,
 206         format_description => 'vendor-id',
 207         description => 'The Hyper-V vendor ID. Some drivers or programs inside Windows guests need a specific ID.',
 208         optional => 1,
 209     },
 210     flags => {
 211         description => "List of additional CPU flags separated by ';'."
 212                      . " Use '+FLAG' to enable, '-FLAG' to disable a flag."
 213                      . " Currently supported flags: @{[join(', ', @supported_cpu_flags)]}.",
 214         format_description => '+FLAG[;-FLAG...]',
 215         type => 'string',
 216         pattern => qr/$cpu_flag(;$cpu_flag)*/,
 217         optional => 1,
 218     },
 219 };
 220
 221 my $watchdog_fmt = {
 222     model => {
 223         default_key => 1,
 224         type => 'string',
 225         enum => [qw(i6300esb ib700)],
 226         description => "Watchdog type to emulate.",
 227         default => 'i6300esb',
 228         optional => 1,
 229     },
 230     action => {
 231         type => 'string',
 232         enum => [qw(reset shutdown poweroff pause debug none)],
 233         description => "The action to perform if after activation the guest fails to poll the watchdog in time.",
 234         optional => 1,
 235     },
 236 };
 237 PVE::JSONSchema::register_format('pve-qm-watchdog', $watchdog_fmt);
 238
 239 my $agent_fmt = {
 240     enabled => {
 241         description => "Enable/disable Qemu GuestAgent.",
 242         type => 'boolean',
 243         default => 0,
 244         default_key => 1,
 245     },
 246     fstrim_cloned_disks => {
 247         description => "Run fstrim after cloning/moving a disk.",
 248         type => 'boolean',
 249         optional => 1,
 250         default => 0
 251     },
 252 };
 253
 254 my $vga_fmt = {
 255     type => {
 256         description => "Select the VGA type.",
 257         type => 'string',
 258         default => 'std',
 259         optional => 1,
 260         default_key => 1,
 261         enum => [qw(cirrus qxl qxl2 qxl3 qxl4 none serial0 serial1 serial2 serial3 std virtio vmware)],
 262     },
 263     memory => {
 264         description => "Sets the VGA memory (in MiB). Has no effect with serial display.",
 265         type => 'integer',
 266         optional => 1,
 267         minimum => 4,
 268         maximum => 512,
 269     },
 270 };
 271
 272 my $ivshmem_fmt = {
 273     size => {
 274         type => 'integer',
 275         minimum => 1,
 276         description => "The size of the file in MB.",
 277     },
 278     name => {
 279         type => 'string',
 280         pattern => '[a-zA-Z0-9\-]+',
 281         optional => 1,
 282         format_description => 'string',
 283         description => "The name of the file. Will be prefixed with 'pve-shm-'. Default is the VMID. Will be deleted when the VM is stopped.",
 284     },
 285 };
 286
 287 my $audio_fmt = {
 288     device => {
 289         type => 'string',
 290         enum => [qw(ich9-intel-hda intel-hda AC97)],
 291         description =>  "Configure an audio device."
 292     },
 293     driver =>  {
 294         type => 'string',
 295         enum => ['spice'],
 296         default => 'spice',
 297         optional => 1,
 298         description => "Driver backend for the audio device."
 299     },
 300 };
 301
 302 my $spice_enhancements_fmt = {
 303     foldersharing => {
 304         type => 'boolean',
 305         optional => 1,
 306         default => '0',
 307         description =>  "Enable folder sharing via SPICE. Needs Spice-WebDAV daemon installed in the VM."
 308     },
 309     videostreaming =>  {
 310         type => 'string',
 311         enum => ['off', 'all', 'filter'],
 312         default => 'off',
 313         optional => 1,
 314         description => "Enable video streaming. Uses compression for detected video streams."
 315     },
 316 };
 317
 318 my $confdesc = {
 319     onboot => {
 320         optional => 1,
 321         type => 'boolean',
 322         description => "Specifies whether a VM will be started during system bootup.",
 323         default => 0,
 324     },
 325     autostart => {
 326         optional => 1,
 327         type => 'boolean',
 328         description => "Automatic restart after crash (currently ignored).",
 329         default => 0,
 330     },
 331     hotplug => {
 332         optional => 1,
 333         type => 'string', format => 'pve-hotplug-features',
 334         description => "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
 335         default => 'network,disk,usb',
 336     },
 337     reboot => {
 338         optional => 1,
 339         type => 'boolean',
 340         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 341         default => 1,
 342     },
 343     lock => {
 344         optional => 1,
 345         type => 'string',
 346         description => "Lock/unlock the VM.",
 347         enum => [qw(backup clone create migrate rollback snapshot snapshot-delete suspending suspended)],
 348     },
 349     cpulimit => {
 350         optional => 1,
 351         type => 'number',
 352         description => "Limit of CPU usage.",
 353         verbose_description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
 354         minimum => 0,
 355         maximum => 128,
 356         default => 0,
 357     },
 358     cpuunits => {
 359         optional => 1,
 360         type => 'integer',
 361         description => "CPU weight for a VM.",
 362         verbose_description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.",
 363         minimum => 2,
 364         maximum => 262144,
 365         default => 1024,
 366     },
 367     memory => {
 368         optional => 1,
 369         type => 'integer',
 370         description => "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
 371         minimum => 16,
 372         default => 512,
 373     },
 374     balloon => {
 375         optional => 1,
 376         type => 'integer',
 377         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 378         minimum => 0,
 379     },
 380     shares => {
 381         optional => 1,
 382         type => 'integer',
 383         description => "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning. Auto-ballooning is done by pvestatd.",
 384         minimum => 0,
 385         maximum => 50000,
 386         default => 1000,
 387     },
 388     keyboard => {
 389         optional => 1,
 390         type => 'string',
 391         description => "Keybord layout for vnc server. Default is read from the '/etc/pve/datacenter.cfg' configuration file.".
 392                        "It should not be necessary to set it.",
 393         enum => PVE::Tools::kvmkeymaplist(),
 394         default => undef,
 395     },
 396     name => {
 397         optional => 1,
 398         type => 'string', format => 'dns-name',
 399         description => "Set a name for the VM. Only used on the configuration web interface.",
 400     },
 401     scsihw => {
 402         optional => 1,
 403         type => 'string',
 404         description => "SCSI controller model",
 405         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 406         default => 'lsi',
 407     },
 408     description => {
 409         optional => 1,
 410         type => 'string',
 411         description => "Description for the VM. Only used on the configuration web interface. This is saved as comment inside the configuration file.",
 412     },
 413     ostype => {
 414         optional => 1,
 415         type => 'string',
 416         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 l24 l26 solaris)],
 417         description => "Specify guest operating system.",
 418         verbose_description => <<EODESC,
 419 Specify guest operating system. This is used to enable special
 420 optimization/features for specific operating systems:
 421
 422 [horizontal]
 423 other;; unspecified OS
 424 wxp;; Microsoft Windows XP
 425 w2k;; Microsoft Windows 2000
 426 w2k3;; Microsoft Windows 2003
 427 w2k8;; Microsoft Windows 2008
 428 wvista;; Microsoft Windows Vista
 429 win7;; Microsoft Windows 7
 430 win8;; Microsoft Windows 8/2012/2012r2
 431 win10;; Microsoft Windows 10/2016
 432 l24;; Linux 2.4 Kernel
 433 l26;; Linux 2.6/3.X Kernel
 434 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
 435 EODESC
 436     },
 437     boot => {
 438         optional => 1,
 439         type => 'string',
 440         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n).",
 441         pattern => '[acdn]{1,4}',
 442         default => 'cdn',
 443     },
 444     bootdisk => {
 445         optional => 1,
 446         type => 'string', format => 'pve-qm-bootdisk',
 447         description => "Enable booting from specified disk.",
 448         pattern => '(ide|sata|scsi|virtio)\d+',
 449     },
 450     smp => {
 451         optional => 1,
 452         type => 'integer',
 453         description => "The number of CPUs. Please use option -sockets instead.",
 454         minimum => 1,
 455         default => 1,
 456     },
 457     sockets => {
 458         optional => 1,
 459         type => 'integer',
 460         description => "The number of CPU sockets.",
 461         minimum => 1,
 462         default => 1,
 463     },
 464     cores => {
 465         optional => 1,
 466         type => 'integer',
 467         description => "The number of cores per socket.",
 468         minimum => 1,
 469         default => 1,
 470     },
 471     numa => {
 472         optional => 1,
 473         type => 'boolean',
 474         description => "Enable/disable NUMA.",
 475         default => 0,
 476     },
 477     hugepages => {
 478         optional => 1,
 479         type => 'string',
 480         description => "Enable/disable hugepages memory.",
 481         enum => [qw(any 2 1024)],
 482     },
 483     vcpus => {
 484         optional => 1,
 485         type => 'integer',
 486         description => "Number of hotplugged vcpus.",
 487         minimum => 1,
 488         default => 0,
 489     },
 490     acpi => {
 491         optional => 1,
 492         type => 'boolean',
 493         description => "Enable/disable ACPI.",
 494         default => 1,
 495     },
 496     agent => {
 497         optional => 1,
 498         description => "Enable/disable Qemu GuestAgent and its properties.",
 499         type => 'string',
 500         format => $agent_fmt,
 501     },
 502     kvm => {
 503         optional => 1,
 504         type => 'boolean',
 505         description => "Enable/disable KVM hardware virtualization.",
 506         default => 1,
 507     },
 508     tdf => {
 509         optional => 1,
 510         type => 'boolean',
 511         description => "Enable/disable time drift fix.",
 512         default => 0,
 513     },
 514     localtime => {
 515         optional => 1,
 516         type => 'boolean',
 517         description => "Set the real time clock to local time. This is enabled by default if ostype indicates a Microsoft OS.",
 518     },
 519     freeze => {
 520         optional => 1,
 521         type => 'boolean',
 522         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 523     },
 524     vga => {
 525         optional => 1,
 526         type => 'string', format => $vga_fmt,
 527         description => "Configure the VGA hardware.",
 528         verbose_description => "Configure the VGA Hardware. If you want to use ".
 529             "high resolution modes (>= 1280x1024x16) you may need to increase " .
 530             "the vga memory option. Since QEMU 2.9 the default VGA display type " .
 531             "is 'std' for all OS types besides some Windows versions (XP and " .
 532             "older) which use 'cirrus'. The 'qxl' option enables the SPICE " .
 533             "display server. For win* OS you can select how many independent " .
 534             "displays you want, Linux guests can add displays them self.\n".
 535             "You can also run without any graphic card, using a serial device as terminal.",
 536     },
 537     watchdog => {
 538         optional => 1,
 539         type => 'string', format => 'pve-qm-watchdog',
 540         description => "Create a virtual hardware watchdog device.",
 541         verbose_description => "Create a virtual hardware watchdog device. Once enabled" .
 542             " (by a guest action), the watchdog must be periodically polled " .
 543             "by an agent inside the guest or else the watchdog will reset " .
 544             "the guest (or execute the respective action specified)",
 545     },
 546     startdate => {
 547         optional => 1,
 548         type => 'string',
 549         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 550         description => "Set the initial date of the real time clock. Valid format for date are: 'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 551         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 552         default => 'now',
 553     },
 554     startup =>  get_standard_option('pve-startup-order'),
 555     template => {
 556         optional => 1,
 557         type => 'boolean',
 558         description => "Enable/disable Template.",
 559         default => 0,
 560     },
 561     args => {
 562         optional => 1,
 563         type => 'string',
 564         description => "Arbitrary arguments passed to kvm.",
 565         verbose_description => <<EODESCR,
 566 Arbitrary arguments passed to kvm, for example:
 567
 568 args: -no-reboot -no-hpet
 569
 570 NOTE: this option is for experts only.
 571 EODESCR
 572     },
 573     tablet => {
 574         optional => 1,
 575         type => 'boolean',
 576         default => 1,
 577         description => "Enable/disable the USB tablet device.",
 578         verbose_description => "Enable/disable the USB tablet device. This device is " .
 579             "usually needed to allow absolute mouse positioning with VNC. " .
 580             "Else the mouse runs out of sync with normal VNC clients. " .
 581             "If you're running lots of console-only guests on one host, " .
 582             "you may consider disabling this to save some context switches. " .
 583             "This is turned off by default if you use spice (-vga=qxl).",
 584     },
 585     migrate_speed => {
 586         optional => 1,
 587         type => 'integer',
 588         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 589         minimum => 0,
 590         default => 0,
 591     },
 592     migrate_downtime => {
 593         optional => 1,
 594         type => 'number',
 595         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 596         minimum => 0,
 597         default => 0.1,
 598     },
 599     cdrom => {
 600         optional => 1,
 601         type => 'string', format => 'pve-qm-ide',
 602         typetext => '<volume>',
 603         description => "This is an alias for option -ide2",
 604     },
 605     cpu => {
 606         optional => 1,
 607         description => "Emulated CPU type.",
 608         type => 'string',
 609         format => $cpu_fmt,
 610     },
 611     parent => get_standard_option('pve-snapshot-name', {
 612         optional => 1,
 613         description => "Parent snapshot name. This is used internally, and should not be modified.",
 614     }),
 615     snaptime => {
 616         optional => 1,
 617         description => "Timestamp for snapshots.",
 618         type => 'integer',
 619         minimum => 0,
 620     },
 621     vmstate => {
 622         optional => 1,
 623         type => 'string', format => 'pve-volume-id',
 624         description => "Reference to a volume which stores the VM state. This is used internally for snapshots.",
 625     },
 626     vmstatestorage => get_standard_option('pve-storage-id', {
 627         description => "Default storage for VM state volumes/files.",
 628         optional => 1,
 629     }),
 630     runningmachine => get_standard_option('pve-qemu-machine', {
 631         description => "Specifies the Qemu machine type of the running vm. This is used internally for snapshots.",
 632     }),
 633     machine => get_standard_option('pve-qemu-machine'),
 634     arch => {
 635         description => "Virtual processor architecture. Defaults to the host.",
 636         optional => 1,
 637         type => 'string',
 638         enum => [qw(x86_64 aarch64)],
 639     },
 640     smbios1 => {
 641         description => "Specify SMBIOS type 1 fields.",
 642         type => 'string', format => 'pve-qm-smbios1',
 643         maxLength => 512,
 644         optional => 1,
 645     },
 646     protection => {
 647         optional => 1,
 648         type => 'boolean',
 649         description => "Sets the protection flag of the VM. This will disable the remove VM and remove disk operations.",
 650         default => 0,
 651     },
 652     bios => {
 653         optional => 1,
 654         type => 'string',
 655         enum => [ qw(seabios ovmf) ],
 656         description => "Select BIOS implementation.",
 657         default => 'seabios',
 658     },
 659     vmgenid => {
 660         type => 'string',
 661         pattern => '(?:[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}|[01])',
 662         format_description => 'UUID',
 663         description => "Set VM Generation ID. Use '1' to autogenerate on create or update, pass '0' to disable explicitly.",
 664         verbose_description => "The VM generation ID (vmgenid) device exposes a".
 665             " 128-bit integer value identifier to the guest OS. This allows to".
 666             " notify the guest operating system when the virtual machine is".
 667             " executed with a different configuration (e.g. snapshot execution".
 668             " or creation from a template). The guest operating system notices".
 669             " the change, and is then able to react as appropriate by marking".
 670             " its copies of distributed databases as dirty, re-initializing its".
 671             " random number generator, etc.\n".
 672             "Note that auto-creation only works when done throug API/CLI create".
 673             " or update methods, but not when manually editing the config file.",
 674         default => "1 (autogenerated)",
 675         optional => 1,
 676     },
 677     hookscript => {
 678         type => 'string',
 679         format => 'pve-volume-id',
 680         optional => 1,
 681         description => "Script that will be executed during various steps in the vms lifetime.",
 682     },
 683     ivshmem => {
 684         type => 'string',
 685         format => $ivshmem_fmt,
 686         description => "Inter-VM shared memory. Useful for direct communication between VMs, or to the host.",
 687         optional => 1,
 688     },
 689     audio0 => {
 690         type => 'string',
 691         format => $audio_fmt,
 692         description => "Configure a audio device, useful in combination with QXL/Spice.",
 693         optional => 1
 694     },
 695     spice_enhancements => {
 696         type => 'string',
 697         format => $spice_enhancements_fmt,
 698         description => "Configure additional enhancements for SPICE.",
 699         optional => 1
 700     },
 701 };
 702
 703 my $cicustom_fmt = {
 704     meta => {
 705         type => 'string',
 706         optional => 1,
 707         description => 'Specify a custom file containing all meta data passed to the VM via cloud-init. This is provider specific meaning configdrive2 and nocloud differ.',
 708         format => 'pve-volume-id',
 709         format_description => 'volume',
 710     },
 711     network => {
 712         type => 'string',
 713         optional => 1,
 714         description => 'Specify a custom file containing all network data passed to the VM via cloud-init.',
 715         format => 'pve-volume-id',
 716         format_description => 'volume',
 717     },
 718     user => {
 719         type => 'string',
 720         optional => 1,
 721         description => 'Specify a custom file containing all user data passed to the VM via cloud-init.',
 722         format => 'pve-volume-id',
 723         format_description => 'volume',
 724     },
 725 };
 726 PVE::JSONSchema::register_format('pve-qm-cicustom', $cicustom_fmt);
 727
 728 my $confdesc_cloudinit = {
 729     citype => {
 730         optional => 1,
 731         type => 'string',
 732         description => 'Specifies the cloud-init configuration format. The default depends on the configured operating system type (`ostype`. We use the `nocloud` format for Linux, and `configdrive2` for windows.',
 733         enum => ['configdrive2', 'nocloud'],
 734     },
 735     ciuser => {
 736         optional => 1,
 737         type => 'string',
 738         description => "cloud-init: User name to change ssh keys and password for instead of the image's configured default user.",
 739     },
 740     cipassword => {
 741         optional => 1,
 742         type => 'string',
 743         description => 'cloud-init: Password to assign the user. Using this is generally not recommended. Use ssh keys instead. Also note that older cloud-init versions do not support hashed passwords.',
 744     },
 745     cicustom => {
 746         optional => 1,
 747         type => 'string',
 748         description => 'cloud-init: Specify custom files to replace the automatically generated ones at start.',
 749         format => 'pve-qm-cicustom',
 750     },
 751     searchdomain => {
 752         optional => 1,
 753         type => 'string',
 754         description => "cloud-init: Sets DNS search domains for a container. Create will automatically use the setting from the host if neither searchdomain nor nameserver are set.",
 755     },
 756     nameserver => {
 757         optional => 1,
 758         type => 'string', format => 'address-list',
 759         description => "cloud-init: Sets DNS server IP address for a container. Create will automatically use the setting from the host if neither searchdomain nor nameserver are set.",
 760     },
 761     sshkeys => {
 762         optional => 1,
 763         type => 'string',
 764         format => 'urlencoded',
 765         description => "cloud-init: Setup public SSH keys (one key per line, OpenSSH format).",
 766     },
 767 };
 768
 769 # what about other qemu settings ?
 770 #cpu => 'string',
 771 #machine => 'string',
 772 #fda => 'file',
 773 #fdb => 'file',
 774 #mtdblock => 'file',
 775 #sd => 'file',
 776 #pflash => 'file',
 777 #snapshot => 'bool',
 778 #bootp => 'file',
 779 ##tftp => 'dir',
 780 ##smb => 'dir',
 781 #kernel => 'file',
 782 #append => 'string',
 783 #initrd => 'file',
 784 ##soundhw => 'string',
 785
 786 while (my ($k, $v) = each %$confdesc) {
 787     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 788 }
 789
 790 my $MAX_IDE_DISKS = 4;
 791 my $MAX_SCSI_DISKS = 14;
 792 my $MAX_VIRTIO_DISKS = 16;
 793 my $MAX_SATA_DISKS = 6;
 794 my $MAX_USB_DEVICES = 5;
 795 my $MAX_NETS = 32;
 796 my $MAX_UNUSED_DISKS = 256;
 797 my $MAX_HOSTPCI_DEVICES = 16;
 798 my $MAX_SERIAL_PORTS = 4;
 799 my $MAX_PARALLEL_PORTS = 3;
 800 my $MAX_NUMA = 8;
 801
 802 my $numa_fmt = {
 803     cpus => {
 804         type => "string",
 805         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 806         description => "CPUs accessing this NUMA node.",
 807         format_description => "id[-id];...",
 808     },
 809     memory => {
 810         type => "number",
 811         description => "Amount of memory this NUMA node provides.",
 812         optional => 1,
 813     },
 814     hostnodes => {
 815         type => "string",
 816         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 817         description => "Host NUMA nodes to use.",
 818         format_description => "id[-id];...",
 819         optional => 1,
 820     },
 821     policy => {
 822         type => 'string',
 823         enum => [qw(preferred bind interleave)],
 824         description => "NUMA allocation policy.",
 825         optional => 1,
 826     },
 827 };
 828 PVE::JSONSchema::register_format('pve-qm-numanode', $numa_fmt);
 829 my $numadesc = {
 830     optional => 1,
 831     type => 'string', format => $numa_fmt,
 832     description => "NUMA topology.",
 833 };
 834 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 835
 836 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 837     $confdesc->{"numa$i"} = $numadesc;
 838 }
 839
 840 my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000',  'pcnet',  'virtio',
 841                       'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3',
 842                       'e1000-82540em', 'e1000-82544gc', 'e1000-82545em'];
 843 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 844
 845 my $net_fmt_bridge_descr = <<__EOD__;
 846 Bridge to attach the network device to. The Proxmox VE standard bridge
 847 is called 'vmbr0'.
 848
 849 If you do not specify a bridge, we create a kvm user (NATed) network
 850 device, which provides DHCP and DNS services. The following addresses
 851 are used:
 852
 853  10.0.2.2   Gateway
 854  10.0.2.3   DNS Server
 855  10.0.2.4   SMB Server
 856
 857 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 858 __EOD__
 859
 860 my $net_fmt = {
 861     macaddr  => get_standard_option('mac-addr', {
 862         description => "MAC address. That address must be unique withing your network. This is automatically generated if not specified.",
 863     }),
 864     model => {
 865         type => 'string',
 866         description => "Network Card Model. The 'virtio' model provides the best performance with very low CPU overhead. If your guest does not support this driver, it is usually best to use 'e1000'.",
 867         enum => $nic_model_list,
 868         default_key => 1,
 869     },
 870     (map { $_ => { keyAlias => 'model', alias => 'macaddr' }} @$nic_model_list),
 871     bridge => {
 872         type => 'string',
 873         description => $net_fmt_bridge_descr,
 874         format_description => 'bridge',
 875         optional => 1,
 876     },
 877     queues => {
 878         type => 'integer',
 879         minimum => 0, maximum => 16,
 880         description => 'Number of packet queues to be used on the device.',
 881         optional => 1,
 882     },
 883     rate => {
 884         type => 'number',
 885         minimum => 0,
 886         description => "Rate limit in mbps (megabytes per second) as floating point number.",
 887         optional => 1,
 888     },
 889     tag => {
 890         type => 'integer',
 891         minimum => 1, maximum => 4094,
 892         description => 'VLAN tag to apply to packets on this interface.',
 893         optional => 1,
 894     },
 895     trunks => {
 896         type => 'string',
 897         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 898         description => 'VLAN trunks to pass through this interface.',
 899         format_description => 'vlanid[;vlanid...]',
 900         optional => 1,
 901     },
 902     firewall => {
 903         type => 'boolean',
 904         description => 'Whether this interface should be protected by the firewall.',
 905         optional => 1,
 906     },
 907     link_down => {
 908         type => 'boolean',
 909         description => 'Whether this interface should be disconnected (like pulling the plug).',
 910         optional => 1,
 911     },
 912 };
 913
 914 my $netdesc = {
 915     optional => 1,
 916     type => 'string', format => $net_fmt,
 917     description => "Specify network devices.",
 918 };
 919
 920 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 921
 922 my $ipconfig_fmt = {
 923     ip => {
 924         type => 'string',
 925         format => 'pve-ipv4-config',
 926         format_description => 'IPv4Format/CIDR',
 927         description => 'IPv4 address in CIDR format.',
 928         optional => 1,
 929         default => 'dhcp',
 930     },
 931     gw => {
 932         type => 'string',
 933         format => 'ipv4',
 934         format_description => 'GatewayIPv4',
 935         description => 'Default gateway for IPv4 traffic.',
 936         optional => 1,
 937         requires => 'ip',
 938     },
 939     ip6 => {
 940         type => 'string',
 941         format => 'pve-ipv6-config',
 942         format_description => 'IPv6Format/CIDR',
 943         description => 'IPv6 address in CIDR format.',
 944         optional => 1,
 945         default => 'dhcp',
 946     },
 947     gw6 => {
 948         type => 'string',
 949         format => 'ipv6',
 950         format_description => 'GatewayIPv6',
 951         description => 'Default gateway for IPv6 traffic.',
 952         optional => 1,
 953         requires => 'ip6',
 954     },
 955 };
 956 PVE::JSONSchema::register_format('pve-qm-ipconfig', $ipconfig_fmt);
 957 my $ipconfigdesc = {
 958     optional => 1,
 959     type => 'string', format => 'pve-qm-ipconfig',
 960     description => <<'EODESCR',
 961 cloud-init: Specify IP addresses and gateways for the corresponding interface.
 962
 963 IP addresses use CIDR notation, gateways are optional but need an IP of the same type specified.
 964
 965 The special string 'dhcp' can be used for IP addresses to use DHCP, in which case no explicit gateway should be provided.
 966 For IPv6 the special string 'auto' can be used to use stateless autoconfiguration.
 967
 968 If cloud-init is enabled and neither an IPv4 nor an IPv6 address is specified, it defaults to using dhcp on IPv4.
 969 EODESCR
 970 };
 971 PVE::JSONSchema::register_standard_option("pve-qm-ipconfig", $netdesc);
 972
 973 for (my $i = 0; $i < $MAX_NETS; $i++)  {
 974     $confdesc->{"net$i"} = $netdesc;
 975     $confdesc_cloudinit->{"ipconfig$i"} = $ipconfigdesc;
 976 }
 977
 978 foreach my $key (keys %$confdesc_cloudinit) {
 979     $confdesc->{$key} = $confdesc_cloudinit->{$key};
 980 }
 981
 982 PVE::JSONSchema::register_format('pve-volume-id-or-qm-path', \&verify_volume_id_or_qm_path);
 983 sub verify_volume_id_or_qm_path {
 984     my ($volid, $noerr) = @_;
 985
 986     if ($volid eq 'none' || $volid eq 'cdrom' || $volid =~ m|^/|) {
 987         return $volid;
 988     }
 989
 990     # if its neither 'none' nor 'cdrom' nor a path, check if its a volume-id
 991     $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') };
 992     if ($@) {
 993         return undef if $noerr;
 994         die $@;
 995     }
 996     return $volid;
 997 }
 998
 999 my $drivename_hash;
1000
1001 my %drivedesc_base = (
1002     volume => { alias => 'file' },
1003     file => {
1004         type => 'string',
1005         format => 'pve-volume-id-or-qm-path',
1006         default_key => 1,
1007         format_description => 'volume',
1008         description => "The drive's backing volume.",
1009     },
1010     media => {
1011         type => 'string',
1012         enum => [qw(cdrom disk)],
1013         description => "The drive's media type.",
1014         default => 'disk',
1015         optional => 1
1016     },
1017     cyls => {
1018         type => 'integer',
1019         description => "Force the drive's physical geometry to have a specific cylinder count.",
1020         optional => 1
1021     },
1022     heads => {
1023         type => 'integer',
1024         description => "Force the drive's physical geometry to have a specific head count.",
1025         optional => 1
1026     },
1027     secs => {
1028         type => 'integer',
1029         description => "Force the drive's physical geometry to have a specific sector count.",
1030         optional => 1
1031     },
1032     trans => {
1033         type => 'string',
1034         enum => [qw(none lba auto)],
1035         description => "Force disk geometry bios translation mode.",
1036         optional => 1,
1037     },
1038     snapshot => {
1039         type => 'boolean',
1040         description => "Controls qemu's snapshot mode feature."
1041             . " If activated, changes made to the disk are temporary and will"
1042             . " be discarded when the VM is shutdown.",
1043         optional => 1,
1044     },
1045     cache => {
1046         type => 'string',
1047         enum => [qw(none writethrough writeback unsafe directsync)],
1048         description => "The drive's cache mode",
1049         optional => 1,
1050     },
1051     format => get_standard_option('pve-qm-image-format'),
1052     size => {
1053         type => 'string',
1054         format => 'disk-size',
1055         format_description => 'DiskSize',
1056         description => "Disk size. This is purely informational and has no effect.",
1057         optional => 1,
1058     },
1059     backup => {
1060         type => 'boolean',
1061         description => "Whether the drive should be included when making backups.",
1062         optional => 1,
1063     },
1064     replicate => {
1065         type => 'boolean',
1066         description => 'Whether the drive should considered for replication jobs.',
1067         optional => 1,
1068         default => 1,
1069     },
1070     rerror => {
1071         type => 'string',
1072         enum => [qw(ignore report stop)],
1073         description => 'Read error action.',
1074         optional => 1,
1075     },
1076     werror => {
1077         type => 'string',
1078         enum => [qw(enospc ignore report stop)],
1079         description => 'Write error action.',
1080         optional => 1,
1081     },
1082     aio => {
1083         type => 'string',
1084         enum => [qw(native threads)],
1085         description => 'AIO type to use.',
1086         optional => 1,
1087     },
1088     discard => {
1089         type => 'string',
1090         enum => [qw(ignore on)],
1091         description => 'Controls whether to pass discard/trim requests to the underlying storage.',
1092         optional => 1,
1093     },
1094     detect_zeroes => {
1095         type => 'boolean',
1096         description => 'Controls whether to detect and try to optimize writes of zeroes.',
1097         optional => 1,
1098     },
1099     serial => {
1100         type => 'string',
1101         format => 'urlencoded',
1102         format_description => 'serial',
1103         maxLength => 20*3, # *3 since it's %xx url enoded
1104         description => "The drive's reported serial number, url-encoded, up to 20 bytes long.",
1105         optional => 1,
1106     },
1107     shared => {
1108         type => 'boolean',
1109         description => 'Mark this locally-managed volume as available on all nodes',
1110         verbose_description => "Mark this locally-managed volume as available on all nodes.\n\nWARNING: This option does not share the volume automatically, it assumes it is shared already!",
1111         optional => 1,
1112         default => 0,
1113     }
1114 );
1115
1116 my %iothread_fmt = ( iothread => {
1117         type => 'boolean',
1118         description => "Whether to use iothreads for this drive",
1119         optional => 1,
1120 });
1121
1122 my %model_fmt = (
1123     model => {
1124         type => 'string',
1125         format => 'urlencoded',
1126         format_description => 'model',
1127         maxLength => 40*3, # *3 since it's %xx url enoded
1128         description => "The drive's reported model name, url-encoded, up to 40 bytes long.",
1129         optional => 1,
1130     },
1131 );
1132
1133 my %queues_fmt = (
1134     queues => {
1135         type => 'integer',
1136         description => "Number of queues.",
1137         minimum => 2,
1138         optional => 1
1139     }
1140 );
1141
1142 my %scsiblock_fmt = (
1143     scsiblock => {
1144         type => 'boolean',
1145         description => "whether to use scsi-block for full passthrough of host block device\n\nWARNING: can lead to I/O errors in combination with low memory or high memory fragmentation on host",
1146         optional => 1,
1147         default => 0,
1148     },
1149 );
1150
1151 my %ssd_fmt = (
1152     ssd => {
1153         type => 'boolean',
1154         description => "Whether to expose this drive as an SSD, rather than a rotational hard disk.",
1155         optional => 1,
1156     },
1157 );
1158
1159 my %wwn_fmt = (
1160     wwn => {
1161         type => 'string',
1162         pattern => qr/^(0x)[0-9a-fA-F]{16}/,
1163         format_description => 'wwn',
1164         description => "The drive's worldwide name, encoded as 16 bytes hex string, prefixed by '0x'.",
1165         optional => 1,
1166     },
1167 );
1168
1169 my $add_throttle_desc = sub {
1170     my ($key, $type, $what, $unit, $longunit, $minimum) = @_;
1171     my $d = {
1172         type => $type,
1173         format_description => $unit,
1174         description => "Maximum $what in $longunit.",
1175         optional => 1,
1176     };
1177     $d->{minimum} = $minimum if defined($minimum);
1178     $drivedesc_base{$key} = $d;
1179 };
1180 # throughput: (leaky bucket)
1181 $add_throttle_desc->('bps',     'integer', 'r/w speed',   'bps',  'bytes per second');
1182 $add_throttle_desc->('bps_rd',  'integer', 'read speed',  'bps',  'bytes per second');
1183 $add_throttle_desc->('bps_wr',  'integer', 'write speed', 'bps',  'bytes per second');
1184 $add_throttle_desc->('mbps',    'number',  'r/w speed',   'mbps', 'megabytes per second');
1185 $add_throttle_desc->('mbps_rd', 'number',  'read speed',  'mbps', 'megabytes per second');
1186 $add_throttle_desc->('mbps_wr', 'number',  'write speed', 'mbps', 'megabytes per second');
1187 $add_throttle_desc->('iops',    'integer', 'r/w I/O',     'iops', 'operations per second');
1188 $add_throttle_desc->('iops_rd', 'integer', 'read I/O',    'iops', 'operations per second');
1189 $add_throttle_desc->('iops_wr', 'integer', 'write I/O',   'iops', 'operations per second');
1190
1191 # pools: (pool of IO before throttling starts taking effect)
1192 $add_throttle_desc->('mbps_max',    'number',  'unthrottled r/w pool',       'mbps', 'megabytes per second');
1193 $add_throttle_desc->('mbps_rd_max', 'number',  'unthrottled read pool',      'mbps', 'megabytes per second');
1194 $add_throttle_desc->('mbps_wr_max', 'number',  'unthrottled write pool',     'mbps', 'megabytes per second');
1195 $add_throttle_desc->('iops_max',    'integer', 'unthrottled r/w I/O pool',   'iops', 'operations per second');
1196 $add_throttle_desc->('iops_rd_max', 'integer', 'unthrottled read I/O pool',  'iops', 'operations per second');
1197 $add_throttle_desc->('iops_wr_max', 'integer', 'unthrottled write I/O pool', 'iops', 'operations per second');
1198
1199 # burst lengths
1200 $add_throttle_desc->('bps_max_length',     'integer', 'length of I/O bursts',       'seconds', 'seconds', 1);
1201 $add_throttle_desc->('bps_rd_max_length',  'integer', 'length of read I/O bursts',  'seconds', 'seconds', 1);
1202 $add_throttle_desc->('bps_wr_max_length',  'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
1203 $add_throttle_desc->('iops_max_length',    'integer', 'length of I/O bursts',       'seconds', 'seconds', 1);
1204 $add_throttle_desc->('iops_rd_max_length', 'integer', 'length of read I/O bursts',  'seconds', 'seconds', 1);
1205 $add_throttle_desc->('iops_wr_max_length', 'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
1206
1207 # legacy support
1208 $drivedesc_base{'bps_rd_length'} = { alias => 'bps_rd_max_length' };
1209 $drivedesc_base{'bps_wr_length'} = { alias => 'bps_wr_max_length' };
1210 $drivedesc_base{'iops_rd_length'} = { alias => 'iops_rd_max_length' };
1211 $drivedesc_base{'iops_wr_length'} = { alias => 'iops_wr_max_length' };
1212
1213 my $ide_fmt = {
1214     %drivedesc_base,
1215     %model_fmt,
1216     %ssd_fmt,
1217     %wwn_fmt,
1218 };
1219 PVE::JSONSchema::register_format("pve-qm-ide", $ide_fmt);
1220
1221 my $idedesc = {
1222     optional => 1,
1223     type => 'string', format => $ide_fmt,
1224     description => "Use volume as IDE hard disk or CD-ROM (n is 0 to " .($MAX_IDE_DISKS -1) . ").",
1225 };
1226 PVE::JSONSchema::register_standard_option("pve-qm-ide", $idedesc);
1227
1228 my $scsi_fmt = {
1229     %drivedesc_base,
1230     %iothread_fmt,
1231     %queues_fmt,
1232     %scsiblock_fmt,
1233     %ssd_fmt,
1234     %wwn_fmt,
1235 };
1236 my $scsidesc = {
1237     optional => 1,
1238     type => 'string', format => $scsi_fmt,
1239     description => "Use volume as SCSI hard disk or CD-ROM (n is 0 to " . ($MAX_SCSI_DISKS - 1) . ").",
1240 };
1241 PVE::JSONSchema::register_standard_option("pve-qm-scsi", $scsidesc);
1242
1243 my $sata_fmt = {
1244     %drivedesc_base,
1245     %ssd_fmt,
1246     %wwn_fmt,
1247 };
1248 my $satadesc = {
1249     optional => 1,
1250     type => 'string', format => $sata_fmt,
1251     description => "Use volume as SATA hard disk or CD-ROM (n is 0 to " . ($MAX_SATA_DISKS - 1). ").",
1252 };
1253 PVE::JSONSchema::register_standard_option("pve-qm-sata", $satadesc);
1254
1255 my $virtio_fmt = {
1256     %drivedesc_base,
1257     %iothread_fmt,
1258 };
1259 my $virtiodesc = {
1260     optional => 1,
1261     type => 'string', format => $virtio_fmt,
1262     description => "Use volume as VIRTIO hard disk (n is 0 to " . ($MAX_VIRTIO_DISKS - 1) . ").",
1263 };
1264 PVE::JSONSchema::register_standard_option("pve-qm-virtio", $virtiodesc);
1265
1266 my $alldrive_fmt = {
1267     %drivedesc_base,
1268     %iothread_fmt,
1269     %model_fmt,
1270     %queues_fmt,
1271     %scsiblock_fmt,
1272     %ssd_fmt,
1273     %wwn_fmt,
1274 };
1275
1276 my $efidisk_fmt = {
1277     volume => { alias => 'file' },
1278     file => {
1279         type => 'string',
1280         format => 'pve-volume-id-or-qm-path',
1281         default_key => 1,
1282         format_description => 'volume',
1283         description => "The drive's backing volume.",
1284     },
1285     format => get_standard_option('pve-qm-image-format'),
1286     size => {
1287         type => 'string',
1288         format => 'disk-size',
1289         format_description => 'DiskSize',
1290         description => "Disk size. This is purely informational and has no effect.",
1291         optional => 1,
1292     },
1293 };
1294
1295 my $efidisk_desc = {
1296     optional => 1,
1297     type => 'string', format => $efidisk_fmt,
1298     description => "Configure a Disk for storing EFI vars",
1299 };
1300
1301 PVE::JSONSchema::register_standard_option("pve-qm-efidisk", $efidisk_desc);
1302
1303 my $usb_fmt = {
1304     host => {
1305         default_key => 1,
1306         type => 'string', format => 'pve-qm-usb-device',
1307         format_description => 'HOSTUSBDEVICE|spice',
1308         description => <<EODESCR,
1309 The Host USB device or port or the value 'spice'. HOSTUSBDEVICE syntax is:
1310
1311  'bus-port(.port)*' (decimal numbers) or
1312  'vendor_id:product_id' (hexadeciaml numbers) or
1313  'spice'
1314
1315 You can use the 'lsusb -t' command to list existing usb devices.
1316
1317 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1318
1319 The value 'spice' can be used to add a usb redirection devices for spice.
1320 EODESCR
1321     },
1322     usb3 => {
1323         optional => 1,
1324         type => 'boolean',
1325         description => "Specifies whether if given host option is a USB3 device or port.",
1326         default => 0,
1327     },
1328 };
1329
1330 my $usbdesc = {
1331     optional => 1,
1332     type => 'string', format => $usb_fmt,
1333     description => "Configure an USB device (n is 0 to 4).",
1334 };
1335 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
1336
1337 my $PCIRE = qr/[a-f0-9]{2}:[a-f0-9]{2}(?:\.[a-f0-9])?/;
1338 my $hostpci_fmt = {
1339     host => {
1340         default_key => 1,
1341         type => 'string',
1342         pattern => qr/$PCIRE(;$PCIRE)*/,
1343         format_description => 'HOSTPCIID[;HOSTPCIID2...]',
1344         description => <<EODESCR,
1345 Host PCI device pass through. The PCI ID of a host's PCI device or a list
1346 of PCI virtual functions of the host. HOSTPCIID syntax is:
1347
1348 'bus:dev.func' (hexadecimal numbers)
1349
1350 You can us the 'lspci' command to list existing PCI devices.
1351 EODESCR
1352     },
1353     rombar => {
1354         type => 'boolean',
1355         description =>  "Specify whether or not the device's ROM will be visible in the guest's memory map.",
1356         optional => 1,
1357         default => 1,
1358     },
1359     romfile => {
1360         type => 'string',
1361         pattern => '[^,;]+',
1362         format_description => 'string',
1363         description => "Custom pci device rom filename (must be located in /usr/share/kvm/).",
1364         optional => 1,
1365     },
1366     pcie => {
1367         type => 'boolean',
1368         description =>  "Choose the PCI-express bus (needs the 'q35' machine model).",
1369         optional => 1,
1370         default => 0,
1371     },
1372     'x-vga' => {
1373         type => 'boolean',
1374         description =>  "Enable vfio-vga device support.",
1375         optional => 1,
1376         default => 0,
1377     },
1378     'mdev' => {
1379         type => 'string',
1380         format_description => 'string',
1381         pattern => '[^/\.:]+',
1382         optional => 1,
1383         description => <<EODESCR
1384 The type of mediated device to use.
1385 An instance of this type will be created on startup of the VM and
1386 will be cleaned up when the VM stops.
1387 EODESCR
1388     }
1389 };
1390 PVE::JSONSchema::register_format('pve-qm-hostpci', $hostpci_fmt);
1391
1392 my $hostpcidesc = {
1393         optional => 1,
1394         type => 'string', format => 'pve-qm-hostpci',
1395         description => "Map host PCI devices into guest.",
1396         verbose_description =>  <<EODESCR,
1397 Map host PCI devices into guest.
1398
1399 NOTE: This option allows direct access to host hardware. So it is no longer
1400 possible to migrate such machines - use with special care.
1401
1402 CAUTION: Experimental! User reported problems with this option.
1403 EODESCR
1404 };
1405 PVE::JSONSchema::register_standard_option("pve-qm-hostpci", $hostpcidesc);
1406
1407 my $serialdesc = {
1408         optional => 1,
1409         type => 'string',
1410         pattern => '(/dev/.+|socket)',
1411         description =>  "Create a serial device inside the VM (n is 0 to 3)",
1412         verbose_description =>  <<EODESCR,
1413 Create a serial device inside the VM (n is 0 to 3), and pass through a
1414 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1415 host side (use 'qm terminal' to open a terminal connection).
1416
1417 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines - use with special care.
1418
1419 CAUTION: Experimental! User reported problems with this option.
1420 EODESCR
1421 };
1422
1423 my $paralleldesc= {
1424         optional => 1,
1425         type => 'string',
1426         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
1427         description =>  "Map host parallel devices (n is 0 to 2).",
1428         verbose_description =>  <<EODESCR,
1429 Map host parallel devices (n is 0 to 2).
1430
1431 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1432
1433 CAUTION: Experimental! User reported problems with this option.
1434 EODESCR
1435 };
1436
1437 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
1438     $confdesc->{"parallel$i"} = $paralleldesc;
1439 }
1440
1441 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
1442     $confdesc->{"serial$i"} = $serialdesc;
1443 }
1444
1445 for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
1446     $confdesc->{"hostpci$i"} = $hostpcidesc;
1447 }
1448
1449 for (my $i = 0; $i < $MAX_IDE_DISKS; $i++)  {
1450     $drivename_hash->{"ide$i"} = 1;
1451     $confdesc->{"ide$i"} = $idedesc;
1452 }
1453
1454 for (my $i = 0; $i < $MAX_SATA_DISKS; $i++)  {
1455     $drivename_hash->{"sata$i"} = 1;
1456     $confdesc->{"sata$i"} = $satadesc;
1457 }
1458
1459 for (my $i = 0; $i < $MAX_SCSI_DISKS; $i++)  {
1460     $drivename_hash->{"scsi$i"} = 1;
1461     $confdesc->{"scsi$i"} = $scsidesc ;
1462 }
1463
1464 for (my $i = 0; $i < $MAX_VIRTIO_DISKS; $i++)  {
1465     $drivename_hash->{"virtio$i"} = 1;
1466     $confdesc->{"virtio$i"} = $virtiodesc;
1467 }
1468
1469 $drivename_hash->{efidisk0} = 1;
1470 $confdesc->{efidisk0} = $efidisk_desc;
1471
1472 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
1473     $confdesc->{"usb$i"} = $usbdesc;
1474 }
1475
1476 my $unuseddesc = {
1477     optional => 1,
1478     type => 'string', format => 'pve-volume-id',
1479     description => "Reference to unused volumes. This is used internally, and should not be modified manually.",
1480 };
1481
1482 for (my $i = 0; $i < $MAX_UNUSED_DISKS; $i++)  {
1483     $confdesc->{"unused$i"} = $unuseddesc;
1484 }
1485
1486 my $kvm_api_version = 0;
1487
1488 sub kvm_version {
1489     return $kvm_api_version if $kvm_api_version;
1490
1491     open my $fh, '<', '/dev/kvm'
1492         or return undef;
1493
1494     # 0xae00 => KVM_GET_API_VERSION
1495     $kvm_api_version = ioctl($fh, 0xae00, 0);
1496
1497     return $kvm_api_version;
1498 }
1499
1500 my $kvm_user_version = {};
1501 my $kvm_mtime = {};
1502
1503 sub kvm_user_version {
1504     my ($binary) = @_;
1505
1506     $binary //= get_command_for_arch(get_host_arch()); # get the native arch by default
1507     my $st = stat($binary);
1508
1509     my $cachedmtime = $kvm_mtime->{$binary} // -1;
1510     return $kvm_user_version->{$binary} if $kvm_user_version->{$binary} &&
1511         $cachedmtime == $st->mtime;
1512
1513     $kvm_user_version->{$binary} = 'unknown';
1514     $kvm_mtime->{$binary} = $st->mtime;
1515
1516     my $code = sub {
1517         my $line = shift;
1518         if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1519             $kvm_user_version->{$binary} = $2;
1520         }
1521     };
1522
1523     eval { run_command([$binary, '--version'], outfunc => $code); };
1524     warn $@ if $@;
1525
1526     return $kvm_user_version->{$binary};
1527
1528 }
1529
1530 sub kernel_has_vhost_net {
1531     return -c '/dev/vhost-net';
1532 }
1533
1534 sub valid_drive_names {
1535     # order is important - used to autoselect boot disk
1536     return ((map { "ide$_" } (0 .. ($MAX_IDE_DISKS - 1))),
1537             (map { "scsi$_" } (0 .. ($MAX_SCSI_DISKS - 1))),
1538             (map { "virtio$_" } (0 .. ($MAX_VIRTIO_DISKS - 1))),
1539             (map { "sata$_" } (0 .. ($MAX_SATA_DISKS - 1))),
1540             'efidisk0');
1541 }
1542
1543 sub is_valid_drivename {
1544     my $dev = shift;
1545
1546     return defined($drivename_hash->{$dev});
1547 }
1548
1549 sub option_exists {
1550     my $key = shift;
1551     return defined($confdesc->{$key});
1552 }
1553
1554 sub nic_models {
1555     return $nic_model_list;
1556 }
1557
1558 sub os_list_description {
1559
1560     return {
1561         other => 'Other',
1562         wxp => 'Windows XP',
1563         w2k => 'Windows 2000',
1564         w2k3 =>, 'Windows 2003',
1565         w2k8 => 'Windows 2008',
1566         wvista => 'Windows Vista',
1567         win7 => 'Windows 7',
1568         win8 => 'Windows 8/2012',
1569         win10 => 'Windows 10/2016',
1570         l24 => 'Linux 2.4',
1571         l26 => 'Linux 2.6',
1572     };
1573 }
1574
1575 my $cdrom_path;
1576
1577 sub get_cdrom_path {
1578
1579     return  $cdrom_path if $cdrom_path;
1580
1581     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
1582     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
1583     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
1584 }
1585
1586 sub get_iso_path {
1587     my ($storecfg, $vmid, $cdrom) = @_;
1588
1589     if ($cdrom eq 'cdrom') {
1590         return get_cdrom_path();
1591     } elsif ($cdrom eq 'none') {
1592         return '';
1593     } elsif ($cdrom =~ m|^/|) {
1594         return $cdrom;
1595     } else {
1596         return PVE::Storage::path($storecfg, $cdrom);
1597     }
1598 }
1599
1600 # try to convert old style file names to volume IDs
1601 sub filename_to_volume_id {
1602     my ($vmid, $file, $media) = @_;
1603
1604      if (!($file eq 'none' || $file eq 'cdrom' ||
1605           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1606
1607         return undef if $file =~ m|/|;
1608
1609         if ($media && $media eq 'cdrom') {
1610             $file = "local:iso/$file";
1611         } else {
1612             $file = "local:$vmid/$file";
1613         }
1614     }
1615
1616     return $file;
1617 }
1618
1619 sub verify_media_type {
1620     my ($opt, $vtype, $media) = @_;
1621
1622     return if !$media;
1623
1624     my $etype;
1625     if ($media eq 'disk') {
1626         $etype = 'images';
1627     } elsif ($media eq 'cdrom') {
1628         $etype = 'iso';
1629     } else {
1630         die "internal error";
1631     }
1632
1633     return if ($vtype eq $etype);
1634
1635     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1636 }
1637
1638 sub cleanup_drive_path {
1639     my ($opt, $storecfg, $drive) = @_;
1640
1641     # try to convert filesystem paths to volume IDs
1642
1643     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1644         ($drive->{file} !~ m|^/dev/.+|) &&
1645         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1646         ($drive->{file} !~ m/^\d+$/)) {
1647         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1648         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"}) if !$vtype;
1649         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1650         verify_media_type($opt, $vtype, $drive->{media});
1651         $drive->{file} = $volid;
1652     }
1653
1654     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1655 }
1656
1657 sub parse_hotplug_features {
1658     my ($data) = @_;
1659
1660     my $res = {};
1661
1662     return $res if $data eq '0';
1663
1664     $data = $confdesc->{hotplug}->{default} if $data eq '1';
1665
1666     foreach my $feature (PVE::Tools::split_list($data)) {
1667         if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
1668             $res->{$1} = 1;
1669         } else {
1670             die "invalid hotplug feature '$feature'\n";
1671         }
1672     }
1673     return $res;
1674 }
1675
1676 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1677 sub pve_verify_hotplug_features {
1678     my ($value, $noerr) = @_;
1679
1680     return $value if parse_hotplug_features($value);
1681
1682     return undef if $noerr;
1683
1684     die "unable to parse hotplug option\n";
1685 }
1686
1687 # ideX = [volume=]volume-id[,media=d][,cyls=c,heads=h,secs=s[,trans=t]]
1688 #        [,snapshot=on|off][,cache=on|off][,format=f][,backup=yes|no]
1689 #        [,rerror=ignore|report|stop][,werror=enospc|ignore|report|stop]
1690 #        [,aio=native|threads][,discard=ignore|on][,detect_zeroes=on|off]
1691 #        [,iothread=on][,serial=serial][,model=model]
1692
1693 sub parse_drive {
1694     my ($key, $data) = @_;
1695
1696     my ($interface, $index);
1697
1698     if ($key =~ m/^([^\d]+)(\d+)$/) {
1699         $interface = $1;
1700         $index = $2;
1701     } else {
1702         return undef;
1703     }
1704
1705     my $desc = $key =~ /^unused\d+$/ ? $alldrive_fmt
1706                                      : $confdesc->{$key}->{format};
1707     if (!$desc) {
1708         warn "invalid drive key: $key\n";
1709         return undef;
1710     }
1711     my $res = eval { PVE::JSONSchema::parse_property_string($desc, $data) };
1712     return undef if !$res;
1713     $res->{interface} = $interface;
1714     $res->{index} = $index;
1715
1716     my $error = 0;
1717     foreach my $opt (qw(bps bps_rd bps_wr)) {
1718         if (my $bps = defined(delete $res->{$opt})) {
1719             if (defined($res->{"m$opt"})) {
1720                 warn "both $opt and m$opt specified\n";
1721                 ++$error;
1722                 next;
1723             }
1724             $res->{"m$opt"} = sprintf("%.3f", $bps / (1024*1024.0));
1725         }
1726     }
1727
1728     # can't use the schema's 'requires' because of the mbps* => bps* "transforming aliases"
1729     for my $requirement (
1730         [mbps_max => 'mbps'],
1731         [mbps_rd_max => 'mbps_rd'],
1732         [mbps_wr_max => 'mbps_wr'],
1733         [miops_max => 'miops'],
1734         [miops_rd_max => 'miops_rd'],
1735         [miops_wr_max => 'miops_wr'],
1736         [bps_max_length => 'mbps_max'],
1737         [bps_rd_max_length => 'mbps_rd_max'],
1738         [bps_wr_max_length => 'mbps_wr_max'],
1739         [iops_max_length => 'iops_max'],
1740         [iops_rd_max_length => 'iops_rd_max'],
1741         [iops_wr_max_length => 'iops_wr_max']) {
1742         my ($option, $requires) = @$requirement;
1743         if ($res->{$option} && !$res->{$requires}) {
1744             warn "$option requires $requires\n";
1745             ++$error;
1746         }
1747     }
1748
1749     return undef if $error;
1750
1751     return undef if $res->{mbps_rd} && $res->{mbps};
1752     return undef if $res->{mbps_wr} && $res->{mbps};
1753     return undef if $res->{iops_rd} && $res->{iops};
1754     return undef if $res->{iops_wr} && $res->{iops};
1755
1756     if ($res->{media} && ($res->{media} eq 'cdrom')) {
1757         return undef if $res->{snapshot} || $res->{trans} || $res->{format};
1758         return undef if $res->{heads} || $res->{secs} || $res->{cyls};
1759         return undef if $res->{interface} eq 'virtio';
1760     }
1761
1762     if (my $size = $res->{size}) {
1763         return undef if !defined($res->{size} = PVE::JSONSchema::parse_size($size));
1764     }
1765
1766     return $res;
1767 }
1768
1769 sub print_drive {
1770     my ($vmid, $drive) = @_;
1771     my $data = { %$drive };
1772     delete $data->{$_} for qw(index interface);
1773     return PVE::JSONSchema::print_property_string($data, $alldrive_fmt);
1774 }
1775
1776 sub scsi_inquiry {
1777     my($fh, $noerr) = @_;
1778
1779     my $SG_IO = 0x2285;
1780     my $SG_GET_VERSION_NUM = 0x2282;
1781
1782     my $versionbuf = "\x00" x 8;
1783     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1784     if (!$ret) {
1785         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1786         return undef;
1787     }
1788     my $version = unpack("I", $versionbuf);
1789     if ($version < 30000) {
1790         die "scsi generic interface too old\n"  if !$noerr;
1791         return undef;
1792     }
1793
1794     my $buf = "\x00" x 36;
1795     my $sensebuf = "\x00" x 8;
1796     my $cmd = pack("C x3 C x1", 0x12, 36);
1797
1798     # see /usr/include/scsi/sg.h
1799     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1800
1801     my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1802                       length($sensebuf), 0, length($buf), $buf,
1803                       $cmd, $sensebuf, 6000);
1804
1805     $ret = ioctl($fh, $SG_IO, $packet);
1806     if (!$ret) {
1807         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1808         return undef;
1809     }
1810
1811     my @res = unpack($sg_io_hdr_t, $packet);
1812     if ($res[17] || $res[18]) {
1813         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1814         return undef;
1815     }
1816
1817     my $res = {};
1818     (my $byte0, my $byte1, $res->{vendor},
1819      $res->{product}, $res->{revision}) = unpack("C C x6 A8 A16 A4", $buf);
1820
1821     $res->{removable} = $byte1 & 128 ? 1 : 0;
1822     $res->{type} = $byte0 & 31;
1823
1824     return $res;
1825 }
1826
1827 sub path_is_scsi {
1828     my ($path) = @_;
1829
1830     my $fh = IO::File->new("+<$path") || return undef;
1831     my $res = scsi_inquiry($fh, 1);
1832     close($fh);
1833
1834     return $res;
1835 }
1836
1837 sub machine_type_is_q35 {
1838     my ($conf) = @_;
1839
1840     return $conf->{machine} && ($conf->{machine} =~ m/q35/) ? 1 : 0;
1841 }
1842
1843 sub print_tabletdevice_full {
1844     my ($conf, $arch) = @_;
1845
1846     my $q35 = machine_type_is_q35($conf);
1847
1848     # we use uhci for old VMs because tablet driver was buggy in older qemu
1849     my $usbbus;
1850     if (machine_type_is_q35($conf) || $arch eq 'aarch64') {
1851         $usbbus = 'ehci';
1852     } else {
1853         $usbbus = 'uhci';
1854     }
1855
1856     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1857 }
1858
1859 sub print_keyboarddevice_full {
1860     my ($conf, $arch, $machine) = @_;
1861
1862     return undef if $arch ne 'aarch64';
1863
1864     return "usb-kbd,id=keyboard,bus=ehci.0,port=2";
1865 }
1866
1867 sub print_drivedevice_full {
1868     my ($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type) = @_;
1869
1870     my $device = '';
1871     my $maxdev = 0;
1872
1873     if ($drive->{interface} eq 'virtio') {
1874         my $pciaddr = print_pci_addr("$drive->{interface}$drive->{index}", $bridges, $arch, $machine_type);
1875         $device = "virtio-blk-pci,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}$pciaddr";
1876         $device .= ",iothread=iothread-$drive->{interface}$drive->{index}" if $drive->{iothread};
1877     } elsif ($drive->{interface} eq 'scsi') {
1878
1879         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1880         my $unit = $drive->{index} % $maxdev;
1881         my $devicetype = 'hd';
1882         my $path = '';
1883         if (drive_is_cdrom($drive)) {
1884             $devicetype = 'cd';
1885         } else {
1886             if ($drive->{file} =~ m|^/|) {
1887                 $path = $drive->{file};
1888                 if (my $info = path_is_scsi($path)) {
1889                     if ($info->{type} == 0 && $drive->{scsiblock}) {
1890                         $devicetype = 'block';
1891                     } elsif ($info->{type} == 1) { # tape
1892                         $devicetype = 'generic';
1893                     }
1894                 }
1895             } else {
1896                  $path = PVE::Storage::path($storecfg, $drive->{file});
1897             }
1898
1899             if($path =~ m/^iscsi\:\/\//){
1900                 $devicetype = 'generic';
1901             }
1902         }
1903
1904         if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)){
1905             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1906         } else {
1907             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0,lun=$drive->{index},drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1908         }
1909
1910         if ($drive->{ssd} && ($devicetype eq 'block' || $devicetype eq 'hd')) {
1911             $device .= ",rotation_rate=1";
1912         }
1913         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1914
1915     } elsif ($drive->{interface} eq 'ide' || $drive->{interface} eq 'sata') {
1916         my $maxdev = ($drive->{interface} eq 'sata') ? $MAX_SATA_DISKS : 2;
1917         my $controller = int($drive->{index} / $maxdev);
1918         my $unit = $drive->{index} % $maxdev;
1919         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1920
1921         $device = "ide-$devicetype";
1922         if ($drive->{interface} eq 'ide') {
1923             $device .= ",bus=ide.$controller,unit=$unit";
1924         } else {
1925             $device .= ",bus=ahci$controller.$unit";
1926         }
1927         $device .= ",drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1928
1929         if ($devicetype eq 'hd') {
1930             if (my $model = $drive->{model}) {
1931                 $model = URI::Escape::uri_unescape($model);
1932                 $device .= ",model=$model";
1933             }
1934             if ($drive->{ssd}) {
1935                 $device .= ",rotation_rate=1";
1936             }
1937         }
1938         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1939     } elsif ($drive->{interface} eq 'usb') {
1940         die "implement me";
1941         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1942     } else {
1943         die "unsupported interface type";
1944     }
1945
1946     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1947
1948     if (my $serial = $drive->{serial}) {
1949         $serial = URI::Escape::uri_unescape($serial);
1950         $device .= ",serial=$serial";
1951     }
1952
1953
1954     return $device;
1955 }
1956
1957 sub get_initiator_name {
1958     my $initiator;
1959
1960     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return undef;
1961     while (defined(my $line = <$fh>)) {
1962         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1963         $initiator = $1;
1964         last;
1965     }
1966     $fh->close();
1967
1968     return $initiator;
1969 }
1970
1971 sub print_drive_full {
1972     my ($storecfg, $vmid, $drive) = @_;
1973
1974     my $path;
1975     my $volid = $drive->{file};
1976     my $format;
1977
1978     if (drive_is_cdrom($drive)) {
1979         $path = get_iso_path($storecfg, $vmid, $volid);
1980     } else {
1981         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1982         if ($storeid) {
1983             $path = PVE::Storage::path($storecfg, $volid);
1984             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
1985             $format = qemu_img_format($scfg, $volname);
1986         } else {
1987             $path = $volid;
1988             $format = "raw";
1989         }
1990    }
1991
1992     my $opts = '';
1993     my @qemu_drive_options = qw(heads secs cyls trans media format cache rerror werror aio discard);
1994     foreach my $o (@qemu_drive_options) {
1995         $opts .= ",$o=$drive->{$o}" if defined($drive->{$o});
1996     }
1997
1998     # snapshot only accepts on|off
1999     if (defined($drive->{snapshot})) {
2000         my $v = $drive->{snapshot} ? 'on' : 'off';
2001         $opts .= ",snapshot=$v";
2002     }
2003
2004     foreach my $type (['', '-total'], [_rd => '-read'], [_wr => '-write']) {
2005         my ($dir, $qmpname) = @$type;
2006         if (my $v = $drive->{"mbps$dir"}) {
2007             $opts .= ",throttling.bps$qmpname=".int($v*1024*1024);
2008         }
2009         if (my $v = $drive->{"mbps${dir}_max"}) {
2010             $opts .= ",throttling.bps$qmpname-max=".int($v*1024*1024);
2011         }
2012         if (my $v = $drive->{"bps${dir}_max_length"}) {
2013             $opts .= ",throttling.bps$qmpname-max-length=$v";
2014         }
2015         if (my $v = $drive->{"iops${dir}"}) {
2016             $opts .= ",throttling.iops$qmpname=$v";
2017         }
2018         if (my $v = $drive->{"iops${dir}_max"}) {
2019             $opts .= ",throttling.iops$qmpname-max=$v";
2020         }
2021         if (my $v = $drive->{"iops${dir}_max_length"}) {
2022             $opts .= ",throttling.iops$qmpname-max-length=$v";
2023         }
2024     }
2025
2026     $opts .= ",format=$format" if $format && !$drive->{format};
2027
2028     my $cache_direct = 0;
2029
2030     if (my $cache = $drive->{cache}) {
2031         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
2032     } elsif (!drive_is_cdrom($drive)) {
2033         $opts .= ",cache=none";
2034         $cache_direct = 1;
2035     }
2036
2037     # aio native works only with O_DIRECT
2038     if (!$drive->{aio}) {
2039         if($cache_direct) {
2040             $opts .= ",aio=native";
2041         } else {
2042             $opts .= ",aio=threads";
2043         }
2044     }
2045
2046     if (!drive_is_cdrom($drive)) {
2047         my $detectzeroes;
2048         if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
2049             $detectzeroes = 'off';
2050         } elsif ($drive->{discard}) {
2051             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
2052         } else {
2053             # This used to be our default with discard not being specified:
2054             $detectzeroes = 'on';
2055         }
2056         $opts .= ",detect-zeroes=$detectzeroes" if $detectzeroes;
2057     }
2058
2059     my $pathinfo = $path ? "file=$path," : '';
2060
2061     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
2062 }
2063
2064 sub print_netdevice_full {
2065     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files, $arch, $machine_type) = @_;
2066
2067     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
2068
2069     my $device = $net->{model};
2070     if ($net->{model} eq 'virtio') {
2071          $device = 'virtio-net-pci';
2072      };
2073
2074     my $pciaddr = print_pci_addr("$netid", $bridges, $arch, $machine_type);
2075     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
2076     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
2077         #Consider we have N queues, the number of vectors needed is 2*N + 2 (plus one config interrupt and control vq)
2078         my $vectors = $net->{queues} * 2 + 2;
2079         $tmpstr .= ",vectors=$vectors,mq=on";
2080     }
2081     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
2082
2083     if ($use_old_bios_files) {
2084         my $romfile;
2085         if ($device eq 'virtio-net-pci') {
2086             $romfile = 'pxe-virtio.rom';
2087         } elsif ($device eq 'e1000') {
2088             $romfile = 'pxe-e1000.rom';
2089         } elsif ($device eq 'ne2k') {
2090             $romfile = 'pxe-ne2k_pci.rom';
2091         } elsif ($device eq 'pcnet') {
2092             $romfile = 'pxe-pcnet.rom';
2093         } elsif ($device eq 'rtl8139') {
2094             $romfile = 'pxe-rtl8139.rom';
2095         }
2096         $tmpstr .= ",romfile=$romfile" if $romfile;
2097     }
2098
2099     return $tmpstr;
2100 }
2101
2102 sub print_netdev_full {
2103     my ($vmid, $conf, $arch, $net, $netid, $hotplug) = @_;
2104
2105     my $i = '';
2106     if ($netid =~ m/^net(\d+)$/) {
2107         $i = int($1);
2108     }
2109
2110     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
2111
2112     my $ifname = "tap${vmid}i$i";
2113
2114     # kvm uses TUNSETIFF ioctl, and that limits ifname length
2115     die "interface name '$ifname' is too long (max 15 character)\n"
2116         if length($ifname) >= 16;
2117
2118     my $vhostparam = '';
2119     if (is_native($arch)) {
2120         $vhostparam = ',vhost=on' if kernel_has_vhost_net() && $net->{model} eq 'virtio';
2121     }
2122
2123     my $vmname = $conf->{name} || "vm$vmid";
2124
2125     my $netdev = "";
2126     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
2127
2128     if ($net->{bridge}) {
2129         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script,downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
2130     } else {
2131         $netdev = "type=user,id=$netid,hostname=$vmname";
2132     }
2133
2134     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
2135
2136     return $netdev;
2137 }
2138
2139
2140 sub print_cpu_device {
2141     my ($conf, $id) = @_;
2142
2143     my $kvm = $conf->{kvm} // 1;
2144     my $cpu = $kvm ? "kvm64" : "qemu64";
2145     if (my $cputype = $conf->{cpu}) {
2146         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
2147             or die "Cannot parse cpu description: $cputype\n";
2148         $cpu = $cpuconf->{cputype};
2149     }
2150
2151     my $cores = $conf->{cores} || 1;
2152
2153     my $current_core = ($id - 1) % $cores;
2154     my $current_socket = int(($id - 1 - $current_core)/$cores);
2155
2156     return "$cpu-x86_64-cpu,id=cpu$id,socket-id=$current_socket,core-id=$current_core,thread-id=0";
2157 }
2158
2159 my $vga_map = {
2160     'cirrus' => 'cirrus-vga',
2161     'std' => 'VGA',
2162     'vmware' => 'vmware-svga',
2163     'virtio' => 'virtio-vga',
2164 };
2165
2166 sub print_vga_device {
2167     my ($conf, $vga, $arch, $machine, $id, $qxlnum, $bridges) = @_;
2168
2169     my $type = $vga_map->{$vga->{type}};
2170     if ($arch eq 'aarch64' && defined($type) && $type eq 'virtio-vga') {
2171         $type = 'virtio-gpu';
2172     }
2173     my $vgamem_mb = $vga->{memory};
2174     if ($qxlnum) {
2175         $type = $id ? 'qxl' : 'qxl-vga';
2176     }
2177     die "no devicetype for $vga->{type}\n" if !$type;
2178
2179     my $memory = "";
2180     if ($vgamem_mb) {
2181         if ($vga->{type} eq 'virtio') {
2182             my $bytes = PVE::Tools::convert_size($vgamem_mb, "mb" => "b");
2183             $memory = ",max_hostmem=$bytes";
2184         } elsif ($qxlnum) {
2185             # from https://www.spice-space.org/multiple-monitors.html
2186             $memory = ",vgamem_mb=$vga->{memory}";
2187             my $ram = $vgamem_mb * 4;
2188             my $vram = $vgamem_mb * 2;
2189             $memory .= ",ram_size_mb=$ram,vram_size_mb=$vram";
2190         } else {
2191             $memory = ",vgamem_mb=$vga->{memory}";
2192         }
2193     } elsif ($qxlnum && $id) {
2194         $memory = ",ram_size=67108864,vram_size=33554432";
2195     }
2196
2197     my $q35 = machine_type_is_q35($conf);
2198     my $vgaid = "vga" . ($id // '');
2199     my $pciaddr;
2200
2201     if ($q35 && $vgaid eq 'vga') {
2202         # the first display uses pcie.0 bus on q35 machines
2203         $pciaddr = print_pcie_addr($vgaid, $bridges, $arch, $machine);
2204     } else {
2205         $pciaddr = print_pci_addr($vgaid, $bridges, $arch, $machine);
2206     }
2207
2208     return "$type,id=${vgaid}${memory}${pciaddr}";
2209 }
2210
2211 sub drive_is_cloudinit {
2212     my ($drive) = @_;
2213     return $drive->{file} =~ m@[:/]vm-\d+-cloudinit(?:\.$QEMU_FORMAT_RE)?$@;
2214 }
2215
2216 sub drive_is_cdrom {
2217     my ($drive, $exclude_cloudinit) = @_;
2218
2219     return 0 if $exclude_cloudinit && drive_is_cloudinit($drive);
2220
2221     return $drive && $drive->{media} && ($drive->{media} eq 'cdrom');
2222
2223 }
2224
2225 sub parse_number_sets {
2226     my ($set) = @_;
2227     my $res = [];
2228     foreach my $part (split(/;/, $set)) {
2229         if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
2230             die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
2231             push @$res, [ $1, $2 ];
2232         } else {
2233             die "invalid range: $part\n";
2234         }
2235     }
2236     return $res;
2237 }
2238
2239 sub parse_numa {
2240     my ($data) = @_;
2241
2242     my $res = PVE::JSONSchema::parse_property_string($numa_fmt, $data);
2243     $res->{cpus} = parse_number_sets($res->{cpus}) if defined($res->{cpus});
2244     $res->{hostnodes} = parse_number_sets($res->{hostnodes}) if defined($res->{hostnodes});
2245     return $res;
2246 }
2247
2248 sub parse_hostpci {
2249     my ($value) = @_;
2250
2251     return undef if !$value;
2252
2253     my $res = PVE::JSONSchema::parse_property_string($hostpci_fmt, $value);
2254
2255     my @idlist = split(/;/, $res->{host});
2256     delete $res->{host};
2257     foreach my $id (@idlist) {
2258         if ($id =~ m/\./) { # full id 00:00.1
2259             push @{$res->{pciid}}, {
2260                 id => $id,
2261             };
2262         } else { # partial id 00:00
2263             $res->{pciid} = PVE::SysFSTools::lspci($id);
2264         }
2265     }
2266     return $res;
2267 }
2268
2269 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
2270 sub parse_net {
2271     my ($data) = @_;
2272
2273     my $res = eval { PVE::JSONSchema::parse_property_string($net_fmt, $data) };
2274     if ($@) {
2275         warn $@;
2276         return undef;
2277     }
2278     if (!defined($res->{macaddr})) {
2279         my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
2280         $res->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix});
2281     }
2282     return $res;
2283 }
2284
2285 # ipconfigX ip=cidr,gw=ip,ip6=cidr,gw6=ip
2286 sub parse_ipconfig {
2287     my ($data) = @_;
2288
2289     my $res = eval { PVE::JSONSchema::parse_property_string($ipconfig_fmt, $data) };
2290     if ($@) {
2291         warn $@;
2292         return undef;
2293     }
2294
2295     if ($res->{gw} && !$res->{ip}) {
2296         warn 'gateway specified without specifying an IP address';
2297         return undef;
2298     }
2299     if ($res->{gw6} && !$res->{ip6}) {
2300         warn 'IPv6 gateway specified without specifying an IPv6 address';
2301         return undef;
2302     }
2303     if ($res->{gw} && $res->{ip} eq 'dhcp') {
2304         warn 'gateway specified together with DHCP';
2305         return undef;
2306     }
2307     if ($res->{gw6} && $res->{ip6} !~ /^$IPV6RE/) {
2308         # gw6 + auto/dhcp
2309         warn "IPv6 gateway specified together with $res->{ip6} address";
2310         return undef;
2311     }
2312
2313     if (!$res->{ip} && !$res->{ip6}) {
2314         return { ip => 'dhcp', ip6 => 'dhcp' };
2315     }
2316
2317     return $res;
2318 }
2319
2320 sub print_net {
2321     my $net = shift;
2322
2323     return PVE::JSONSchema::print_property_string($net, $net_fmt);
2324 }
2325
2326 sub add_random_macs {
2327     my ($settings) = @_;
2328
2329     foreach my $opt (keys %$settings) {
2330         next if $opt !~ m/^net(\d+)$/;
2331         my $net = parse_net($settings->{$opt});
2332         next if !$net;
2333         $settings->{$opt} = print_net($net);
2334     }
2335 }
2336
2337 sub vm_is_volid_owner {
2338     my ($storecfg, $vmid, $volid) = @_;
2339
2340     if ($volid !~  m|^/|) {
2341         my ($path, $owner);
2342         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
2343         if ($owner && ($owner == $vmid)) {
2344             return 1;
2345         }
2346     }
2347
2348     return undef;
2349 }
2350
2351 sub split_flagged_list {
2352     my $text = shift || '';
2353     $text =~ s/[,;]/ /g;
2354     $text =~ s/^\s+//;
2355     return { map { /^(!?)(.*)$/ && ($2, $1) } ($text =~ /\S+/g) };
2356 }
2357
2358 sub join_flagged_list {
2359     my ($how, $lst) = @_;
2360     join $how, map { $lst->{$_} . $_ } keys %$lst;
2361 }
2362
2363 sub vmconfig_delete_pending_option {
2364     my ($conf, $key, $force) = @_;
2365
2366     delete $conf->{pending}->{$key};
2367     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
2368     $pending_delete_hash->{$key} = $force ? '!' : '';
2369     $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
2370 }
2371
2372 sub vmconfig_undelete_pending_option {
2373     my ($conf, $key) = @_;
2374
2375     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
2376     delete $pending_delete_hash->{$key};
2377
2378     if (%$pending_delete_hash) {
2379         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
2380     } else {
2381         delete $conf->{pending}->{delete};
2382     }
2383 }
2384
2385 sub vmconfig_register_unused_drive {
2386     my ($storecfg, $vmid, $conf, $drive) = @_;
2387
2388     if (drive_is_cloudinit($drive)) {
2389         eval { PVE::Storage::vdisk_free($storecfg, $drive->{file}) };
2390         warn $@ if $@;
2391     } elsif (!drive_is_cdrom($drive)) {
2392         my $volid = $drive->{file};
2393         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
2394             PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
2395         }
2396     }
2397 }
2398
2399 sub vmconfig_cleanup_pending {
2400     my ($conf) = @_;
2401
2402     # remove pending changes when nothing changed
2403     my $changes;
2404     foreach my $opt (keys %{$conf->{pending}}) {
2405         if (defined($conf->{$opt}) && ($conf->{pending}->{$opt} eq  $conf->{$opt})) {
2406             $changes = 1;
2407             delete $conf->{pending}->{$opt};
2408         }
2409     }
2410
2411     my $current_delete_hash = split_flagged_list($conf->{pending}->{delete});
2412     my $pending_delete_hash = {};
2413     while (my ($opt, $force) = each %$current_delete_hash) {
2414         if (defined($conf->{$opt})) {
2415             $pending_delete_hash->{$opt} = $force;
2416         } else {
2417             $changes = 1;
2418         }
2419     }
2420
2421     if (%$pending_delete_hash) {
2422         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
2423     } else {
2424         delete $conf->{pending}->{delete};
2425     }
2426
2427     return $changes;
2428 }
2429
2430 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str][,base64=bool]
2431 my $smbios1_fmt = {
2432     uuid => {
2433         type => 'string',
2434         pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
2435         format_description => 'UUID',
2436         description => "Set SMBIOS1 UUID.",
2437         optional => 1,
2438     },
2439     version => {
2440         type => 'string',
2441         pattern => '[A-Za-z0-9+\/]+={0,2}',
2442         format_description => 'Base64 encoded string',
2443         description => "Set SMBIOS1 version.",
2444         optional => 1,
2445     },
2446     serial => {
2447         type => 'string',
2448         pattern => '[A-Za-z0-9+\/]+={0,2}',
2449         format_description => 'Base64 encoded string',
2450         description => "Set SMBIOS1 serial number.",
2451         optional => 1,
2452     },
2453     manufacturer => {
2454         type => 'string',
2455         pattern => '[A-Za-z0-9+\/]+={0,2}',
2456         format_description => 'Base64 encoded string',
2457         description => "Set SMBIOS1 manufacturer.",
2458         optional => 1,
2459     },
2460     product => {
2461         type => 'string',
2462         pattern => '[A-Za-z0-9+\/]+={0,2}',
2463         format_description => 'Base64 encoded string',
2464         description => "Set SMBIOS1 product ID.",
2465         optional => 1,
2466     },
2467     sku => {
2468         type => 'string',
2469         pattern => '[A-Za-z0-9+\/]+={0,2}',
2470         format_description => 'Base64 encoded string',
2471         description => "Set SMBIOS1 SKU string.",
2472         optional => 1,
2473     },
2474     family => {
2475         type => 'string',
2476         pattern => '[A-Za-z0-9+\/]+={0,2}',
2477         format_description => 'Base64 encoded string',
2478         description => "Set SMBIOS1 family string.",
2479         optional => 1,
2480     },
2481     base64 => {
2482         type => 'boolean',
2483         description => 'Flag to indicate that the SMBIOS values are base64 encoded',
2484         optional => 1,
2485     },
2486 };
2487
2488 sub parse_smbios1 {
2489     my ($data) = @_;
2490
2491     my $res = eval { PVE::JSONSchema::parse_property_string($smbios1_fmt, $data) };
2492     warn $@ if $@;
2493     return $res;
2494 }
2495
2496 sub print_smbios1 {
2497     my ($smbios1) = @_;
2498     return PVE::JSONSchema::print_property_string($smbios1, $smbios1_fmt);
2499 }
2500
2501 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_fmt);
2502
2503 PVE::JSONSchema::register_format('pve-qm-bootdisk', \&verify_bootdisk);
2504 sub verify_bootdisk {
2505     my ($value, $noerr) = @_;
2506
2507     return $value if is_valid_drivename($value);
2508
2509     return undef if $noerr;
2510
2511     die "invalid boot disk '$value'\n";
2512 }
2513
2514 sub parse_watchdog {
2515     my ($value) = @_;
2516
2517     return undef if !$value;
2518
2519     my $res = eval { PVE::JSONSchema::parse_property_string($watchdog_fmt, $value) };
2520     warn $@ if $@;
2521     return $res;
2522 }
2523
2524 sub parse_guest_agent {
2525     my ($value) = @_;
2526
2527     return {} if !defined($value->{agent});
2528
2529     my $res = eval { PVE::JSONSchema::parse_property_string($agent_fmt, $value->{agent}) };
2530     warn $@ if $@;
2531
2532     # if the agent is disabled ignore the other potentially set properties
2533     return {} if !$res->{enabled};
2534     return $res;
2535 }
2536
2537 sub parse_vga {
2538     my ($value) = @_;
2539
2540     return {} if !$value;
2541     my $res = eval { PVE::JSONSchema::parse_property_string($vga_fmt, $value) };
2542     warn $@ if $@;
2543     return $res;
2544 }
2545
2546 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
2547 sub verify_usb_device {
2548     my ($value, $noerr) = @_;
2549
2550     return $value if parse_usb_device($value);
2551
2552     return undef if $noerr;
2553
2554     die "unable to parse usb device\n";
2555 }
2556
2557 # add JSON properties for create and set function
2558 sub json_config_properties {
2559     my $prop = shift;
2560
2561     foreach my $opt (keys %$confdesc) {
2562         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate' || $opt eq 'runningmachine';
2563         $prop->{$opt} = $confdesc->{$opt};
2564     }
2565
2566     return $prop;
2567 }
2568
2569 # return copy of $confdesc_cloudinit to generate documentation
2570 sub cloudinit_config_properties {
2571
2572     return dclone($confdesc_cloudinit);
2573 }
2574
2575 sub check_type {
2576     my ($key, $value) = @_;
2577
2578     die "unknown setting '$key'\n" if !$confdesc->{$key};
2579
2580     my $type = $confdesc->{$key}->{type};
2581
2582     if (!defined($value)) {
2583         die "got undefined value\n";
2584     }
2585
2586     if ($value =~ m/[\n\r]/) {
2587         die "property contains a line feed\n";
2588     }
2589
2590     if ($type eq 'boolean') {
2591         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
2592         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
2593         die "type check ('boolean') failed - got '$value'\n";
2594     } elsif ($type eq 'integer') {
2595         return int($1) if $value =~ m/^(\d+)$/;
2596         die "type check ('integer') failed - got '$value'\n";
2597     } elsif ($type eq 'number') {
2598         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
2599         die "type check ('number') failed - got '$value'\n";
2600     } elsif ($type eq 'string') {
2601         if (my $fmt = $confdesc->{$key}->{format}) {
2602             PVE::JSONSchema::check_format($fmt, $value);
2603             return $value;
2604         }
2605         $value =~ s/^\"(.*)\"$/$1/;
2606         return $value;
2607     } else {
2608         die "internal error"
2609     }
2610 }
2611
2612 sub touch_config {
2613     my ($vmid) = @_;
2614
2615     my $conf = PVE::QemuConfig->config_file($vmid);
2616     utime undef, undef, $conf;
2617 }
2618
2619 sub destroy_vm {
2620     my ($storecfg, $vmid, $keep_empty_config, $skiplock) = @_;
2621
2622     my $conf = PVE::QemuConfig->load_config($vmid);
2623
2624     PVE::QemuConfig->check_lock($conf) if !$skiplock;
2625
2626     if ($conf->{template}) {
2627         # check if any base image is still used by a linked clone
2628         foreach_drive($conf, sub {
2629                 my ($ds, $drive) = @_;
2630
2631                 return if drive_is_cdrom($drive);
2632
2633                 my $volid = $drive->{file};
2634
2635                 return if !$volid || $volid =~ m|^/|;
2636
2637                 die "base volume '$volid' is still in use by linked cloned\n"
2638                     if PVE::Storage::volume_is_base_and_used($storecfg, $volid);
2639
2640         });
2641     }
2642
2643     # only remove disks owned by this VM
2644     foreach_drive($conf, sub {
2645         my ($ds, $drive) = @_;
2646
2647         return if drive_is_cdrom($drive, 1);
2648
2649         my $volid = $drive->{file};
2650
2651         return if !$volid || $volid =~ m|^/|;
2652
2653         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
2654         return if !$path || !$owner || ($owner != $vmid);
2655
2656         eval {
2657             PVE::Storage::vdisk_free($storecfg, $volid);
2658         };
2659         warn "Could not remove disk '$volid', check manually: $@" if $@;
2660
2661     });
2662
2663     if ($keep_empty_config) {
2664         PVE::QemuConfig->write_config($vmid, "memory: 128\n");
2665     } else {
2666         PVE::QemuConfig->destroy_config($vmid);
2667     }
2668
2669     # also remove unused disk
2670     eval {
2671         my $dl = PVE::Storage::vdisk_list($storecfg, undef, $vmid);
2672
2673         eval {
2674             PVE::Storage::foreach_volid($dl, sub {
2675                 my ($volid, $sid, $volname, $d) = @_;
2676                 PVE::Storage::vdisk_free($storecfg, $volid);
2677             });
2678         };
2679         warn $@ if $@;
2680
2681     };
2682     warn $@ if $@;
2683 }
2684
2685 sub parse_vm_config {
2686     my ($filename, $raw) = @_;
2687
2688     return undef if !defined($raw);
2689
2690     my $res = {
2691         digest => Digest::SHA::sha1_hex($raw),
2692         snapshots => {},
2693         pending => {},
2694     };
2695
2696     $filename =~ m|/qemu-server/(\d+)\.conf$|
2697         || die "got strange filename '$filename'";
2698
2699     my $vmid = $1;
2700
2701     my $conf = $res;
2702     my $descr;
2703     my $section = '';
2704
2705     my @lines = split(/\n/, $raw);
2706     foreach my $line (@lines) {
2707         next if $line =~ m/^\s*$/;
2708
2709         if ($line =~ m/^\[PENDING\]\s*$/i) {
2710             $section = 'pending';
2711             if (defined($descr)) {
2712                 $descr =~ s/\s+$//;
2713                 $conf->{description} = $descr;
2714             }
2715             $descr = undef;
2716             $conf = $res->{$section} = {};
2717             next;
2718
2719         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2720             $section = $1;
2721             if (defined($descr)) {
2722                 $descr =~ s/\s+$//;
2723                 $conf->{description} = $descr;
2724             }
2725             $descr = undef;
2726             $conf = $res->{snapshots}->{$section} = {};
2727             next;
2728         }
2729
2730         if ($line =~ m/^\#(.*)\s*$/) {
2731             $descr = '' if !defined($descr);
2732             $descr .= PVE::Tools::decode_text($1) . "\n";
2733             next;
2734         }
2735
2736         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2737             $descr = '' if !defined($descr);
2738             $descr .= PVE::Tools::decode_text($2);
2739         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2740             $conf->{snapstate} = $1;
2741         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2742             my $key = $1;
2743             my $value = $2;
2744             $conf->{$key} = $value;
2745         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2746             my $value = $1;
2747             if ($section eq 'pending') {
2748                 $conf->{delete} = $value; # we parse this later
2749             } else {
2750                 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
2751             }
2752         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(.+?)\s*$/) {
2753             my $key = $1;
2754             my $value = $2;
2755             eval { $value = check_type($key, $value); };
2756             if ($@) {
2757                 warn "vm $vmid - unable to parse value of '$key' - $@";
2758             } else {
2759                 $key = 'ide2' if $key eq 'cdrom';
2760                 my $fmt = $confdesc->{$key}->{format};
2761                 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2762                     my $v = parse_drive($key, $value);
2763                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2764                         $v->{file} = $volid;
2765                         $value = print_drive($vmid, $v);
2766                     } else {
2767                         warn "vm $vmid - unable to parse value of '$key'\n";
2768                         next;
2769                     }
2770                 }
2771
2772                 $conf->{$key} = $value;
2773             }
2774         }
2775     }
2776
2777     if (defined($descr)) {
2778         $descr =~ s/\s+$//;
2779         $conf->{description} = $descr;
2780     }
2781     delete $res->{snapstate}; # just to be sure
2782
2783     return $res;
2784 }
2785
2786 sub write_vm_config {
2787     my ($filename, $conf) = @_;
2788
2789     delete $conf->{snapstate}; # just to be sure
2790
2791     if ($conf->{cdrom}) {
2792         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2793         $conf->{ide2} = $conf->{cdrom};
2794         delete $conf->{cdrom};
2795     }
2796
2797     # we do not use 'smp' any longer
2798     if ($conf->{sockets}) {
2799         delete $conf->{smp};
2800     } elsif ($conf->{smp}) {
2801         $conf->{sockets} = $conf->{smp};
2802         delete $conf->{cores};
2803         delete $conf->{smp};
2804     }
2805
2806     my $used_volids = {};
2807
2808     my $cleanup_config = sub {
2809         my ($cref, $pending, $snapname) = @_;
2810
2811         foreach my $key (keys %$cref) {
2812             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2813                 $key eq 'snapstate' || $key eq 'pending';
2814             my $value = $cref->{$key};
2815             if ($key eq 'delete') {
2816                 die "propertry 'delete' is only allowed in [PENDING]\n"
2817                     if !$pending;
2818                 # fixme: check syntax?
2819                 next;
2820             }
2821             eval { $value = check_type($key, $value); };
2822             die "unable to parse value of '$key' - $@" if $@;
2823
2824             $cref->{$key} = $value;
2825
2826             if (!$snapname && is_valid_drivename($key)) {
2827                 my $drive = parse_drive($key, $value);
2828                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2829             }
2830         }
2831     };
2832
2833     &$cleanup_config($conf);
2834
2835     &$cleanup_config($conf->{pending}, 1);
2836
2837     foreach my $snapname (keys %{$conf->{snapshots}}) {
2838         die "internal error" if $snapname eq 'pending';
2839         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2840     }
2841
2842     # remove 'unusedX' settings if we re-add a volume
2843     foreach my $key (keys %$conf) {
2844         my $value = $conf->{$key};
2845         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2846             delete $conf->{$key};
2847         }
2848     }
2849
2850     my $generate_raw_config = sub {
2851         my ($conf, $pending) = @_;
2852
2853         my $raw = '';
2854
2855         # add description as comment to top of file
2856         if (defined(my $descr = $conf->{description})) {
2857             if ($descr) {
2858                 foreach my $cl (split(/\n/, $descr)) {
2859                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2860                 }
2861             } else {
2862                 $raw .= "#\n" if $pending;
2863             }
2864         }
2865
2866         foreach my $key (sort keys %$conf) {
2867             next if $key eq 'digest' || $key eq 'description' || $key eq 'pending' || $key eq 'snapshots';
2868             $raw .= "$key: $conf->{$key}\n";
2869         }
2870         return $raw;
2871     };
2872
2873     my $raw = &$generate_raw_config($conf);
2874
2875     if (scalar(keys %{$conf->{pending}})){
2876         $raw .= "\n[PENDING]\n";
2877         $raw .= &$generate_raw_config($conf->{pending}, 1);
2878     }
2879
2880     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2881         $raw .= "\n[$snapname]\n";
2882         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2883     }
2884
2885     return $raw;
2886 }
2887
2888 sub load_defaults {
2889
2890     my $res = {};
2891
2892     # we use static defaults from our JSON schema configuration
2893     foreach my $key (keys %$confdesc) {
2894         if (defined(my $default = $confdesc->{$key}->{default})) {
2895             $res->{$key} = $default;
2896         }
2897     }
2898
2899     return $res;
2900 }
2901
2902 sub config_list {
2903     my $vmlist = PVE::Cluster::get_vmlist();
2904     my $res = {};
2905     return $res if !$vmlist || !$vmlist->{ids};
2906     my $ids = $vmlist->{ids};
2907
2908     foreach my $vmid (keys %$ids) {
2909         my $d = $ids->{$vmid};
2910         next if !$d->{node} || $d->{node} ne $nodename;
2911         next if !$d->{type} || $d->{type} ne 'qemu';
2912         $res->{$vmid}->{exists} = 1;
2913     }
2914     return $res;
2915 }
2916
2917 # test if VM uses local resources (to prevent migration)
2918 sub check_local_resources {
2919     my ($conf, $noerr) = @_;
2920
2921     my @loc_res = ();
2922
2923     push @loc_res, "hostusb" if $conf->{hostusb}; # old syntax
2924     push @loc_res, "hostpci" if $conf->{hostpci}; # old syntax
2925
2926     push @loc_res, "ivshmem" if $conf->{ivshmem};
2927
2928     foreach my $k (keys %$conf) {
2929         next if $k =~ m/^usb/ && ($conf->{$k} =~ m/^spice(?![^,])/);
2930         # sockets are safe: they will recreated be on the target side post-migrate
2931         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2932         push @loc_res, $k if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2933     }
2934
2935     die "VM uses local resources\n" if scalar @loc_res && !$noerr;
2936
2937     return \@loc_res;
2938 }
2939
2940 # check if used storages are available on all nodes (use by migrate)
2941 sub check_storage_availability {
2942     my ($storecfg, $conf, $node) = @_;
2943
2944     foreach_drive($conf, sub {
2945         my ($ds, $drive) = @_;
2946
2947         my $volid = $drive->{file};
2948         return if !$volid;
2949
2950         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2951         return if !$sid;
2952
2953         # check if storage is available on both nodes
2954         my $scfg = PVE::Storage::storage_check_node($storecfg, $sid);
2955         PVE::Storage::storage_check_node($storecfg, $sid, $node);
2956    });
2957 }
2958
2959 # list nodes where all VM images are available (used by has_feature API)
2960 sub shared_nodes {
2961     my ($conf, $storecfg) = @_;
2962
2963     my $nodelist = PVE::Cluster::get_nodelist();
2964     my $nodehash = { map { $_ => 1 } @$nodelist };
2965     my $nodename = PVE::INotify::nodename();
2966
2967     foreach_drive($conf, sub {
2968         my ($ds, $drive) = @_;
2969
2970         my $volid = $drive->{file};
2971         return if !$volid;
2972
2973         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2974         if ($storeid) {
2975             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2976             if ($scfg->{disable}) {
2977                 $nodehash = {};
2978             } elsif (my $avail = $scfg->{nodes}) {
2979                 foreach my $node (keys %$nodehash) {
2980                     delete $nodehash->{$node} if !$avail->{$node};
2981                 }
2982             } elsif (!$scfg->{shared}) {
2983                 foreach my $node (keys %$nodehash) {
2984                     delete $nodehash->{$node} if $node ne $nodename
2985                 }
2986             }
2987         }
2988     });
2989
2990     return $nodehash
2991 }
2992
2993 sub check_local_storage_availability {
2994     my ($conf, $storecfg) = @_;
2995
2996     my $nodelist = PVE::Cluster::get_nodelist();
2997     my $nodehash = { map { $_ => {} } @$nodelist };
2998
2999     foreach_drive($conf, sub {
3000         my ($ds, $drive) = @_;
3001
3002         my $volid = $drive->{file};
3003         return if !$volid;
3004
3005         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
3006         if ($storeid) {
3007             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
3008
3009             if ($scfg->{disable}) {
3010                 foreach my $node (keys %$nodehash) {
3011                     $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
3012                 }
3013             } elsif (my $avail = $scfg->{nodes}) {
3014                 foreach my $node (keys %$nodehash) {
3015                     if (!$avail->{$node}) {
3016                         $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
3017                     }
3018                 }
3019             }
3020         }
3021     });
3022
3023     foreach my $node (values %$nodehash) {
3024         if (my $unavail = $node->{unavailable_storages}) {
3025             $node->{unavailable_storages} = [ sort keys %$unavail ];
3026         }
3027     }
3028
3029     return $nodehash
3030 }
3031
3032 sub check_cmdline {
3033     my ($pidfile, $pid) = @_;
3034
3035     my $fh = IO::File->new("/proc/$pid/cmdline", "r");
3036     if (defined($fh)) {
3037         my $line = <$fh>;
3038         $fh->close;
3039         return undef if !$line;
3040         my @param = split(/\0/, $line);
3041
3042         my $cmd = $param[0];
3043         return if !$cmd || ($cmd !~ m|kvm$| && $cmd !~ m@(?:^|/)qemu-system-[^/]+$@);
3044
3045         for (my $i = 0; $i < scalar (@param); $i++) {
3046             my $p = $param[$i];
3047             next if !$p;
3048             if (($p eq '-pidfile') || ($p eq '--pidfile')) {
3049                 my $p = $param[$i+1];
3050                 return 1 if $p && ($p eq $pidfile);
3051                 return undef;
3052             }
3053         }
3054     }
3055     return undef;
3056 }
3057
3058 sub check_running {
3059     my ($vmid, $nocheck, $node) = @_;
3060
3061     my $filename = PVE::QemuConfig->config_file($vmid, $node);
3062
3063     die "unable to find configuration file for VM $vmid - no such machine\n"
3064         if !$nocheck && ! -f $filename;
3065
3066     my $pidfile = pidfile_name($vmid);
3067
3068     if (my $fd = IO::File->new("<$pidfile")) {
3069         my $st = stat($fd);
3070         my $line = <$fd>;
3071         close($fd);
3072
3073         my $mtime = $st->mtime;
3074         if ($mtime > time()) {
3075             warn "file '$filename' modified in future\n";
3076         }
3077
3078         if ($line =~ m/^(\d+)$/) {
3079             my $pid = $1;
3080             if (check_cmdline($pidfile, $pid)) {
3081                 if (my $pinfo = PVE::ProcFSTools::check_process_running($pid)) {
3082                     return $pid;
3083                 }
3084             }
3085         }
3086     }
3087
3088     return undef;
3089 }
3090
3091 sub vzlist {
3092
3093     my $vzlist = config_list();
3094
3095     my $fd = IO::Dir->new($var_run_tmpdir) || return $vzlist;
3096
3097     while (defined(my $de = $fd->read)) {
3098         next if $de !~ m/^(\d+)\.pid$/;
3099         my $vmid = $1;
3100         next if !defined($vzlist->{$vmid});
3101         if (my $pid = check_running($vmid)) {
3102             $vzlist->{$vmid}->{pid} = $pid;
3103         }
3104     }
3105
3106     return $vzlist;
3107 }
3108
3109 sub disksize {
3110     my ($storecfg, $conf) = @_;
3111
3112     my $bootdisk = $conf->{bootdisk};
3113     return undef if !$bootdisk;
3114     return undef if !is_valid_drivename($bootdisk);
3115
3116     return undef if !$conf->{$bootdisk};
3117
3118     my $drive = parse_drive($bootdisk, $conf->{$bootdisk});
3119     return undef if !defined($drive);
3120
3121     return undef if drive_is_cdrom($drive);
3122
3123     my $volid = $drive->{file};
3124     return undef if !$volid;
3125
3126     return $drive->{size};
3127 }
3128
3129 our $vmstatus_return_properties = {
3130     vmid => get_standard_option('pve-vmid'),
3131     status => {
3132         description => "Qemu process status.",
3133         type => 'string',
3134         enum => ['stopped', 'running'],
3135     },
3136     maxmem => {
3137         description => "Maximum memory in bytes.",
3138         type => 'integer',
3139         optional => 1,
3140         renderer => 'bytes',
3141     },
3142     maxdisk => {
3143         description => "Root disk size in bytes.",
3144         type => 'integer',
3145         optional => 1,
3146         renderer => 'bytes',
3147     },
3148     name => {
3149         description => "VM name.",
3150         type => 'string',
3151         optional => 1,
3152     },
3153     qmpstatus => {
3154         description => "Qemu QMP agent status.",
3155         type => 'string',
3156         optional => 1,
3157     },
3158     pid => {
3159         description => "PID of running qemu process.",
3160         type => 'integer',
3161         optional => 1,
3162     },
3163     uptime => {
3164         description => "Uptime.",
3165         type => 'integer',
3166         optional => 1,
3167         renderer => 'duration',
3168     },
3169     cpus => {
3170         description => "Maximum usable CPUs.",
3171         type => 'number',
3172         optional => 1,
3173     },
3174     lock => {
3175         description => "The current config lock, if any.",
3176         type => 'string',
3177         optional => 1,
3178     }
3179 };
3180
3181 my $last_proc_pid_stat;
3182
3183 # get VM status information
3184 # This must be fast and should not block ($full == false)
3185 # We only query KVM using QMP if $full == true (this can be slow)
3186 sub vmstatus {
3187     my ($opt_vmid, $full) = @_;
3188
3189     my $res = {};
3190
3191     my $storecfg = PVE::Storage::config();
3192
3193     my $list = vzlist();
3194     my $defaults = load_defaults();
3195
3196     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
3197
3198     my $cpucount = $cpuinfo->{cpus} || 1;
3199
3200     foreach my $vmid (keys %$list) {
3201         next if $opt_vmid && ($vmid ne $opt_vmid);
3202
3203         my $cfspath = PVE::QemuConfig->cfs_config_path($vmid);
3204         my $conf = PVE::Cluster::cfs_read_file($cfspath) || {};
3205
3206         my $d = { vmid => $vmid };
3207         $d->{pid} = $list->{$vmid}->{pid};
3208
3209         # fixme: better status?
3210         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
3211
3212         my $size = disksize($storecfg, $conf);
3213         if (defined($size)) {
3214             $d->{disk} = 0; # no info available
3215             $d->{maxdisk} = $size;
3216         } else {
3217             $d->{disk} = 0;
3218             $d->{maxdisk} = 0;
3219         }
3220
3221         $d->{cpus} = ($conf->{sockets} || $defaults->{sockets})
3222             * ($conf->{cores} || $defaults->{cores});
3223         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
3224         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
3225
3226         $d->{name} = $conf->{name} || "VM $vmid";
3227         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024)
3228             : $defaults->{memory}*(1024*1024);
3229
3230         if ($conf->{balloon}) {
3231             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
3232             $d->{shares} = defined($conf->{shares}) ? $conf->{shares}
3233                 : $defaults->{shares};
3234         }
3235
3236         $d->{uptime} = 0;
3237         $d->{cpu} = 0;
3238         $d->{mem} = 0;
3239
3240         $d->{netout} = 0;
3241         $d->{netin} = 0;
3242
3243         $d->{diskread} = 0;
3244         $d->{diskwrite} = 0;
3245
3246         $d->{template} = PVE::QemuConfig->is_template($conf);
3247
3248         $d->{serial} = 1 if conf_has_serial($conf);
3249         $d->{lock} = $conf->{lock} if $conf->{lock};
3250
3251         $res->{$vmid} = $d;
3252     }
3253
3254     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
3255     foreach my $dev (keys %$netdev) {
3256         next if $dev !~ m/^tap([1-9]\d*)i/;
3257         my $vmid = $1;
3258         my $d = $res->{$vmid};
3259         next if !$d;
3260
3261         $d->{netout} += $netdev->{$dev}->{receive};
3262         $d->{netin} += $netdev->{$dev}->{transmit};
3263
3264         if ($full) {
3265             $d->{nics}->{$dev}->{netout} = $netdev->{$dev}->{receive};
3266             $d->{nics}->{$dev}->{netin} = $netdev->{$dev}->{transmit};
3267         }
3268
3269     }
3270
3271     my $ctime = gettimeofday;
3272
3273     foreach my $vmid (keys %$list) {
3274
3275         my $d = $res->{$vmid};
3276         my $pid = $d->{pid};
3277         next if !$pid;
3278
3279         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
3280         next if !$pstat; # not running
3281
3282         my $used = $pstat->{utime} + $pstat->{stime};
3283
3284         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
3285
3286         if ($pstat->{vsize}) {
3287             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
3288         }
3289
3290         my $old = $last_proc_pid_stat->{$pid};
3291         if (!$old) {
3292             $last_proc_pid_stat->{$pid} = {
3293                 time => $ctime,
3294                 used => $used,
3295                 cpu => 0,
3296             };
3297             next;
3298         }
3299
3300         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
3301
3302         if ($dtime > 1000) {
3303             my $dutime = $used -  $old->{used};
3304
3305             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
3306             $last_proc_pid_stat->{$pid} = {
3307                 time => $ctime,
3308                 used => $used,
3309                 cpu => $d->{cpu},
3310             };
3311         } else {
3312             $d->{cpu} = $old->{cpu};
3313         }
3314     }
3315
3316     return $res if !$full;
3317
3318     my $qmpclient = PVE::QMPClient->new();
3319
3320     my $ballooncb = sub {
3321         my ($vmid, $resp) = @_;
3322
3323         my $info = $resp->{'return'};
3324         return if !$info->{max_mem};
3325
3326         my $d = $res->{$vmid};
3327
3328         # use memory assigned to VM
3329         $d->{maxmem} = $info->{max_mem};
3330         $d->{balloon} = $info->{actual};
3331
3332         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
3333             $d->{mem} = $info->{total_mem} - $info->{free_mem};
3334             $d->{freemem} = $info->{free_mem};
3335         }
3336
3337         $d->{ballooninfo} = $info;
3338     };
3339
3340     my $blockstatscb = sub {
3341         my ($vmid, $resp) = @_;
3342         my $data = $resp->{'return'} || [];
3343         my $totalrdbytes = 0;
3344         my $totalwrbytes = 0;
3345
3346         for my $blockstat (@$data) {
3347             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
3348             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
3349
3350             $blockstat->{device} =~ s/drive-//;
3351             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
3352         }
3353         $res->{$vmid}->{diskread} = $totalrdbytes;
3354         $res->{$vmid}->{diskwrite} = $totalwrbytes;
3355     };
3356
3357     my $statuscb = sub {
3358         my ($vmid, $resp) = @_;
3359
3360         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
3361         # this fails if ballon driver is not loaded, so this must be
3362         # the last commnand (following command are aborted if this fails).
3363         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
3364
3365         my $status = 'unknown';
3366         if (!defined($status = $resp->{'return'}->{status})) {
3367             warn "unable to get VM status\n";
3368             return;
3369         }
3370
3371         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
3372     };
3373
3374     foreach my $vmid (keys %$list) {
3375         next if $opt_vmid && ($vmid ne $opt_vmid);
3376         next if !$res->{$vmid}->{pid}; # not running
3377         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
3378     }
3379
3380     $qmpclient->queue_execute(undef, 2);
3381
3382     foreach my $vmid (keys %$list) {
3383         next if $opt_vmid && ($vmid ne $opt_vmid);
3384         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
3385     }
3386
3387     return $res;
3388 }
3389
3390 sub foreach_drive {
3391     my ($conf, $func, @param) = @_;
3392
3393     foreach my $ds (valid_drive_names()) {
3394         next if !defined($conf->{$ds});
3395
3396         my $drive = parse_drive($ds, $conf->{$ds});
3397         next if !$drive;
3398
3399         &$func($ds, $drive, @param);
3400     }
3401 }
3402
3403 sub foreach_volid {
3404     my ($conf, $func, @param) = @_;
3405
3406     my $volhash = {};
3407
3408     my $test_volid = sub {
3409         my ($volid, $is_cdrom, $replicate, $shared, $snapname, $size) = @_;
3410
3411         return if !$volid;
3412
3413         $volhash->{$volid}->{cdrom} //= 1;
3414         $volhash->{$volid}->{cdrom} = 0 if !$is_cdrom;
3415
3416         $volhash->{$volid}->{replicate} //= 0;
3417         $volhash->{$volid}->{replicate} = 1 if $replicate;
3418
3419         $volhash->{$volid}->{shared} //= 0;
3420         $volhash->{$volid}->{shared} = 1 if $shared;
3421
3422         $volhash->{$volid}->{referenced_in_config} //= 0;
3423         $volhash->{$volid}->{referenced_in_config} = 1 if !defined($snapname);
3424
3425         $volhash->{$volid}->{referenced_in_snapshot}->{$snapname} = 1
3426             if defined($snapname);
3427         $volhash->{$volid}->{size} = $size if $size;
3428     };
3429
3430     foreach_drive($conf, sub {
3431         my ($ds, $drive) = @_;
3432         $test_volid->($drive->{file}, drive_is_cdrom($drive), $drive->{replicate} // 1, $drive->{shared}, undef, $drive->{size});
3433     });
3434
3435     foreach my $snapname (keys %{$conf->{snapshots}}) {
3436         my $snap = $conf->{snapshots}->{$snapname};
3437         $test_volid->($snap->{vmstate}, 0, 1, $snapname);
3438         foreach_drive($snap, sub {
3439             my ($ds, $drive) = @_;
3440             $test_volid->($drive->{file}, drive_is_cdrom($drive), $drive->{replicate} // 1, $drive->{shared}, $snapname);
3441         });
3442     }
3443
3444     foreach my $volid (keys %$volhash) {
3445         &$func($volid, $volhash->{$volid}, @param);
3446     }
3447 }
3448
3449 sub conf_has_serial {
3450     my ($conf) = @_;
3451
3452     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3453         if ($conf->{"serial$i"}) {
3454             return 1;
3455         }
3456     }
3457
3458     return 0;
3459 }
3460
3461 sub conf_has_audio {
3462     my ($conf, $id) = @_;
3463
3464     $id //= 0;
3465     my $audio = $conf->{"audio$id"};
3466     return undef if !defined($audio);
3467
3468     my $audioproperties = PVE::JSONSchema::parse_property_string($audio_fmt, $audio);
3469     my $audiodriver = $audioproperties->{driver} // 'spice';
3470
3471     return {
3472         dev => $audioproperties->{device},
3473         dev_id => "audiodev$id",
3474         backend => $audiodriver,
3475         backend_id => "$audiodriver-backend${id}",
3476     };
3477 }
3478
3479 sub vga_conf_has_spice {
3480     my ($vga) = @_;
3481
3482     my $vgaconf = parse_vga($vga);
3483     my $vgatype = $vgaconf->{type};
3484     return 0 if !$vgatype || $vgatype !~ m/^qxl([234])?$/;
3485
3486     return $1 || 1;
3487 }
3488
3489 my $host_arch; # FIXME: fix PVE::Tools::get_host_arch
3490 sub get_host_arch() {
3491     $host_arch = (POSIX::uname())[4] if !$host_arch;
3492     return $host_arch;
3493 }
3494
3495 sub is_native($) {
3496     my ($arch) = @_;
3497     return get_host_arch() eq $arch;
3498 }
3499
3500 my $default_machines = {
3501     x86_64 => 'pc',
3502     aarch64 => 'virt',
3503 };
3504
3505 sub get_basic_machine_info {
3506     my ($conf, $forcemachine) = @_;
3507
3508     my $arch = $conf->{arch} // get_host_arch();
3509     my $machine = $forcemachine || $conf->{machine} || $default_machines->{$arch};
3510     return ($arch, $machine);
3511 }
3512
3513 sub get_ovmf_files($) {
3514     my ($arch) = @_;
3515
3516     my $ovmf = $OVMF->{$arch}
3517         or die "no OVMF images known for architecture '$arch'\n";
3518
3519     return @$ovmf;
3520 }
3521
3522 my $Arch2Qemu = {
3523     aarch64 => '/usr/bin/qemu-system-aarch64',
3524     x86_64 => '/usr/bin/qemu-system-x86_64',
3525 };
3526 sub get_command_for_arch($) {
3527     my ($arch) = @_;
3528     return '/usr/bin/kvm' if is_native($arch);
3529
3530     my $cmd = $Arch2Qemu->{$arch}
3531         or die "don't know how to emulate architecture '$arch'\n";
3532     return $cmd;
3533 }
3534
3535 sub get_cpu_options {
3536     my ($conf, $arch, $kvm, $machine_type, $kvm_off, $kvmver, $winversion, $gpu_passthrough) = @_;
3537
3538     my $cpuFlags = [];
3539     my $ostype = $conf->{ostype};
3540
3541     my $cpu = $kvm ? "kvm64" : "qemu64";
3542     if ($arch eq 'aarch64') {
3543         $cpu = 'cortex-a57';
3544     }
3545     my $hv_vendor_id;
3546     if (my $cputype = $conf->{cpu}) {
3547         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
3548             or die "Cannot parse cpu description: $cputype\n";
3549         $cpu = $cpuconf->{cputype};
3550         $kvm_off = 1 if $cpuconf->{hidden};
3551         $hv_vendor_id = $cpuconf->{'hv-vendor-id'};
3552
3553         if (defined(my $flags = $cpuconf->{flags})) {
3554             push @$cpuFlags, split(";", $flags);
3555         }
3556     }
3557
3558     push @$cpuFlags , '+lahf_lm' if $cpu eq 'kvm64' && $arch eq 'x86_64';
3559
3560     push @$cpuFlags , '-x2apic'
3561         if $conf->{ostype} && $conf->{ostype} eq 'solaris';
3562
3563     push @$cpuFlags, '+sep' if $cpu eq 'kvm64' || $cpu eq 'kvm32';
3564
3565     push @$cpuFlags, '-rdtscp' if $cpu =~ m/^Opteron/;
3566
3567     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3) && $arch eq 'x86_64') {
3568
3569         push @$cpuFlags , '+kvm_pv_unhalt' if $kvm;
3570         push @$cpuFlags , '+kvm_pv_eoi' if $kvm;
3571     }
3572
3573     add_hyperv_enlightenments($cpuFlags, $winversion, $machine_type, $kvmver, $conf->{bios}, $gpu_passthrough, $hv_vendor_id) if $kvm;
3574
3575     push @$cpuFlags, 'enforce' if $cpu ne 'host' && $kvm && $arch eq 'x86_64';
3576
3577     push @$cpuFlags, 'kvm=off' if $kvm_off;
3578
3579     if (my $cpu_vendor = $cpu_vendor_list->{$cpu}) {
3580         push @$cpuFlags, "vendor=${cpu_vendor}"
3581             if $cpu_vendor ne 'default';
3582     } elsif ($arch ne 'aarch64') {
3583         die "internal error"; # should not happen
3584     }
3585
3586     $cpu .= "," . join(',', @$cpuFlags) if scalar(@$cpuFlags);
3587
3588     return ('-cpu', $cpu);
3589 }
3590
3591 sub config_to_command {
3592     my ($storecfg, $vmid, $conf, $defaults, $forcemachine) = @_;
3593
3594     my $cmd = [];
3595     my $globalFlags = [];
3596     my $machineFlags = [];
3597     my $rtcFlags = [];
3598     my $devices = [];
3599     my $pciaddr = '';
3600     my $bridges = {};
3601     my $vernum = 0; # unknown
3602     my $ostype = $conf->{ostype};
3603     my $winversion = windows_version($ostype);
3604     my $kvm = $conf->{kvm};
3605
3606     my ($arch, $machine_type) = get_basic_machine_info($conf, $forcemachine);
3607     my $kvm_binary = get_command_for_arch($arch);
3608     my $kvmver = kvm_user_version($kvm_binary);
3609     $kvm //= 1 if is_native($arch);
3610
3611     if ($kvm) {
3612         die "KVM virtualisation configured, but not available. Either disable in VM configuration or enable in BIOS.\n"
3613             if !defined kvm_version();
3614     }
3615
3616     if ($kvmver =~ m/^(\d+)\.(\d+)$/) {
3617         $vernum = $1*1000000+$2*1000;
3618     } elsif ($kvmver =~ m/^(\d+)\.(\d+)\.(\d+)$/) {
3619         $vernum = $1*1000000+$2*1000+$3;
3620     }
3621
3622     die "detected old qemu-kvm binary ($kvmver)\n" if $vernum < 15000;
3623
3624     my $q35 = machine_type_is_q35($conf);
3625     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
3626     my $use_old_bios_files = undef;
3627     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3628
3629     my $cpuunits = defined($conf->{cpuunits}) ?
3630             $conf->{cpuunits} : $defaults->{cpuunits};
3631
3632     push @$cmd, $kvm_binary;
3633
3634     push @$cmd, '-id', $vmid;
3635
3636     my $vmname = $conf->{name} || "vm$vmid";
3637
3638     push @$cmd, '-name', $vmname;
3639
3640     my $use_virtio = 0;
3641
3642     my $qmpsocket = qmp_socket($vmid);
3643     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server,nowait";
3644     push @$cmd, '-mon', "chardev=qmp,mode=control";
3645
3646     if (qemu_machine_feature_enabled($machine_type, $kvmver, 2, 12)) {
3647         push @$cmd, '-chardev', "socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5";
3648         push @$cmd, '-mon', "chardev=qmp-event,mode=control";
3649     }
3650
3651     push @$cmd, '-pidfile' , pidfile_name($vmid);
3652
3653     push @$cmd, '-daemonize';
3654
3655     if ($conf->{smbios1}) {
3656         my $smbios_conf = parse_smbios1($conf->{smbios1});
3657         if ($smbios_conf->{base64}) {
3658             # Do not pass base64 flag to qemu
3659             delete $smbios_conf->{base64};
3660             my $smbios_string = "";
3661             foreach my $key (keys %$smbios_conf) {
3662                 my $value;
3663                 if ($key eq "uuid") {
3664                     $value = $smbios_conf->{uuid}
3665                 } else {
3666                     $value = decode_base64($smbios_conf->{$key});
3667                 }
3668                 # qemu accepts any binary data, only commas need escaping by double comma
3669                 $value =~ s/,/,,/g;
3670                 $smbios_string .= "," . $key . "=" . $value if $value;
3671             }
3672             push @$cmd, '-smbios', "type=1" . $smbios_string;
3673         } else {
3674             push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
3675         }
3676     }
3677
3678     if ($conf->{vmgenid}) {
3679         push @$devices, '-device', 'vmgenid,guid='.$conf->{vmgenid};
3680     }
3681
3682     my ($ovmf_code, $ovmf_vars) = get_ovmf_files($arch);
3683     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3684         die "uefi base image not found\n" if ! -f $ovmf_code;
3685
3686         my $path;
3687         my $format;
3688         if (my $efidisk = $conf->{efidisk0}) {
3689             my $d = PVE::JSONSchema::parse_property_string($efidisk_fmt, $efidisk);
3690             my ($storeid, $volname) = PVE::Storage::parse_volume_id($d->{file}, 1);
3691             $format = $d->{format};
3692             if ($storeid) {
3693                 $path = PVE::Storage::path($storecfg, $d->{file});
3694                 if (!defined($format)) {
3695                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
3696                     $format = qemu_img_format($scfg, $volname);
3697                 }
3698             } else {
3699                 $path = $d->{file};
3700                 die "efidisk format must be specified\n"
3701                     if !defined($format);
3702             }
3703         } else {
3704             warn "no efidisk configured! Using temporary efivars disk.\n";
3705             $path = "/tmp/$vmid-ovmf.fd";
3706             PVE::Tools::file_copy($ovmf_vars, $path, -s $ovmf_vars);
3707             $format = 'raw';
3708         }
3709
3710         push @$cmd, '-drive', "if=pflash,unit=0,format=raw,readonly,file=$ovmf_code";
3711         push @$cmd, '-drive', "if=pflash,unit=1,format=$format,id=drive-efidisk0,file=$path";
3712     }
3713
3714     # load q35 config
3715     if ($q35) {
3716         # we use different pcie-port hardware for qemu >= 4.0 for passthrough
3717         if (qemu_machine_feature_enabled($machine_type, $kvmver, 4, 0)) {
3718             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35-4.0.cfg';
3719         } else {
3720             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35.cfg';
3721         }
3722     }
3723
3724     # add usb controllers
3725     my @usbcontrollers = PVE::QemuServer::USB::get_usb_controllers($conf, $bridges, $arch, $machine_type, $usbdesc->{format}, $MAX_USB_DEVICES);
3726     push @$devices, @usbcontrollers if @usbcontrollers;
3727     my $vga = parse_vga($conf->{vga});
3728
3729     my $qxlnum = vga_conf_has_spice($conf->{vga});
3730     $vga->{type} = 'qxl' if $qxlnum;
3731
3732     if (!$vga->{type}) {
3733         if ($arch eq 'aarch64') {
3734             $vga->{type} = 'virtio';
3735         } elsif (qemu_machine_feature_enabled($machine_type, $kvmver, 2, 9)) {
3736             $vga->{type} = (!$winversion || $winversion >= 6) ? 'std' : 'cirrus';
3737         } else {
3738             $vga->{type} = ($winversion >= 6) ? 'std' : 'cirrus';
3739         }
3740     }
3741
3742     # enable absolute mouse coordinates (needed by vnc)
3743     my $tablet;
3744     if (defined($conf->{tablet})) {
3745         $tablet = $conf->{tablet};
3746     } else {
3747         $tablet = $defaults->{tablet};
3748         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
3749         $tablet = 0 if $vga->{type} =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
3750     }
3751
3752     if ($tablet) {
3753         push @$devices, '-device', print_tabletdevice_full($conf, $arch) if $tablet;
3754         my $kbd = print_keyboarddevice_full($conf, $arch);
3755         push @$devices, '-device', $kbd if defined($kbd);
3756     }
3757
3758     my $kvm_off = 0;
3759     my $gpu_passthrough;
3760
3761     # host pci devices
3762     for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
3763         my $id = "hostpci$i";
3764         my $d = parse_hostpci($conf->{$id});
3765         next if !$d;
3766
3767         if (my $pcie = $d->{pcie}) {
3768             die "q35 machine model is not enabled" if !$q35;
3769             # win7 wants to have the pcie devices directly on the pcie bus
3770             # instead of in the root port
3771             if ($winversion == 7) {
3772                 $pciaddr = print_pcie_addr("${id}bus0");
3773             } else {
3774                 # add more root ports if needed, 4 are present by default
3775                 # by pve-q35 cfgs, rest added here on demand.
3776                 if ($i > 3) {
3777                     push @$devices, '-device', print_pcie_root_port($i);
3778                 }
3779                 $pciaddr = print_pcie_addr($id);
3780             }
3781         } else {
3782             $pciaddr = print_pci_addr($id, $bridges, $arch, $machine_type);
3783         }
3784
3785         my $xvga = '';
3786         if ($d->{'x-vga'}) {
3787             $xvga = ',x-vga=on' if !($conf->{bios} && $conf->{bios} eq 'ovmf');
3788             $kvm_off = 1;
3789             $vga->{type} = 'none' if !defined($conf->{vga});
3790             $gpu_passthrough = 1;
3791         }
3792
3793         my $pcidevices = $d->{pciid};
3794         my $multifunction = 1 if @$pcidevices > 1;
3795
3796         my $sysfspath;
3797         if ($d->{mdev} && scalar(@$pcidevices) == 1) {
3798             my $pci_id = $pcidevices->[0]->{id};
3799             my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $i);
3800             $sysfspath = "/sys/bus/pci/devices/0000:$pci_id/$uuid";
3801         } elsif ($d->{mdev}) {
3802             warn "ignoring mediated device '$id' with multifunction device\n";
3803         }
3804
3805         my $j=0;
3806         foreach my $pcidevice (@$pcidevices) {
3807             my $devicestr = "vfio-pci";
3808
3809             if ($sysfspath) {
3810                 $devicestr .= ",sysfsdev=$sysfspath";
3811             } else {
3812                 $devicestr .= ",host=$pcidevice->{id}";
3813             }
3814
3815             my $mf_addr = $multifunction ? ".$j" : '';
3816             $devicestr .= ",id=${id}${mf_addr}${pciaddr}${mf_addr}";
3817
3818             if ($j == 0) {
3819                 $devicestr .= ',rombar=0' if defined($d->{rombar}) && !$d->{rombar};
3820                 $devicestr .= "$xvga";
3821                 $devicestr .= ",multifunction=on" if $multifunction;
3822                 $devicestr .= ",romfile=/usr/share/kvm/$d->{romfile}" if $d->{romfile};
3823             }
3824
3825             push @$devices, '-device', $devicestr;
3826             $j++;
3827         }
3828     }
3829
3830     # usb devices
3831     my $usb_dev_features = {};
3832     $usb_dev_features->{spice_usb3} = 1 if qemu_machine_feature_enabled($machine_type, $kvmver, 4, 0);
3833
3834     my @usbdevices = PVE::QemuServer::USB::get_usb_devices($conf, $usbdesc->{format}, $MAX_USB_DEVICES, $usb_dev_features);
3835     push @$devices, @usbdevices if @usbdevices;
3836     # serial devices
3837     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3838         if (my $path = $conf->{"serial$i"}) {
3839             if ($path eq 'socket') {
3840                 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
3841                 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server,nowait";
3842                 # On aarch64, serial0 is the UART device. Qemu only allows
3843                 # connecting UART devices via the '-serial' command line, as
3844                 # the device has a fixed slot on the hardware...
3845                 if ($arch eq 'aarch64' && $i == 0) {
3846                     push @$devices, '-serial', "chardev:serial$i";
3847                 } else {
3848                     push @$devices, '-device', "isa-serial,chardev=serial$i";
3849                 }
3850             } else {
3851                 die "no such serial device\n" if ! -c $path;
3852                 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
3853                 push @$devices, '-device', "isa-serial,chardev=serial$i";
3854             }
3855         }
3856     }
3857
3858     # parallel devices
3859     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
3860         if (my $path = $conf->{"parallel$i"}) {
3861             die "no such parallel device\n" if ! -c $path;
3862             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
3863             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
3864             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
3865         }
3866     }
3867
3868     if (my $audio = conf_has_audio($conf)) {
3869
3870         my $audiopciaddr = print_pci_addr("audio0", $bridges, $arch, $machine_type);
3871
3872         my $id = $audio->{dev_id};
3873         if ($audio->{dev} eq 'AC97') {
3874             push @$devices, '-device', "AC97,id=${id}${audiopciaddr}";
3875         } elsif ($audio->{dev} =~ /intel\-hda$/) {
3876             push @$devices, '-device', "$audio->{dev},id=${id}${audiopciaddr}";
3877             push @$devices, '-device', "hda-micro,id=${id}-codec0,bus=${id}.0,cad=0";
3878             push @$devices, '-device', "hda-duplex,id=${id}-codec1,bus=${id}.0,cad=1";
3879         } else {
3880             die "unkown audio device '$audio->{dev}', implement me!";
3881         }
3882
3883         push @$devices, '-audiodev', "$audio->{backend},id=$audio->{backend_id}";
3884     }
3885
3886     my $sockets = 1;
3887     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3888     $sockets = $conf->{sockets} if  $conf->{sockets};
3889
3890     my $cores = $conf->{cores} || 1;
3891
3892     my $maxcpus = $sockets * $cores;
3893
3894     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
3895
3896     my $allowed_vcpus = $cpuinfo->{cpus};
3897
3898     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n"
3899         if ($allowed_vcpus < $maxcpus);
3900
3901     if($hotplug_features->{cpu} && qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 7)) {
3902
3903         push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3904         for (my $i = 2; $i <= $vcpus; $i++)  {
3905             my $cpustr = print_cpu_device($conf,$i);
3906             push @$cmd, '-device', $cpustr;
3907         }
3908
3909     } else {
3910
3911         push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3912     }
3913     push @$cmd, '-nodefaults';
3914
3915     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
3916
3917     my $bootindex_hash = {};
3918     my $i = 1;
3919     foreach my $o (split(//, $bootorder)) {
3920         $bootindex_hash->{$o} = $i*100;
3921         $i++;
3922     }
3923
3924     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
3925
3926     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
3927
3928     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
3929
3930     if ($vga->{type} && $vga->{type} !~ m/^serial\d+$/ && $vga->{type} ne 'none'){
3931         push @$devices, '-device', print_vga_device($conf, $vga, $arch, $machine_type, undef, $qxlnum, $bridges);
3932         my $socket = vnc_socket($vmid);
3933         push @$cmd,  '-vnc', "unix:$socket,password";
3934     } else {
3935         push @$cmd, '-vga', 'none' if $vga->{type} eq 'none';
3936         push @$cmd, '-nographic';
3937     }
3938
3939     # time drift fix
3940     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
3941
3942     my $useLocaltime = $conf->{localtime};
3943
3944     if ($winversion >= 5) { # windows
3945         $useLocaltime = 1 if !defined($conf->{localtime});
3946
3947         # use time drift fix when acpi is enabled
3948         if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
3949             $tdf = 1 if !defined($conf->{tdf});
3950         }
3951     }
3952
3953     if ($winversion >= 6) {
3954         push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3955         push @$cmd, '-no-hpet';
3956     }
3957
3958     push @$rtcFlags, 'driftfix=slew' if $tdf;
3959
3960     if (!$kvm) {
3961         push @$machineFlags, 'accel=tcg';
3962     }
3963
3964     if ($machine_type) {
3965         push @$machineFlags, "type=${machine_type}";
3966     }
3967
3968     if (($conf->{startdate}) && ($conf->{startdate} ne 'now')) {
3969         push @$rtcFlags, "base=$conf->{startdate}";
3970     } elsif ($useLocaltime) {
3971         push @$rtcFlags, 'base=localtime';
3972     }
3973
3974     push @$cmd, get_cpu_options($conf, $arch, $kvm, $machine_type, $kvm_off, $kvmver, $winversion, $gpu_passthrough);
3975
3976     PVE::QemuServer::Memory::config($conf, $vmid, $sockets, $cores, $defaults, $hotplug_features, $cmd);
3977
3978     push @$cmd, '-S' if $conf->{freeze};
3979
3980     push @$cmd, '-k', $conf->{keyboard} if defined($conf->{keyboard});
3981
3982     if (parse_guest_agent($conf)->{enabled}) {
3983         my $qgasocket = qmp_socket($vmid, 1);
3984         my $pciaddr = print_pci_addr("qga0", $bridges, $arch, $machine_type);
3985         push @$devices, '-chardev', "socket,path=$qgasocket,server,nowait,id=qga0";
3986         push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3987         push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3988     }
3989
3990     my $spice_port;
3991
3992     if ($qxlnum) {
3993         if ($qxlnum > 1) {
3994             if ($winversion){
3995                 for(my $i = 1; $i < $qxlnum; $i++){
3996                     push @$devices, '-device', print_vga_device($conf, $vga, $arch, $machine_type, $i, $qxlnum, $bridges);
3997                 }
3998             } else {
3999                 # assume other OS works like Linux
4000                 my ($ram, $vram) = ("134217728", "67108864");
4001                 if ($vga->{memory}) {
4002                     $ram = PVE::Tools::convert_size($qxlnum*4*$vga->{memory}, 'mb' => 'b');
4003                     $vram = PVE::Tools::convert_size($qxlnum*2*$vga->{memory}, 'mb' => 'b');
4004                 }
4005                 push @$cmd, '-global', "qxl-vga.ram_size=$ram";
4006                 push @$cmd, '-global', "qxl-vga.vram_size=$vram";
4007             }
4008         }
4009
4010         my $pciaddr = print_pci_addr("spice", $bridges, $arch, $machine_type);
4011
4012         my $nodename = PVE::INotify::nodename();
4013         my $pfamily = PVE::Tools::get_host_address_family($nodename);
4014         my @nodeaddrs = PVE::Tools::getaddrinfo_all('localhost', family => $pfamily);
4015         die "failed to get an ip address of type $pfamily for 'localhost'\n" if !@nodeaddrs;
4016
4017         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
4018         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
4019         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
4020
4021         my $localhost = PVE::Network::addr_to_ip($nodeaddrs[0]->{addr});
4022         $spice_port = PVE::Tools::next_spice_port($pfamily, $localhost);
4023
4024         my $spice_enhancement = PVE::JSONSchema::parse_property_string($spice_enhancements_fmt, $conf->{spice_enhancements} // '');
4025         if ($spice_enhancement->{foldersharing}) {
4026             push @$devices, '-chardev', "spiceport,id=foldershare,name=org.spice-space.webdav.0";
4027             push @$devices, '-device', "virtserialport,chardev=foldershare,name=org.spice-space.webdav.0";
4028         }
4029
4030         my $spice_opts = "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";
4031         $spice_opts .= ",streaming-video=$spice_enhancement->{videostreaming}" if $spice_enhancement->{videostreaming};
4032         push @$devices, '-spice', "$spice_opts";
4033     }
4034
4035     # enable balloon by default, unless explicitly disabled
4036     if (!defined($conf->{balloon}) || $conf->{balloon}) {
4037         $pciaddr = print_pci_addr("balloon0", $bridges, $arch, $machine_type);
4038         push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
4039     }
4040
4041     if ($conf->{watchdog}) {
4042         my $wdopts = parse_watchdog($conf->{watchdog});
4043         $pciaddr = print_pci_addr("watchdog", $bridges, $arch, $machine_type);
4044         my $watchdog = $wdopts->{model} || 'i6300esb';
4045         push @$devices, '-device', "$watchdog$pciaddr";
4046         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
4047     }
4048
4049     my $vollist = [];
4050     my $scsicontroller = {};
4051     my $ahcicontroller = {};
4052     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
4053
4054     # Add iscsi initiator name if available
4055     if (my $initiator = get_initiator_name()) {
4056         push @$devices, '-iscsi', "initiator-name=$initiator";
4057     }
4058
4059     foreach_drive($conf, sub {
4060         my ($ds, $drive) = @_;
4061
4062         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
4063             push @$vollist, $drive->{file};
4064         }
4065
4066         # ignore efidisk here, already added in bios/fw handling code above
4067         return if $drive->{interface} eq 'efidisk';
4068
4069         $use_virtio = 1 if $ds =~ m/^virtio/;
4070
4071         if (drive_is_cdrom ($drive)) {
4072             if ($bootindex_hash->{d}) {
4073                 $drive->{bootindex} = $bootindex_hash->{d};
4074                 $bootindex_hash->{d} += 1;
4075             }
4076         } else {
4077             if ($bootindex_hash->{c}) {
4078                 $drive->{bootindex} = $bootindex_hash->{c} if $conf->{bootdisk} && ($conf->{bootdisk} eq $ds);
4079                 $bootindex_hash->{c} += 1;
4080             }
4081         }
4082
4083         if($drive->{interface} eq 'virtio'){
4084            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
4085         }
4086
4087         if ($drive->{interface} eq 'scsi') {
4088
4089             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
4090
4091             $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges, $arch, $machine_type);
4092             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
4093
4094             my $iothread = '';
4095             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
4096                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
4097                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
4098             } elsif ($drive->{iothread}) {
4099                 warn "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n";
4100             }
4101
4102             my $queues = '';
4103             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
4104                 $queues = ",num_queues=$drive->{queues}";
4105             }
4106
4107             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues" if !$scsicontroller->{$controller};
4108             $scsicontroller->{$controller}=1;
4109         }
4110
4111         if ($drive->{interface} eq 'sata') {
4112            my $controller = int($drive->{index} / $MAX_SATA_DISKS);
4113            $pciaddr = print_pci_addr("ahci$controller", $bridges, $arch, $machine_type);
4114            push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr" if !$ahcicontroller->{$controller};
4115            $ahcicontroller->{$controller}=1;
4116         }
4117
4118         my $drive_cmd = print_drive_full($storecfg, $vmid, $drive);
4119         push @$devices, '-drive',$drive_cmd;
4120         push @$devices, '-device', print_drivedevice_full($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type);
4121     });
4122
4123     for (my $i = 0; $i < $MAX_NETS; $i++) {
4124          next if !$conf->{"net$i"};
4125          my $d = parse_net($conf->{"net$i"});
4126          next if !$d;
4127
4128          $use_virtio = 1 if $d->{model} eq 'virtio';
4129
4130          if ($bootindex_hash->{n}) {
4131             $d->{bootindex} = $bootindex_hash->{n};
4132             $bootindex_hash->{n} += 1;
4133          }
4134
4135          my $netdevfull = print_netdev_full($vmid, $conf, $arch, $d, "net$i");
4136          push @$devices, '-netdev', $netdevfull;
4137
4138          my $netdevicefull = print_netdevice_full($vmid, $conf, $d, "net$i", $bridges, $use_old_bios_files, $arch, $machine_type);
4139          push @$devices, '-device', $netdevicefull;
4140     }
4141
4142     if ($conf->{ivshmem}) {
4143         my $ivshmem = PVE::JSONSchema::parse_property_string($ivshmem_fmt, $conf->{ivshmem});
4144
4145         my $bus;
4146         if ($q35) {
4147             $bus = print_pcie_addr("ivshmem");
4148         } else {
4149             $bus = print_pci_addr("ivshmem", $bridges, $arch, $machine_type);
4150         }
4151
4152         my $ivshmem_name = $ivshmem->{name} // $vmid;
4153         my $path = '/dev/shm/pve-shm-' . $ivshmem_name;
4154
4155         push @$devices, '-device', "ivshmem-plain,memdev=ivshmem$bus,";
4156         push @$devices, '-object', "memory-backend-file,id=ivshmem,share=on,mem-path=$path,size=$ivshmem->{size}M";
4157     }
4158
4159     if (!$q35) {
4160         # add pci bridges
4161         if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
4162            $bridges->{1} = 1;
4163            $bridges->{2} = 1;
4164         }
4165
4166         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
4167
4168         for my $k (sort {$b cmp $a} keys %$bridges) {
4169             $pciaddr = print_pci_addr("pci.$k", undef, $arch, $machine_type);
4170             unshift @$devices, '-device', "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr" if $k > 0;
4171         }
4172     }
4173
4174     push @$cmd, @$devices;
4175     push @$cmd, '-rtc', join(',', @$rtcFlags)
4176         if scalar(@$rtcFlags);
4177     push @$cmd, '-machine', join(',', @$machineFlags)
4178         if scalar(@$machineFlags);
4179     push @$cmd, '-global', join(',', @$globalFlags)
4180         if scalar(@$globalFlags);
4181
4182     if (my $vmstate = $conf->{vmstate}) {
4183         my $statepath = PVE::Storage::path($storecfg, $vmstate);
4184         push @$vollist, $statepath;
4185         push @$cmd, '-loadstate', $statepath;
4186     }
4187
4188     # add custom args
4189     if ($conf->{args}) {
4190         my $aa = PVE::Tools::split_args($conf->{args});
4191         push @$cmd, @$aa;
4192     }
4193
4194     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
4195 }
4196
4197 sub vnc_socket {
4198     my ($vmid) = @_;
4199     return "${var_run_tmpdir}/$vmid.vnc";
4200 }
4201
4202 sub spice_port {
4203     my ($vmid) = @_;
4204
4205     my $res = vm_mon_cmd($vmid, 'query-spice');
4206
4207     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
4208 }
4209
4210 sub qmp_socket {
4211     my ($vmid, $qga, $name) = @_;
4212     my $sockettype = $qga ? 'qga' : 'qmp';
4213     my $ext = $name ? '-'.$name : '';
4214     return "${var_run_tmpdir}/$vmid$ext.$sockettype";
4215 }
4216
4217 sub pidfile_name {
4218     my ($vmid) = @_;
4219     return "${var_run_tmpdir}/$vmid.pid";
4220 }
4221
4222 sub vm_devices_list {
4223     my ($vmid) = @_;
4224
4225     my $res = vm_mon_cmd($vmid, 'query-pci');
4226     my $devices_to_check = [];
4227     my $devices = {};
4228     foreach my $pcibus (@$res) {
4229         push @$devices_to_check, @{$pcibus->{devices}},
4230     }
4231
4232     while (@$devices_to_check) {
4233         my $to_check = [];
4234         for my $d (@$devices_to_check) {
4235             $devices->{$d->{'qdev_id'}} = 1 if $d->{'qdev_id'};
4236             next if !$d->{'pci_bridge'};
4237
4238             $devices->{$d->{'qdev_id'}} += scalar(@{$d->{'pci_bridge'}->{devices}});
4239             push @$to_check, @{$d->{'pci_bridge'}->{devices}};
4240         }
4241         $devices_to_check = $to_check;
4242     }
4243
4244     my $resblock = vm_mon_cmd($vmid, 'query-block');
4245     foreach my $block (@$resblock) {
4246         if($block->{device} =~ m/^drive-(\S+)/){
4247                 $devices->{$1} = 1;
4248         }
4249     }
4250
4251     my $resmice = vm_mon_cmd($vmid, 'query-mice');
4252     foreach my $mice (@$resmice) {
4253         if ($mice->{name} eq 'QEMU HID Tablet') {
4254             $devices->{tablet} = 1;
4255             last;
4256         }
4257     }
4258
4259     # for usb devices there is no query-usb
4260     # but we can iterate over the entries in
4261     # qom-list path=/machine/peripheral
4262     my $resperipheral = vm_mon_cmd($vmid, 'qom-list', path => '/machine/peripheral');
4263     foreach my $per (@$resperipheral) {
4264         if ($per->{name} =~ m/^usb\d+$/) {
4265             $devices->{$per->{name}} = 1;
4266         }
4267     }
4268
4269     return $devices;
4270 }
4271
4272 sub vm_deviceplug {
4273     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4274
4275     my $q35 = machine_type_is_q35($conf);
4276
4277     my $devices_list = vm_devices_list($vmid);
4278     return 1 if defined($devices_list->{$deviceid});
4279
4280     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid, $arch, $machine_type); # add PCI bridge if we need it for the device
4281
4282     if ($deviceid eq 'tablet') {
4283
4284         qemu_deviceadd($vmid, print_tabletdevice_full($conf, $arch));
4285
4286     } elsif ($deviceid eq 'keyboard') {
4287
4288         qemu_deviceadd($vmid, print_keyboarddevice_full($conf, $arch));
4289
4290     } elsif ($deviceid =~ m/^usb(\d+)$/) {
4291
4292         die "usb hotplug currently not reliable\n";
4293         # since we can't reliably hot unplug all added usb devices
4294         # and usb passthrough disables live migration
4295         # we disable usb hotplugging for now
4296         qemu_deviceadd($vmid, PVE::QemuServer::USB::print_usbdevice_full($conf, $deviceid, $device));
4297
4298     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4299
4300         qemu_iothread_add($vmid, $deviceid, $device);
4301
4302         qemu_driveadd($storecfg, $vmid, $device);
4303         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, $arch, $machine_type);
4304
4305         qemu_deviceadd($vmid, $devicefull);
4306         eval { qemu_deviceaddverify($vmid, $deviceid); };
4307         if (my $err = $@) {
4308             eval { qemu_drivedel($vmid, $deviceid); };
4309             warn $@ if $@;
4310             die $err;
4311         }
4312
4313     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4314
4315
4316         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
4317         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
4318         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
4319
4320         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
4321
4322         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
4323             qemu_iothread_add($vmid, $deviceid, $device);
4324             $devicefull .= ",iothread=iothread-$deviceid";
4325         }
4326
4327         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
4328             $devicefull .= ",num_queues=$device->{queues}";
4329         }
4330
4331         qemu_deviceadd($vmid, $devicefull);
4332         qemu_deviceaddverify($vmid, $deviceid);
4333
4334     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4335
4336         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device, $arch, $machine_type);
4337         qemu_driveadd($storecfg, $vmid, $device);
4338
4339         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, $arch, $machine_type);
4340         eval { qemu_deviceadd($vmid, $devicefull); };
4341         if (my $err = $@) {
4342             eval { qemu_drivedel($vmid, $deviceid); };
4343             warn $@ if $@;
4344             die $err;
4345         }
4346
4347     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4348
4349         return undef if !qemu_netdevadd($vmid, $conf, $arch, $device, $deviceid);
4350
4351         my $machine_type = PVE::QemuServer::qemu_machine_pxe($vmid, $conf);
4352         my $use_old_bios_files = undef;
4353         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
4354
4355         my $netdevicefull = print_netdevice_full($vmid, $conf, $device, $deviceid, undef, $use_old_bios_files, $arch, $machine_type);
4356         qemu_deviceadd($vmid, $netdevicefull);
4357         eval {
4358             qemu_deviceaddverify($vmid, $deviceid);
4359             qemu_set_link_status($vmid, $deviceid, !$device->{link_down});
4360         };
4361         if (my $err = $@) {
4362             eval { qemu_netdevdel($vmid, $deviceid); };
4363             warn $@ if $@;
4364             die $err;
4365         }
4366
4367     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
4368
4369         my $bridgeid = $2;
4370         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
4371         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
4372
4373         qemu_deviceadd($vmid, $devicefull);
4374         qemu_deviceaddverify($vmid, $deviceid);
4375
4376     } else {
4377         die "can't hotplug device '$deviceid'\n";
4378     }
4379
4380     return 1;
4381 }
4382
4383 # fixme: this should raise exceptions on error!
4384 sub vm_deviceunplug {
4385     my ($vmid, $conf, $deviceid) = @_;
4386
4387     my $devices_list = vm_devices_list($vmid);
4388     return 1 if !defined($devices_list->{$deviceid});
4389
4390     die "can't unplug bootdisk" if $conf->{bootdisk} && $conf->{bootdisk} eq $deviceid;
4391
4392     if ($deviceid eq 'tablet' || $deviceid eq 'keyboard') {
4393
4394         qemu_devicedel($vmid, $deviceid);
4395
4396     } elsif ($deviceid =~ m/^usb\d+$/) {
4397
4398         die "usb hotplug currently not reliable\n";
4399         # when unplugging usb devices this way,
4400         # there may be remaining usb controllers/hubs
4401         # so we disable it for now
4402         qemu_devicedel($vmid, $deviceid);
4403         qemu_devicedelverify($vmid, $deviceid);
4404
4405     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4406
4407         qemu_devicedel($vmid, $deviceid);
4408         qemu_devicedelverify($vmid, $deviceid);
4409         qemu_drivedel($vmid, $deviceid);
4410         qemu_iothread_del($conf, $vmid, $deviceid);
4411
4412     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4413
4414         qemu_devicedel($vmid, $deviceid);
4415         qemu_devicedelverify($vmid, $deviceid);
4416         qemu_iothread_del($conf, $vmid, $deviceid);
4417
4418     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4419
4420         qemu_devicedel($vmid, $deviceid);
4421         qemu_drivedel($vmid, $deviceid);
4422         qemu_deletescsihw($conf, $vmid, $deviceid);
4423
4424     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4425
4426         qemu_devicedel($vmid, $deviceid);
4427         qemu_devicedelverify($vmid, $deviceid);
4428         qemu_netdevdel($vmid, $deviceid);
4429
4430     } else {
4431         die "can't unplug device '$deviceid'\n";
4432     }
4433
4434     return 1;
4435 }
4436
4437 sub qemu_deviceadd {
4438     my ($vmid, $devicefull) = @_;
4439
4440     $devicefull = "driver=".$devicefull;
4441     my %options =  split(/[=,]/, $devicefull);
4442
4443     vm_mon_cmd($vmid, "device_add" , %options);
4444 }
4445
4446 sub qemu_devicedel {
4447     my ($vmid, $deviceid) = @_;
4448
4449     my $ret = vm_mon_cmd($vmid, "device_del", id => $deviceid);
4450 }
4451
4452 sub qemu_iothread_add {
4453     my($vmid, $deviceid, $device) = @_;
4454
4455     if ($device->{iothread}) {
4456         my $iothreads = vm_iothreads_list($vmid);
4457         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
4458     }
4459 }
4460
4461 sub qemu_iothread_del {
4462     my($conf, $vmid, $deviceid) = @_;
4463
4464     my $confid = $deviceid;
4465     if ($deviceid =~ m/^(?:virtioscsi|scsihw)(\d+)$/) {
4466         $confid = 'scsi' . $1;
4467     }
4468     my $device = parse_drive($confid, $conf->{$confid});
4469     if ($device->{iothread}) {
4470         my $iothreads = vm_iothreads_list($vmid);
4471         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
4472     }
4473 }
4474
4475 sub qemu_objectadd {
4476     my($vmid, $objectid, $qomtype) = @_;
4477
4478     vm_mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
4479
4480     return 1;
4481 }
4482
4483 sub qemu_objectdel {
4484     my($vmid, $objectid) = @_;
4485
4486     vm_mon_cmd($vmid, "object-del", id => $objectid);
4487
4488     return 1;
4489 }
4490
4491 sub qemu_driveadd {
4492     my ($storecfg, $vmid, $device) = @_;
4493
4494     my $drive = print_drive_full($storecfg, $vmid, $device);
4495     $drive =~ s/\\/\\\\/g;
4496     my $ret = vm_human_monitor_command($vmid, "drive_add auto \"$drive\"");
4497
4498     # If the command succeeds qemu prints: "OK"
4499     return 1 if $ret =~ m/OK/s;
4500
4501     die "adding drive failed: $ret\n";
4502 }
4503
4504 sub qemu_drivedel {
4505     my($vmid, $deviceid) = @_;
4506
4507     my $ret = vm_human_monitor_command($vmid, "drive_del drive-$deviceid");
4508     $ret =~ s/^\s+//;
4509
4510     return 1 if $ret eq "";
4511
4512     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
4513     return 1 if $ret =~ m/Device \'.*?\' not found/s;
4514
4515     die "deleting drive $deviceid failed : $ret\n";
4516 }
4517
4518 sub qemu_deviceaddverify {
4519     my ($vmid, $deviceid) = @_;
4520
4521     for (my $i = 0; $i <= 5; $i++) {
4522          my $devices_list = vm_devices_list($vmid);
4523          return 1 if defined($devices_list->{$deviceid});
4524          sleep 1;
4525     }
4526
4527     die "error on hotplug device '$deviceid'\n";
4528 }
4529
4530
4531 sub qemu_devicedelverify {
4532     my ($vmid, $deviceid) = @_;
4533
4534     # need to verify that the device is correctly removed as device_del
4535     # is async and empty return is not reliable
4536
4537     for (my $i = 0; $i <= 5; $i++) {
4538          my $devices_list = vm_devices_list($vmid);
4539          return 1 if !defined($devices_list->{$deviceid});
4540          sleep 1;
4541     }
4542
4543     die "error on hot-unplugging device '$deviceid'\n";
4544 }
4545
4546 sub qemu_findorcreatescsihw {
4547     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4548
4549     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4550
4551     my $scsihwid="$controller_prefix$controller";
4552     my $devices_list = vm_devices_list($vmid);
4553
4554     if(!defined($devices_list->{$scsihwid})) {
4555         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device, $arch, $machine_type);
4556     }
4557
4558     return 1;
4559 }
4560
4561 sub qemu_deletescsihw {
4562     my ($conf, $vmid, $opt) = @_;
4563
4564     my $device = parse_drive($opt, $conf->{$opt});
4565
4566     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
4567         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
4568         return 1;
4569     }
4570
4571     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4572
4573     my $devices_list = vm_devices_list($vmid);
4574     foreach my $opt (keys %{$devices_list}) {
4575         if (PVE::QemuServer::is_valid_drivename($opt)) {
4576             my $drive = PVE::QemuServer::parse_drive($opt, $conf->{$opt});
4577             if($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
4578                 return 1;
4579             }
4580         }
4581     }
4582
4583     my $scsihwid="scsihw$controller";
4584
4585     vm_deviceunplug($vmid, $conf, $scsihwid);
4586
4587     return 1;
4588 }
4589
4590 sub qemu_add_pci_bridge {
4591     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4592
4593     my $bridges = {};
4594
4595     my $bridgeid;
4596
4597     print_pci_addr($device, $bridges, $arch, $machine_type);
4598
4599     while (my ($k, $v) = each %$bridges) {
4600         $bridgeid = $k;
4601     }
4602     return 1 if !defined($bridgeid) || $bridgeid < 1;
4603
4604     my $bridge = "pci.$bridgeid";
4605     my $devices_list = vm_devices_list($vmid);
4606
4607     if (!defined($devices_list->{$bridge})) {
4608         vm_deviceplug($storecfg, $conf, $vmid, $bridge, $arch, $machine_type);
4609     }
4610
4611     return 1;
4612 }
4613
4614 sub qemu_set_link_status {
4615     my ($vmid, $device, $up) = @_;
4616
4617     vm_mon_cmd($vmid, "set_link", name => $device,
4618                up => $up ? JSON::true : JSON::false);
4619 }
4620
4621 sub qemu_netdevadd {
4622     my ($vmid, $conf, $arch, $device, $deviceid) = @_;
4623
4624     my $netdev = print_netdev_full($vmid, $conf, $arch, $device, $deviceid, 1);
4625     my %options =  split(/[=,]/, $netdev);
4626
4627     vm_mon_cmd($vmid, "netdev_add",  %options);
4628     return 1;
4629 }
4630
4631 sub qemu_netdevdel {
4632     my ($vmid, $deviceid) = @_;
4633
4634     vm_mon_cmd($vmid, "netdev_del", id => $deviceid);
4635 }
4636
4637 sub qemu_usb_hotplug {
4638     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4639
4640     return if !$device;
4641
4642     # remove the old one first
4643     vm_deviceunplug($vmid, $conf, $deviceid);
4644
4645     # check if xhci controller is necessary and available
4646     if ($device->{usb3}) {
4647
4648         my $devicelist = vm_devices_list($vmid);
4649
4650         if (!$devicelist->{xhci}) {
4651             my $pciaddr = print_pci_addr("xhci", undef, $arch, $machine_type);
4652             qemu_deviceadd($vmid, "nec-usb-xhci,id=xhci$pciaddr");
4653         }
4654     }
4655     my $d = parse_usb_device($device->{host});
4656     $d->{usb3} = $device->{usb3};
4657
4658     # add the new one
4659     vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $d, $arch, $machine_type);
4660 }
4661
4662 sub qemu_cpu_hotplug {
4663     my ($vmid, $conf, $vcpus) = @_;
4664
4665     my $machine_type = PVE::QemuServer::get_current_qemu_machine($vmid);
4666
4667     my $sockets = 1;
4668     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
4669     $sockets = $conf->{sockets} if  $conf->{sockets};
4670     my $cores = $conf->{cores} || 1;
4671     my $maxcpus = $sockets * $cores;
4672
4673     $vcpus = $maxcpus if !$vcpus;
4674
4675     die "you can't add more vcpus than maxcpus\n"
4676         if $vcpus > $maxcpus;
4677
4678     my $currentvcpus = $conf->{vcpus} || $maxcpus;
4679
4680     if ($vcpus < $currentvcpus) {
4681
4682         if (qemu_machine_feature_enabled ($machine_type, undef, 2, 7)) {
4683
4684             for (my $i = $currentvcpus; $i > $vcpus; $i--) {
4685                 qemu_devicedel($vmid, "cpu$i");
4686                 my $retry = 0;
4687                 my $currentrunningvcpus = undef;
4688                 while (1) {
4689                     $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
4690                     last if scalar(@{$currentrunningvcpus}) == $i-1;
4691                     raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
4692                     $retry++;
4693                     sleep 1;
4694                 }
4695                 #update conf after each succesfull cpu unplug
4696                 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4697                 PVE::QemuConfig->write_config($vmid, $conf);
4698             }
4699         } else {
4700             die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
4701         }
4702
4703         return;
4704     }
4705
4706     my $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
4707     die "vcpus in running vm does not match its configuration\n"
4708         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
4709
4710     if (qemu_machine_feature_enabled ($machine_type, undef, 2, 7)) {
4711
4712         for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
4713             my $cpustr = print_cpu_device($conf, $i);
4714             qemu_deviceadd($vmid, $cpustr);
4715
4716             my $retry = 0;
4717             my $currentrunningvcpus = undef;
4718             while (1) {
4719                 $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
4720                 last if scalar(@{$currentrunningvcpus}) == $i;
4721                 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
4722                 sleep 1;
4723                 $retry++;
4724             }
4725             #update conf after each succesfull cpu hotplug
4726             $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4727             PVE::QemuConfig->write_config($vmid, $conf);
4728         }
4729     } else {
4730
4731         for (my $i = $currentvcpus; $i < $vcpus; $i++) {
4732             vm_mon_cmd($vmid, "cpu-add", id => int($i));
4733         }
4734     }
4735 }
4736
4737 sub qemu_block_set_io_throttle {
4738     my ($vmid, $deviceid,
4739         $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
4740         $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max,
4741         $bps_max_length, $bps_rd_max_length, $bps_wr_max_length,
4742         $iops_max_length, $iops_rd_max_length, $iops_wr_max_length) = @_;
4743
4744     return if !check_running($vmid) ;
4745
4746     vm_mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
4747         bps => int($bps),
4748         bps_rd => int($bps_rd),
4749         bps_wr => int($bps_wr),
4750         iops => int($iops),
4751         iops_rd => int($iops_rd),
4752         iops_wr => int($iops_wr),
4753         bps_max => int($bps_max),
4754         bps_rd_max => int($bps_rd_max),
4755         bps_wr_max => int($bps_wr_max),
4756         iops_max => int($iops_max),
4757         iops_rd_max => int($iops_rd_max),
4758         iops_wr_max => int($iops_wr_max),
4759         bps_max_length => int($bps_max_length),
4760         bps_rd_max_length => int($bps_rd_max_length),
4761         bps_wr_max_length => int($bps_wr_max_length),
4762         iops_max_length => int($iops_max_length),
4763         iops_rd_max_length => int($iops_rd_max_length),
4764         iops_wr_max_length => int($iops_wr_max_length),
4765     );
4766
4767 }
4768
4769 # old code, only used to shutdown old VM after update
4770 sub __read_avail {
4771     my ($fh, $timeout) = @_;
4772
4773     my $sel = new IO::Select;
4774     $sel->add($fh);
4775
4776     my $res = '';
4777     my $buf;
4778
4779     my @ready;
4780     while (scalar (@ready = $sel->can_read($timeout))) {
4781         my $count;
4782         if ($count = $fh->sysread($buf, 8192)) {
4783             if ($buf =~ /^(.*)\(qemu\) $/s) {
4784                 $res .= $1;
4785                 last;
4786             } else {
4787                 $res .= $buf;
4788             }
4789         } else {
4790             if (!defined($count)) {
4791                 die "$!\n";
4792             }
4793             last;
4794         }
4795     }
4796
4797     die "monitor read timeout\n" if !scalar(@ready);
4798
4799     return $res;
4800 }
4801
4802 sub qemu_block_resize {
4803     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4804
4805     my $running = check_running($vmid);
4806
4807     $size = 0 if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4808
4809     return if !$running;
4810
4811     vm_mon_cmd($vmid, "block_resize", device => $deviceid, size => int($size));
4812
4813 }
4814
4815 sub qemu_volume_snapshot {
4816     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4817
4818     my $running = check_running($vmid);
4819
4820     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4821         vm_mon_cmd($vmid, 'blockdev-snapshot-internal-sync', device => $deviceid, name => $snap);
4822     } else {
4823         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4824     }
4825 }
4826
4827 sub qemu_volume_snapshot_delete {
4828     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4829
4830     my $running = check_running($vmid);
4831
4832     if($running) {
4833
4834         $running = undef;
4835         my $conf = PVE::QemuConfig->load_config($vmid);
4836         foreach_drive($conf, sub {
4837             my ($ds, $drive) = @_;
4838             $running = 1 if $drive->{file} eq $volid;
4839         });
4840     }
4841
4842     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4843         vm_mon_cmd($vmid, 'blockdev-snapshot-delete-internal-sync', device => $deviceid, name => $snap);
4844     } else {
4845         PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4846     }
4847 }
4848
4849 sub set_migration_caps {
4850     my ($vmid) = @_;
4851
4852     my $cap_ref = [];
4853
4854     my $enabled_cap = {
4855         "auto-converge" => 1,
4856         "xbzrle" => 1,
4857         "x-rdma-pin-all" => 0,
4858         "zero-blocks" => 0,
4859         "compress" => 0
4860     };
4861
4862     my $supported_capabilities = vm_mon_cmd_nocheck($vmid, "query-migrate-capabilities");
4863
4864     for my $supported_capability (@$supported_capabilities) {
4865         push @$cap_ref, {
4866             capability => $supported_capability->{capability},
4867             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4868         };
4869     }
4870
4871     vm_mon_cmd_nocheck($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4872 }
4873
4874 my $fast_plug_option = {
4875     'lock' => 1,
4876     'name' => 1,
4877     'onboot' => 1,
4878     'shares' => 1,
4879     'startup' => 1,
4880     'description' => 1,
4881     'protection' => 1,
4882     'vmstatestorage' => 1,
4883     'hookscript' => 1,
4884 };
4885
4886 # hotplug changes in [PENDING]
4887 # $selection hash can be used to only apply specified options, for
4888 # example: { cores => 1 } (only apply changed 'cores')
4889 # $errors ref is used to return error messages
4890 sub vmconfig_hotplug_pending {
4891     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4892
4893     my $defaults = load_defaults();
4894     my ($arch, $machine_type) = get_basic_machine_info($conf, undef);
4895
4896     # commit values which do not have any impact on running VM first
4897     # Note: those option cannot raise errors, we we do not care about
4898     # $selection and always apply them.
4899
4900     my $add_error = sub {
4901         my ($opt, $msg) = @_;
4902         $errors->{$opt} = "hotplug problem - $msg";
4903     };
4904
4905     my $changes = 0;
4906     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4907         if ($fast_plug_option->{$opt}) {
4908             $conf->{$opt} = $conf->{pending}->{$opt};
4909             delete $conf->{pending}->{$opt};
4910             $changes = 1;
4911         }
4912     }
4913
4914     if ($changes) {
4915         PVE::QemuConfig->write_config($vmid, $conf);
4916         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4917     }
4918
4919     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4920
4921     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4922     while (my ($opt, $force) = each %$pending_delete_hash) {
4923         next if $selection && !$selection->{$opt};
4924         eval {
4925             if ($opt eq 'hotplug') {
4926                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
4927             } elsif ($opt eq 'tablet') {
4928                 die "skip\n" if !$hotplug_features->{usb};
4929                 if ($defaults->{tablet}) {
4930                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4931                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4932                         if $arch eq 'aarch64';
4933                 } else {
4934                     vm_deviceunplug($vmid, $conf, 'tablet');
4935                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4936                 }
4937             } elsif ($opt =~ m/^usb\d+/) {
4938                 die "skip\n";
4939                 # since we cannot reliably hot unplug usb devices
4940                 # we are disabling it
4941                 die "skip\n" if !$hotplug_features->{usb} || $conf->{$opt} =~ m/spice/i;
4942                 vm_deviceunplug($vmid, $conf, $opt);
4943             } elsif ($opt eq 'vcpus') {
4944                 die "skip\n" if !$hotplug_features->{cpu};
4945                 qemu_cpu_hotplug($vmid, $conf, undef);
4946             } elsif ($opt eq 'balloon') {
4947                 # enable balloon device is not hotpluggable
4948                 die "skip\n" if defined($conf->{balloon}) && $conf->{balloon} == 0;
4949                 # here we reset the ballooning value to memory
4950                 my $balloon = $conf->{memory} || $defaults->{memory};
4951                 vm_mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4952             } elsif ($fast_plug_option->{$opt}) {
4953                 # do nothing
4954             } elsif ($opt =~ m/^net(\d+)$/) {
4955                 die "skip\n" if !$hotplug_features->{network};
4956                 vm_deviceunplug($vmid, $conf, $opt);
4957             } elsif (is_valid_drivename($opt)) {
4958                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4959                 vm_deviceunplug($vmid, $conf, $opt);
4960                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4961             } elsif ($opt =~ m/^memory$/) {
4962                 die "skip\n" if !$hotplug_features->{memory};
4963                 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
4964             } elsif ($opt eq 'cpuunits') {
4965                 cgroups_write("cpu", $vmid, "cpu.shares", $defaults->{cpuunits});
4966             } elsif ($opt eq 'cpulimit') {
4967                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", -1);
4968             } else {
4969                 die "skip\n";
4970             }
4971         };
4972         if (my $err = $@) {
4973             &$add_error($opt, $err) if $err ne "skip\n";
4974         } else {
4975             # save new config if hotplug was successful
4976             delete $conf->{$opt};
4977             vmconfig_undelete_pending_option($conf, $opt);
4978             PVE::QemuConfig->write_config($vmid, $conf);
4979             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4980         }
4981     }
4982
4983     my $apply_pending_cloudinit;
4984     $apply_pending_cloudinit = sub {
4985         my ($key, $value) = @_;
4986         $apply_pending_cloudinit = sub {}; # once is enough
4987
4988         my @cloudinit_opts = keys %$confdesc_cloudinit;
4989         foreach my $opt (keys %{$conf->{pending}}) {
4990             next if !grep { $_ eq $opt } @cloudinit_opts;
4991             $conf->{$opt} = delete $conf->{pending}->{$opt};
4992         }
4993
4994         my $new_conf = { %$conf };
4995         $new_conf->{$key} = $value;
4996         PVE::QemuServer::Cloudinit::generate_cloudinitconfig($new_conf, $vmid);
4997     };
4998
4999     foreach my $opt (keys %{$conf->{pending}}) {
5000         next if $selection && !$selection->{$opt};
5001         my $value = $conf->{pending}->{$opt};
5002         eval {
5003             if ($opt eq 'hotplug') {
5004                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
5005             } elsif ($opt eq 'tablet') {
5006                 die "skip\n" if !$hotplug_features->{usb};
5007                 if ($value == 1) {
5008                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
5009                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
5010                         if $arch eq 'aarch64';
5011                 } elsif ($value == 0) {
5012                     vm_deviceunplug($vmid, $conf, 'tablet');
5013                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
5014                 }
5015             } elsif ($opt =~ m/^usb\d+$/) {
5016                 die "skip\n";
5017                 # since we cannot reliably hot unplug usb devices
5018                 # we are disabling it
5019                 die "skip\n" if !$hotplug_features->{usb} || $value =~ m/spice/i;
5020                 my $d = eval { PVE::JSONSchema::parse_property_string($usbdesc->{format}, $value) };
5021                 die "skip\n" if !$d;
5022                 qemu_usb_hotplug($storecfg, $conf, $vmid, $opt, $d, $arch, $machine_type);
5023             } elsif ($opt eq 'vcpus') {
5024                 die "skip\n" if !$hotplug_features->{cpu};
5025                 qemu_cpu_hotplug($vmid, $conf, $value);
5026             } elsif ($opt eq 'balloon') {
5027                 # enable/disable balloning device is not hotpluggable
5028                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
5029                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
5030                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
5031
5032                 # allow manual ballooning if shares is set to zero
5033                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
5034                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
5035                     vm_mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
5036                 }
5037             } elsif ($opt =~ m/^net(\d+)$/) {
5038                 # some changes can be done without hotplug
5039                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
5040                                     $vmid, $opt, $value, $arch, $machine_type);
5041             } elsif (is_valid_drivename($opt)) {
5042                 # some changes can be done without hotplug
5043                 my $drive = parse_drive($opt, $value);
5044                 if (drive_is_cloudinit($drive)) {
5045                     &$apply_pending_cloudinit($opt, $value);
5046                 }
5047                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
5048                                      $vmid, $opt, $value, 1, $arch, $machine_type);
5049             } elsif ($opt =~ m/^memory$/) { #dimms
5050                 die "skip\n" if !$hotplug_features->{memory};
5051                 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
5052             } elsif ($opt eq 'cpuunits') {
5053                 cgroups_write("cpu", $vmid, "cpu.shares", $conf->{pending}->{$opt});
5054             } elsif ($opt eq 'cpulimit') {
5055                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
5056                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", $cpulimit);
5057             } else {
5058                 die "skip\n";  # skip non-hot-pluggable options
5059             }
5060         };
5061         if (my $err = $@) {
5062             &$add_error($opt, $err) if $err ne "skip\n";
5063         } else {
5064             # save new config if hotplug was successful
5065             $conf->{$opt} = $value;
5066             delete $conf->{pending}->{$opt};
5067             PVE::QemuConfig->write_config($vmid, $conf);
5068             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5069         }
5070     }
5071 }
5072
5073 sub try_deallocate_drive {
5074     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
5075
5076     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
5077         my $volid = $drive->{file};
5078         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
5079             my $sid = PVE::Storage::parse_volume_id($volid);
5080             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
5081
5082             # check if the disk is really unused
5083             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
5084                 if is_volume_in_use($storecfg, $conf, $key, $volid);
5085             PVE::Storage::vdisk_free($storecfg, $volid);
5086             return 1;
5087         } else {
5088             # If vm is not owner of this disk remove from config
5089             return 1;
5090         }
5091     }
5092
5093     return undef;
5094 }
5095
5096 sub vmconfig_delete_or_detach_drive {
5097     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
5098
5099     my $drive = parse_drive($opt, $conf->{$opt});
5100
5101     my $rpcenv = PVE::RPCEnvironment::get();
5102     my $authuser = $rpcenv->get_user();
5103
5104     if ($force) {
5105         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
5106         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
5107     } else {
5108         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
5109     }
5110 }
5111
5112 sub vmconfig_apply_pending {
5113     my ($vmid, $conf, $storecfg) = @_;
5114
5115     # cold plug
5116
5117     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
5118     while (my ($opt, $force) = each %$pending_delete_hash) {
5119         die "internal error" if $opt =~ m/^unused/;
5120         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5121         if (!defined($conf->{$opt})) {
5122             vmconfig_undelete_pending_option($conf, $opt);
5123             PVE::QemuConfig->write_config($vmid, $conf);
5124         } elsif (is_valid_drivename($opt)) {
5125             vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
5126             vmconfig_undelete_pending_option($conf, $opt);
5127             delete $conf->{$opt};
5128             PVE::QemuConfig->write_config($vmid, $conf);
5129         } else {
5130             vmconfig_undelete_pending_option($conf, $opt);
5131             delete $conf->{$opt};
5132             PVE::QemuConfig->write_config($vmid, $conf);
5133         }
5134     }
5135
5136     $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5137
5138     foreach my $opt (keys %{$conf->{pending}}) { # add/change
5139         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5140
5141         if (defined($conf->{$opt}) && ($conf->{$opt} eq $conf->{pending}->{$opt})) {
5142             # skip if nothing changed
5143         } elsif (is_valid_drivename($opt)) {
5144             vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
5145                 if defined($conf->{$opt});
5146             $conf->{$opt} = $conf->{pending}->{$opt};
5147         } else {
5148             $conf->{$opt} = $conf->{pending}->{$opt};
5149         }
5150
5151         delete $conf->{pending}->{$opt};
5152         PVE::QemuConfig->write_config($vmid, $conf);
5153     }
5154 }
5155
5156 my $safe_num_ne = sub {
5157     my ($a, $b) = @_;
5158
5159     return 0 if !defined($a) && !defined($b);
5160     return 1 if !defined($a);
5161     return 1 if !defined($b);
5162
5163     return $a != $b;
5164 };
5165
5166 my $safe_string_ne = sub {
5167     my ($a, $b) = @_;
5168
5169     return 0 if !defined($a) && !defined($b);
5170     return 1 if !defined($a);
5171     return 1 if !defined($b);
5172
5173     return $a ne $b;
5174 };
5175
5176 sub vmconfig_update_net {
5177     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
5178
5179     my $newnet = parse_net($value);
5180
5181     if ($conf->{$opt}) {
5182         my $oldnet = parse_net($conf->{$opt});
5183
5184         if (&$safe_string_ne($oldnet->{model}, $newnet->{model}) ||
5185             &$safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
5186             &$safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
5187             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
5188
5189             # for non online change, we try to hot-unplug
5190             die "skip\n" if !$hotplug;
5191             vm_deviceunplug($vmid, $conf, $opt);
5192         } else {
5193
5194             die "internal error" if $opt !~ m/net(\d+)/;
5195             my $iface = "tap${vmid}i$1";
5196
5197             if (&$safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
5198                 &$safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
5199                 &$safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
5200                 &$safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
5201                 PVE::Network::tap_unplug($iface);
5202                 PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
5203             } elsif (&$safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
5204                 # Rate can be applied on its own but any change above needs to
5205                 # include the rate in tap_plug since OVS resets everything.
5206                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
5207             }
5208
5209             if (&$safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
5210                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
5211             }
5212
5213             return 1;
5214         }
5215     }
5216
5217     if ($hotplug) {
5218         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet, $arch, $machine_type);
5219     } else {
5220         die "skip\n";
5221     }
5222 }
5223
5224 sub vmconfig_update_disk {
5225     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $force, $arch, $machine_type) = @_;
5226
5227     # fixme: do we need force?
5228
5229     my $drive = parse_drive($opt, $value);
5230
5231     if ($conf->{$opt}) {
5232
5233         if (my $old_drive = parse_drive($opt, $conf->{$opt}))  {
5234
5235             my $media = $drive->{media} || 'disk';
5236             my $oldmedia = $old_drive->{media} || 'disk';
5237             die "unable to change media type\n" if $media ne $oldmedia;
5238
5239             if (!drive_is_cdrom($old_drive)) {
5240
5241                 if ($drive->{file} ne $old_drive->{file}) {
5242
5243                     die "skip\n" if !$hotplug;
5244
5245                     # unplug and register as unused
5246                     vm_deviceunplug($vmid, $conf, $opt);
5247                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
5248
5249                 } else {
5250                     # update existing disk
5251
5252                     # skip non hotpluggable value
5253                     if (&$safe_string_ne($drive->{discard}, $old_drive->{discard}) ||
5254                         &$safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
5255                         &$safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
5256                         &$safe_string_ne($drive->{cache}, $old_drive->{cache})) {
5257                         die "skip\n";
5258                     }
5259
5260                     # apply throttle
5261                     if (&$safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
5262                         &$safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
5263                         &$safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
5264                         &$safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
5265                         &$safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
5266                         &$safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
5267                         &$safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
5268                         &$safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
5269                         &$safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
5270                         &$safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
5271                         &$safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
5272                         &$safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max}) ||
5273                         &$safe_num_ne($drive->{bps_max_length}, $old_drive->{bps_max_length}) ||
5274                         &$safe_num_ne($drive->{bps_rd_max_length}, $old_drive->{bps_rd_max_length}) ||
5275                         &$safe_num_ne($drive->{bps_wr_max_length}, $old_drive->{bps_wr_max_length}) ||
5276                         &$safe_num_ne($drive->{iops_max_length}, $old_drive->{iops_max_length}) ||
5277                         &$safe_num_ne($drive->{iops_rd_max_length}, $old_drive->{iops_rd_max_length}) ||
5278                         &$safe_num_ne($drive->{iops_wr_max_length}, $old_drive->{iops_wr_max_length})) {
5279
5280                         qemu_block_set_io_throttle($vmid,"drive-$opt",
5281                                                    ($drive->{mbps} || 0)*1024*1024,
5282                                                    ($drive->{mbps_rd} || 0)*1024*1024,
5283                                                    ($drive->{mbps_wr} || 0)*1024*1024,
5284                                                    $drive->{iops} || 0,
5285                                                    $drive->{iops_rd} || 0,
5286                                                    $drive->{iops_wr} || 0,
5287                                                    ($drive->{mbps_max} || 0)*1024*1024,
5288                                                    ($drive->{mbps_rd_max} || 0)*1024*1024,
5289                                                    ($drive->{mbps_wr_max} || 0)*1024*1024,
5290                                                    $drive->{iops_max} || 0,
5291                                                    $drive->{iops_rd_max} || 0,
5292                                                    $drive->{iops_wr_max} || 0,
5293                                                    $drive->{bps_max_length} || 1,
5294                                                    $drive->{bps_rd_max_length} || 1,
5295                                                    $drive->{bps_wr_max_length} || 1,
5296                                                    $drive->{iops_max_length} || 1,
5297                                                    $drive->{iops_rd_max_length} || 1,
5298                                                    $drive->{iops_wr_max_length} || 1);
5299
5300                     }
5301
5302                     return 1;
5303                 }
5304
5305             } else { # cdrom
5306
5307                 if ($drive->{file} eq 'none') {
5308                     vm_mon_cmd($vmid, "eject",force => JSON::true,device => "drive-$opt");
5309                     if (drive_is_cloudinit($old_drive)) {
5310                         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive);
5311                     }
5312                 } else {
5313                     my $path = get_iso_path($storecfg, $vmid, $drive->{file});
5314                     vm_mon_cmd($vmid, "eject", force => JSON::true,device => "drive-$opt"); # force eject if locked
5315                     vm_mon_cmd($vmid, "change", device => "drive-$opt",target => "$path") if $path;
5316                 }
5317
5318                 return 1;
5319             }
5320         }
5321     }
5322
5323     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
5324     # hotplug new disks
5325     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
5326     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive, $arch, $machine_type);
5327 }
5328
5329 sub vm_start {
5330     my ($storecfg, $vmid, $statefile, $skiplock, $migratedfrom, $paused,
5331         $forcemachine, $spice_ticket, $migration_network, $migration_type, $targetstorage) = @_;
5332
5333     PVE::QemuConfig->lock_config($vmid, sub {
5334         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
5335
5336         die "you can't start a vm if it's a template\n" if PVE::QemuConfig->is_template($conf);
5337
5338         my $is_suspended = PVE::QemuConfig->has_lock($conf, 'suspended');
5339
5340         PVE::QemuConfig->check_lock($conf)
5341             if !($skiplock || $is_suspended);
5342
5343         die "VM $vmid already running\n" if check_running($vmid, undef, $migratedfrom);
5344
5345         # clean up leftover reboot request files
5346         eval { clear_reboot_request($vmid); };
5347         warn $@ if $@;
5348
5349         if (!$statefile && scalar(keys %{$conf->{pending}})) {
5350             vmconfig_apply_pending($vmid, $conf, $storecfg);
5351             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5352         }
5353
5354         PVE::QemuServer::Cloudinit::generate_cloudinitconfig($conf, $vmid);
5355
5356         my $defaults = load_defaults();
5357
5358         # set environment variable useful inside network script
5359         $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
5360
5361         my $local_volumes = {};
5362
5363         if ($targetstorage) {
5364             foreach_drive($conf, sub {
5365                 my ($ds, $drive) = @_;
5366
5367                 return if drive_is_cdrom($drive);
5368
5369                 my $volid = $drive->{file};
5370
5371                 return if !$volid;
5372
5373                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
5374
5375                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5376                 return if $scfg->{shared};
5377                 $local_volumes->{$ds} = [$volid, $storeid, $volname];
5378             });
5379
5380             my $format = undef;
5381
5382             foreach my $opt (sort keys %$local_volumes) {
5383
5384                 my ($volid, $storeid, $volname) = @{$local_volumes->{$opt}};
5385                 my $drive = parse_drive($opt, $conf->{$opt});
5386
5387                 #if remote storage is specified, use default format
5388                 if ($targetstorage && $targetstorage ne "1") {
5389                     $storeid = $targetstorage;
5390                     my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
5391                     $format = $defFormat;
5392                 } else {
5393                     #else we use same format than original
5394                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5395                     $format = qemu_img_format($scfg, $volid);
5396                 }
5397
5398                 my $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $format, undef, ($drive->{size}/1024));
5399                 my $newdrive = $drive;
5400                 $newdrive->{format} = $format;
5401                 $newdrive->{file} = $newvolid;
5402                 my $drivestr = PVE::QemuServer::print_drive($vmid, $newdrive);
5403                 $local_volumes->{$opt} = $drivestr;
5404                 #pass drive to conf for command line
5405                 $conf->{$opt} = $drivestr;
5406             }
5407         }
5408
5409         PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-start', 1);
5410
5411         if ($is_suspended) {
5412             # enforce machine type on suspended vm to ensure HW compatibility
5413             $forcemachine = $conf->{runningmachine};
5414             print "Resuming suspended VM\n";
5415         }
5416
5417         my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine);
5418
5419         my $migrate_uri;
5420         if ($statefile) {
5421             if ($statefile eq 'tcp') {
5422                 my $localip = "localhost";
5423                 my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
5424                 my $nodename = PVE::INotify::nodename();
5425
5426                 if (!defined($migration_type)) {
5427                     if (defined($datacenterconf->{migration}->{type})) {
5428                         $migration_type = $datacenterconf->{migration}->{type};
5429                     } else {
5430                         $migration_type = 'secure';
5431                     }
5432                 }
5433
5434                 if ($migration_type eq 'insecure') {
5435                     my $migrate_network_addr = PVE::Cluster::get_local_migration_ip($migration_network);
5436                     if ($migrate_network_addr) {
5437                         $localip = $migrate_network_addr;
5438                     } else {
5439                         $localip = PVE::Cluster::remote_node_ip($nodename, 1);
5440                     }
5441
5442                     $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5443                 }
5444
5445                 my $pfamily = PVE::Tools::get_host_address_family($nodename);
5446                 my $migrate_port = PVE::Tools::next_migrate_port($pfamily);
5447                 $migrate_uri = "tcp:${localip}:${migrate_port}";
5448                 push @$cmd, '-incoming', $migrate_uri;
5449                 push @$cmd, '-S';
5450
5451             } elsif ($statefile eq 'unix') {
5452                 # should be default for secure migrations as a ssh TCP forward
5453                 # tunnel is not deterministic reliable ready and fails regurarly
5454                 # to set up in time, so use UNIX socket forwards
5455                 my $socket_addr = "/run/qemu-server/$vmid.migrate";
5456                 unlink $socket_addr;
5457
5458                 $migrate_uri = "unix:$socket_addr";
5459
5460                 push @$cmd, '-incoming', $migrate_uri;
5461                 push @$cmd, '-S';
5462
5463             } elsif (-e $statefile) {
5464                 push @$cmd, '-loadstate', $statefile;
5465             } else {
5466                 my $statepath = PVE::Storage::path($storecfg, $statefile);
5467                 push @$vollist, $statepath;
5468                 push @$cmd, '-loadstate', $statepath;
5469             }
5470         } elsif ($paused) {
5471             push @$cmd, '-S';
5472         }
5473
5474         # host pci devices
5475         for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
5476           my $d = parse_hostpci($conf->{"hostpci$i"});
5477           next if !$d;
5478           my $pcidevices = $d->{pciid};
5479           foreach my $pcidevice (@$pcidevices) {
5480                 my $pciid = $pcidevice->{id};
5481
5482                 my $info = PVE::SysFSTools::pci_device_info("0000:$pciid");
5483                 die "IOMMU not present\n" if !PVE::SysFSTools::check_iommu_support();
5484                 die "no pci device info for device '$pciid'\n" if !$info;
5485
5486                 if ($d->{mdev}) {
5487                     my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $i);
5488                     PVE::SysFSTools::pci_create_mdev_device($pciid, $uuid, $d->{mdev});
5489                 } else {
5490                     die "can't unbind/bind pci group to vfio '$pciid'\n"
5491                         if !PVE::SysFSTools::pci_dev_group_bind_to_vfio($pciid);
5492                     die "can't reset pci device '$pciid'\n"
5493                         if $info->{has_fl_reset} and !PVE::SysFSTools::pci_dev_reset($info);
5494                 }
5495           }
5496         }
5497
5498         PVE::Storage::activate_volumes($storecfg, $vollist);
5499
5500         eval {
5501             run_command(['/bin/systemctl', 'stop', "$vmid.scope"],
5502                 outfunc => sub {}, errfunc => sub {});
5503         };
5504         # Issues with the above 'stop' not being fully completed are extremely rare, a very low
5505         # timeout should be more than enough here...
5506         PVE::Systemd::wait_for_unit_removed("$vmid.scope", 5);
5507
5508         my $cpuunits = defined($conf->{cpuunits}) ? $conf->{cpuunits}
5509                                                   : $defaults->{cpuunits};
5510
5511         my $start_timeout = ($conf->{hugepages} || $is_suspended) ? 300 : 30;
5512         my %run_params = (timeout => $statefile ? undef : $start_timeout, umask => 0077);
5513
5514         my %properties = (
5515             Slice => 'qemu.slice',
5516             KillMode => 'none',
5517             CPUShares => $cpuunits
5518         );
5519
5520         if (my $cpulimit = $conf->{cpulimit}) {
5521             $properties{CPUQuota} = int($cpulimit * 100);
5522         }
5523         $properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
5524
5525         my $run_qemu = sub {
5526             PVE::Tools::run_fork sub {
5527                 PVE::Systemd::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
5528                 run_command($cmd, %run_params);
5529             };
5530         };
5531
5532         if ($conf->{hugepages}) {
5533
5534             my $code = sub {
5535                 my $hugepages_topology = PVE::QemuServer::Memory::hugepages_topology($conf);
5536                 my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
5537
5538                 PVE::QemuServer::Memory::hugepages_mount();
5539                 PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
5540
5541                 eval { $run_qemu->() };
5542                 if (my $err = $@) {
5543                     PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology);
5544                     die $err;
5545                 }
5546
5547                 PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology);
5548             };
5549             eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
5550
5551         } else {
5552             eval { $run_qemu->() };
5553         }
5554
5555         if (my $err = $@) {
5556             # deactivate volumes if start fails
5557             eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
5558             die "start failed: $err";
5559         }
5560
5561         print "migration listens on $migrate_uri\n" if $migrate_uri;
5562
5563         if ($statefile && $statefile ne 'tcp' && $statefile ne 'unix')  {
5564             eval { vm_mon_cmd_nocheck($vmid, "cont"); };
5565             warn $@ if $@;
5566         }
5567
5568         #start nbd server for storage migration
5569         if ($targetstorage) {
5570             my $nodename = PVE::INotify::nodename();
5571             my $migrate_network_addr = PVE::Cluster::get_local_migration_ip($migration_network);
5572             my $localip = $migrate_network_addr ? $migrate_network_addr : PVE::Cluster::remote_node_ip($nodename, 1);
5573             my $pfamily = PVE::Tools::get_host_address_family($nodename);
5574             my $storage_migrate_port = PVE::Tools::next_migrate_port($pfamily);
5575
5576             vm_mon_cmd_nocheck($vmid, "nbd-server-start", addr => { type => 'inet', data => { host => "${localip}", port => "${storage_migrate_port}" } } );
5577
5578             $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5579
5580             foreach my $opt (sort keys %$local_volumes) {
5581                 my $volid = $local_volumes->{$opt};
5582                 vm_mon_cmd_nocheck($vmid, "nbd-server-add", device => "drive-$opt", writable => JSON::true );
5583                 my $migrate_storage_uri = "nbd:${localip}:${storage_migrate_port}:exportname=drive-$opt";
5584                 print "storage migration listens on $migrate_storage_uri volume:$volid\n";
5585             }
5586         }
5587
5588         if ($migratedfrom) {
5589             eval {
5590                 set_migration_caps($vmid);
5591             };
5592             warn $@ if $@;
5593
5594             if ($spice_port) {
5595                 print "spice listens on port $spice_port\n";
5596                 if ($spice_ticket) {
5597                     vm_mon_cmd_nocheck($vmid, "set_password", protocol => 'spice', password => $spice_ticket);
5598                     vm_mon_cmd_nocheck($vmid, "expire_password", protocol => 'spice', time => "+30");
5599                 }
5600             }
5601
5602         } else {
5603             vm_mon_cmd_nocheck($vmid, "balloon", value => $conf->{balloon}*1024*1024)
5604                 if !$statefile && $conf->{balloon};
5605
5606             foreach my $opt (keys %$conf) {
5607                 next if $opt !~  m/^net\d+$/;
5608                 my $nicconf = parse_net($conf->{$opt});
5609                 qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
5610             }
5611         }
5612
5613         vm_mon_cmd_nocheck($vmid, 'qom-set',
5614                     path => "machine/peripheral/balloon0",
5615                     property => "guest-stats-polling-interval",
5616                     value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
5617
5618         if ($is_suspended && (my $vmstate = $conf->{vmstate})) {
5619             print "Resumed VM, removing state\n";
5620             delete $conf->@{qw(lock vmstate runningmachine)};
5621             PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
5622             PVE::Storage::vdisk_free($storecfg, $vmstate);
5623             PVE::QemuConfig->write_config($vmid, $conf);
5624         }
5625
5626         PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'post-start');
5627     });
5628 }
5629
5630 sub vm_mon_cmd {
5631     my ($vmid, $execute, %params) = @_;
5632
5633     my $cmd = { execute => $execute, arguments => \%params };
5634     vm_qmp_command($vmid, $cmd);
5635 }
5636
5637 sub vm_mon_cmd_nocheck {
5638     my ($vmid, $execute, %params) = @_;
5639
5640     my $cmd = { execute => $execute, arguments => \%params };
5641     vm_qmp_command($vmid, $cmd, 1);
5642 }
5643
5644 sub vm_qmp_command {
5645     my ($vmid, $cmd, $nocheck) = @_;
5646
5647     my $res;
5648
5649     my $timeout;
5650     if ($cmd->{arguments}) {
5651         $timeout = delete $cmd->{arguments}->{timeout};
5652     }
5653
5654     eval {
5655         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
5656         my $sname = qmp_socket($vmid);
5657         if (-e $sname) { # test if VM is reasonambe new and supports qmp/qga
5658             my $qmpclient = PVE::QMPClient->new();
5659
5660             $res = $qmpclient->cmd($vmid, $cmd, $timeout);
5661         } else {
5662             die "unable to open monitor socket\n";
5663         }
5664     };
5665     if (my $err = $@) {
5666         syslog("err", "VM $vmid qmp command failed - $err");
5667         die $err;
5668     }
5669
5670     return $res;
5671 }
5672
5673 sub vm_human_monitor_command {
5674     my ($vmid, $cmdline) = @_;
5675
5676     my $cmd = {
5677         execute => 'human-monitor-command',
5678         arguments => { 'command-line' => $cmdline},
5679     };
5680
5681     return vm_qmp_command($vmid, $cmd);
5682 }
5683
5684 sub vm_commandline {
5685     my ($storecfg, $vmid, $snapname) = @_;
5686
5687     my $conf = PVE::QemuConfig->load_config($vmid);
5688
5689     if ($snapname) {
5690         my $snapshot = $conf->{snapshots}->{$snapname};
5691         die "snapshot '$snapname' does not exist\n" if !defined($snapshot);
5692
5693         $snapshot->{digest} = $conf->{digest}; # keep file digest for API
5694
5695         $conf = $snapshot;
5696     }
5697
5698     my $defaults = load_defaults();
5699
5700     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults);
5701
5702     return PVE::Tools::cmd2string($cmd);
5703 }
5704
5705 sub vm_reset {
5706     my ($vmid, $skiplock) = @_;
5707
5708     PVE::QemuConfig->lock_config($vmid, sub {
5709
5710         my $conf = PVE::QemuConfig->load_config($vmid);
5711
5712         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5713
5714         vm_mon_cmd($vmid, "system_reset");
5715     });
5716 }
5717
5718 sub get_vm_volumes {
5719     my ($conf) = @_;
5720
5721     my $vollist = [];
5722     foreach_volid($conf, sub {
5723         my ($volid, $attr) = @_;
5724
5725         return if $volid =~ m|^/|;
5726
5727         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5728         return if !$sid;
5729
5730         push @$vollist, $volid;
5731     });
5732
5733     return $vollist;
5734 }
5735
5736 sub vm_stop_cleanup {
5737     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
5738
5739     eval {
5740
5741         if (!$keepActive) {
5742             my $vollist = get_vm_volumes($conf);
5743             PVE::Storage::deactivate_volumes($storecfg, $vollist);
5744         }
5745
5746         foreach my $ext (qw(mon qmp pid vnc qga)) {
5747             unlink "/var/run/qemu-server/${vmid}.$ext";
5748         }
5749
5750         if ($conf->{ivshmem}) {
5751             my $ivshmem = PVE::JSONSchema::parse_property_string($ivshmem_fmt, $conf->{ivshmem});
5752             # just delete it for now, VMs which have this already open do not
5753             # are affected, but new VMs will get a separated one. If this
5754             # becomes an issue we either add some sort of ref-counting or just
5755             # add a "don't delete on stop" flag to the ivshmem format.
5756             unlink '/dev/shm/pve-shm-' . ($ivshmem->{name} // $vmid);
5757         }
5758
5759         foreach my $key (keys %$conf) {
5760             next if $key !~ m/^hostpci(\d+)$/;
5761             my $hostpciindex = $1;
5762             my $d = parse_hostpci($conf->{$key});
5763             my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $hostpciindex);
5764
5765             foreach my $pci (@{$d->{pciid}}) {
5766                 my $pciid = $pci->{id};
5767                 PVE::SysFSTools::pci_cleanup_mdev_device($pciid, $uuid);
5768             }
5769         }
5770
5771         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
5772     };
5773     warn $@ if $@; # avoid errors - just warn
5774 }
5775
5776 # call only in locked context
5777 sub _do_vm_stop {
5778     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive) = @_;
5779
5780     my $pid = check_running($vmid, $nocheck);
5781     return if !$pid;
5782
5783     my $conf;
5784     if (!$nocheck) {
5785         $conf = PVE::QemuConfig->load_config($vmid);
5786         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5787         if (!defined($timeout) && $shutdown && $conf->{startup}) {
5788             my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
5789             $timeout = $opts->{down} if $opts->{down};
5790         }
5791         PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-stop');
5792     }
5793
5794     eval {
5795         if ($shutdown) {
5796             if (defined($conf) && parse_guest_agent($conf)->{enabled}) {
5797                 vm_qmp_command($vmid, {
5798                         execute => "guest-shutdown",
5799                         arguments => { timeout => $timeout }
5800                     }, $nocheck);
5801             } else {
5802                 vm_qmp_command($vmid, { execute => "system_powerdown" }, $nocheck);
5803             }
5804         } else {
5805             vm_qmp_command($vmid, { execute => "quit" }, $nocheck);
5806         }
5807     };
5808     my $err = $@;
5809
5810     if (!$err) {
5811         $timeout = 60 if !defined($timeout);
5812
5813         my $count = 0;
5814         while (($count < $timeout) && check_running($vmid, $nocheck)) {
5815             $count++;
5816             sleep 1;
5817         }
5818
5819         if ($count >= $timeout) {
5820             if ($force) {
5821                 warn "VM still running - terminating now with SIGTERM\n";
5822                 kill 15, $pid;
5823             } else {
5824                 die "VM quit/powerdown failed - got timeout\n";
5825             }
5826         } else {
5827             vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5828             return;
5829         }
5830     } else {
5831         if ($force) {
5832             warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
5833             kill 15, $pid;
5834         } else {
5835             die "VM quit/powerdown failed\n";
5836         }
5837     }
5838
5839     # wait again
5840     $timeout = 10;
5841
5842     my $count = 0;
5843     while (($count < $timeout) && check_running($vmid, $nocheck)) {
5844         $count++;
5845         sleep 1;
5846     }
5847
5848     if ($count >= $timeout) {
5849         warn "VM still running - terminating now with SIGKILL\n";
5850         kill 9, $pid;
5851         sleep 1;
5852     }
5853
5854     vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5855 }
5856
5857 # Note: use $nocheck to skip tests if VM configuration file exists.
5858 # We need that when migration VMs to other nodes (files already moved)
5859 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
5860 sub vm_stop {
5861     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
5862
5863     $force = 1 if !defined($force) && !$shutdown;
5864
5865     if ($migratedfrom){
5866         my $pid = check_running($vmid, $nocheck, $migratedfrom);
5867         kill 15, $pid if $pid;
5868         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
5869         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
5870         return;
5871     }
5872
5873     PVE::QemuConfig->lock_config($vmid, sub {
5874         _do_vm_stop($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive);
5875    });
5876 }
5877
5878 sub vm_reboot {
5879     my ($vmid, $timeout) = @_;
5880
5881     PVE::QemuConfig->lock_config($vmid, sub {
5882
5883         # only reboot if running, as qmeventd starts it again on a stop event
5884         return if !check_running($vmid);
5885
5886         create_reboot_request($vmid);
5887
5888         my $storecfg = PVE::Storage::config();
5889         _do_vm_stop($storecfg, $vmid, undef, undef, $timeout, 1);
5890
5891    });
5892 }
5893
5894 sub vm_suspend {
5895     my ($vmid, $skiplock, $includestate, $statestorage) = @_;
5896
5897     my $conf;
5898     my $path;
5899     my $storecfg;
5900     my $vmstate;
5901
5902     PVE::QemuConfig->lock_config($vmid, sub {
5903
5904         $conf = PVE::QemuConfig->load_config($vmid);
5905
5906         my $is_backing_up = PVE::QemuConfig->has_lock($conf, 'backup');
5907         PVE::QemuConfig->check_lock($conf)
5908             if !($skiplock || $is_backing_up);
5909
5910         die "cannot suspend to disk during backup\n"
5911             if $is_backing_up && $includestate;
5912
5913         if ($includestate) {
5914             $conf->{lock} = 'suspending';
5915             my $date = strftime("%Y-%m-%d", localtime(time()));
5916             $storecfg = PVE::Storage::config();
5917             $vmstate = PVE::QemuConfig->__snapshot_save_vmstate($vmid, $conf, "suspend-$date", $storecfg, $statestorage, 1);
5918             $path = PVE::Storage::path($storecfg, $vmstate);
5919             PVE::QemuConfig->write_config($vmid, $conf);
5920         } else {
5921             vm_mon_cmd($vmid, "stop");
5922         }
5923     });
5924
5925     if ($includestate) {
5926         # save vm state
5927         PVE::Storage::activate_volumes($storecfg, [$vmstate]);
5928
5929         eval {
5930             vm_mon_cmd($vmid, "savevm-start", statefile => $path);
5931             for(;;) {
5932                 my $state = vm_mon_cmd_nocheck($vmid, "query-savevm");
5933                 if (!$state->{status}) {
5934                     die "savevm not active\n";
5935                 } elsif ($state->{status} eq 'active') {
5936                     sleep(1);
5937                     next;
5938                 } elsif ($state->{status} eq 'completed') {
5939                     print "State saved, quitting\n";
5940                     last;
5941                 } elsif ($state->{status} eq 'failed' && $state->{error}) {
5942                     die "query-savevm failed with error '$state->{error}'\n"
5943                 } else {
5944                     die "query-savevm returned status '$state->{status}'\n";
5945                 }
5946             }
5947         };
5948         my $err = $@;
5949
5950         PVE::QemuConfig->lock_config($vmid, sub {
5951             $conf = PVE::QemuConfig->load_config($vmid);
5952             if ($err) {
5953                 # cleanup, but leave suspending lock, to indicate something went wrong
5954                 eval {
5955                     vm_mon_cmd($vmid, "savevm-end");
5956                     PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
5957                     PVE::Storage::vdisk_free($storecfg, $vmstate);
5958                     delete $conf->@{qw(vmstate runningmachine)};
5959                     PVE::QemuConfig->write_config($vmid, $conf);
5960                 };
5961                 warn $@ if $@;
5962                 die $err;
5963             }
5964
5965             die "lock changed unexpectedly\n"
5966                 if !PVE::QemuConfig->has_lock($conf, 'suspending');
5967
5968             vm_qmp_command($vmid, { execute => "quit" });
5969             $conf->{lock} = 'suspended';
5970             PVE::QemuConfig->write_config($vmid, $conf);
5971         });
5972     }
5973 }
5974
5975 sub vm_resume {
5976     my ($vmid, $skiplock, $nocheck) = @_;
5977
5978     PVE::QemuConfig->lock_config($vmid, sub {
5979         my $vm_mon_cmd = $nocheck ? \&vm_mon_cmd_nocheck : \&vm_mon_cmd;
5980         my $res = $vm_mon_cmd->($vmid, 'query-status');
5981         my $resume_cmd = 'cont';
5982
5983         if ($res->{status} && $res->{status} eq 'suspended') {
5984             $resume_cmd = 'system_wakeup';
5985         }
5986
5987         if (!$nocheck) {
5988
5989             my $conf = PVE::QemuConfig->load_config($vmid);
5990
5991             PVE::QemuConfig->check_lock($conf)
5992                 if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
5993         }
5994
5995         $vm_mon_cmd->($vmid, $resume_cmd);
5996     });
5997 }
5998
5999 sub vm_sendkey {
6000     my ($vmid, $skiplock, $key) = @_;
6001
6002     PVE::QemuConfig->lock_config($vmid, sub {
6003
6004         my $conf = PVE::QemuConfig->load_config($vmid);
6005
6006         # there is no qmp command, so we use the human monitor command
6007         my $res = vm_human_monitor_command($vmid, "sendkey $key");
6008         die $res if $res ne '';
6009     });
6010 }
6011
6012 sub vm_destroy {
6013     my ($storecfg, $vmid, $skiplock) = @_;
6014
6015     PVE::QemuConfig->lock_config($vmid, sub {
6016
6017         my $conf = PVE::QemuConfig->load_config($vmid);
6018
6019         if (!check_running($vmid)) {
6020             destroy_vm($storecfg, $vmid, undef, $skiplock);
6021         } else {
6022             die "VM $vmid is running - destroy failed\n";
6023         }
6024     });
6025 }
6026
6027 # vzdump restore implementaion
6028
6029 sub tar_archive_read_firstfile {
6030     my $archive = shift;
6031
6032     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
6033
6034     # try to detect archive type first
6035     my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
6036         die "unable to open file '$archive'\n";
6037     my $firstfile = <$fh>;
6038     kill 15, $pid;
6039     close $fh;
6040
6041     die "ERROR: archive contaions no data\n" if !$firstfile;
6042     chomp $firstfile;
6043
6044     return $firstfile;
6045 }
6046
6047 sub tar_restore_cleanup {
6048     my ($storecfg, $statfile) = @_;
6049
6050     print STDERR "starting cleanup\n";
6051
6052     if (my $fd = IO::File->new($statfile, "r")) {
6053         while (defined(my $line = <$fd>)) {
6054             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
6055                 my $volid = $2;
6056                 eval {
6057                     if ($volid =~ m|^/|) {
6058                         unlink $volid || die 'unlink failed\n';
6059                     } else {
6060                         PVE::Storage::vdisk_free($storecfg, $volid);
6061                     }
6062                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
6063                 };
6064                 print STDERR "unable to cleanup '$volid' - $@" if $@;
6065             } else {
6066                 print STDERR "unable to parse line in statfile - $line";
6067             }
6068         }
6069         $fd->close();
6070     }
6071 }
6072
6073 sub restore_archive {
6074     my ($archive, $vmid, $user, $opts) = @_;
6075
6076     my $format = $opts->{format};
6077     my $comp;
6078
6079     if ($archive =~ m/\.tgz$/ || $archive =~ m/\.tar\.gz$/) {
6080         $format = 'tar' if !$format;
6081         $comp = 'gzip';
6082     } elsif ($archive =~ m/\.tar$/) {
6083         $format = 'tar' if !$format;
6084     } elsif ($archive =~ m/.tar.lzo$/) {
6085         $format = 'tar' if !$format;
6086         $comp = 'lzop';
6087     } elsif ($archive =~ m/\.vma$/) {
6088         $format = 'vma' if !$format;
6089     } elsif ($archive =~ m/\.vma\.gz$/) {
6090         $format = 'vma' if !$format;
6091         $comp = 'gzip';
6092     } elsif ($archive =~ m/\.vma\.lzo$/) {
6093         $format = 'vma' if !$format;
6094         $comp = 'lzop';
6095     } else {
6096         $format = 'vma' if !$format; # default
6097     }
6098
6099     # try to detect archive format
6100     if ($format eq 'tar') {
6101         return restore_tar_archive($archive, $vmid, $user, $opts);
6102     } else {
6103         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
6104     }
6105 }
6106
6107 sub restore_update_config_line {
6108     my ($outfd, $cookie, $vmid, $map, $line, $unique) = @_;
6109
6110     return if $line =~ m/^\#qmdump\#/;
6111     return if $line =~ m/^\#vzdump\#/;
6112     return if $line =~ m/^lock:/;
6113     return if $line =~ m/^unused\d+:/;
6114     return if $line =~ m/^parent:/;
6115
6116     my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
6117     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
6118         # try to convert old 1.X settings
6119         my ($id, $ind, $ethcfg) = ($1, $2, $3);
6120         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
6121             my ($model, $macaddr) = split(/\=/, $devconfig);
6122             $macaddr = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if !$macaddr || $unique;
6123             my $net = {
6124                 model => $model,
6125                 bridge => "vmbr$ind",
6126                 macaddr => $macaddr,
6127             };
6128             my $netstr = print_net($net);
6129
6130             print $outfd "net$cookie->{netcount}: $netstr\n";
6131             $cookie->{netcount}++;
6132         }
6133     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
6134         my ($id, $netstr) = ($1, $2);
6135         my $net = parse_net($netstr);
6136         $net->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if $net->{macaddr};
6137         $netstr = print_net($net);
6138         print $outfd "$id: $netstr\n";
6139     } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk)\d+):\s*(\S+)\s*$/) {
6140         my $virtdev = $1;
6141         my $value = $3;
6142         my $di = parse_drive($virtdev, $value);
6143         if (defined($di->{backup}) && !$di->{backup}) {
6144             print $outfd "#$line";
6145         } elsif ($map->{$virtdev}) {
6146             delete $di->{format}; # format can change on restore
6147             $di->{file} = $map->{$virtdev};
6148             $value = print_drive($vmid, $di);
6149             print $outfd "$virtdev: $value\n";
6150         } else {
6151             print $outfd $line;
6152         }
6153     } elsif (($line =~ m/^vmgenid: (.*)/)) {
6154         my $vmgenid = $1;
6155         if ($vmgenid ne '0') {
6156             # always generate a new vmgenid if there was a valid one setup
6157             $vmgenid = generate_uuid();
6158         }
6159         print $outfd "vmgenid: $vmgenid\n";
6160     } elsif (($line =~ m/^(smbios1: )(.*)/) && $unique) {
6161         my ($uuid, $uuid_str);
6162         UUID::generate($uuid);
6163         UUID::unparse($uuid, $uuid_str);
6164         my $smbios1 = parse_smbios1($2);
6165         $smbios1->{uuid} = $uuid_str;
6166         print $outfd $1.print_smbios1($smbios1)."\n";
6167     } else {
6168         print $outfd $line;
6169     }
6170 }
6171
6172 sub scan_volids {
6173     my ($cfg, $vmid) = @_;
6174
6175     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid);
6176
6177     my $volid_hash = {};
6178     foreach my $storeid (keys %$info) {
6179         foreach my $item (@{$info->{$storeid}}) {
6180             next if !($item->{volid} && $item->{size});
6181             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
6182             $volid_hash->{$item->{volid}} = $item;
6183         }
6184     }
6185
6186     return $volid_hash;
6187 }
6188
6189 sub is_volume_in_use {
6190     my ($storecfg, $conf, $skip_drive, $volid) = @_;
6191
6192     my $path = PVE::Storage::path($storecfg, $volid);
6193
6194     my $scan_config = sub {
6195         my ($cref, $snapname) = @_;
6196
6197         foreach my $key (keys %$cref) {
6198             my $value = $cref->{$key};
6199             if (is_valid_drivename($key)) {
6200                 next if $skip_drive && $key eq $skip_drive;
6201                 my $drive = parse_drive($key, $value);
6202                 next if !$drive || !$drive->{file} || drive_is_cdrom($drive);
6203                 return 1 if $volid eq $drive->{file};
6204                 if ($drive->{file} =~ m!^/!) {
6205                     return 1 if $drive->{file} eq $path;
6206                 } else {
6207                     my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file}, 1);
6208                     next if !$storeid;
6209                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid, 1);
6210                     next if !$scfg;
6211                     return 1 if $path eq PVE::Storage::path($storecfg, $drive->{file}, $snapname);
6212                 }
6213             }
6214         }
6215
6216         return 0;
6217     };
6218
6219     return 1 if &$scan_config($conf);
6220
6221     undef $skip_drive;
6222
6223     foreach my $snapname (keys %{$conf->{snapshots}}) {
6224         return 1 if &$scan_config($conf->{snapshots}->{$snapname}, $snapname);
6225     }
6226
6227     return 0;
6228 }
6229
6230 sub update_disksize {
6231     my ($vmid, $conf, $volid_hash) = @_;
6232
6233     my $changes;
6234     my $prefix = "VM $vmid:";
6235
6236     # used and unused disks
6237     my $referenced = {};
6238
6239     # Note: it is allowed to define multiple storages with same path (alias), so
6240     # we need to check both 'volid' and real 'path' (two different volid can point
6241     # to the same path).
6242
6243     my $referencedpath = {};
6244
6245     # update size info
6246     foreach my $opt (keys %$conf) {
6247         if (is_valid_drivename($opt)) {
6248             my $drive = parse_drive($opt, $conf->{$opt});
6249             my $volid = $drive->{file};
6250             next if !$volid;
6251
6252             $referenced->{$volid} = 1;
6253             if ($volid_hash->{$volid} &&
6254                 (my $path = $volid_hash->{$volid}->{path})) {
6255                 $referencedpath->{$path} = 1;
6256             }
6257
6258             next if drive_is_cdrom($drive);
6259             next if !$volid_hash->{$volid};
6260
6261             $drive->{size} = $volid_hash->{$volid}->{size};
6262             my $new = print_drive($vmid, $drive);
6263             if ($new ne $conf->{$opt}) {
6264                 $changes = 1;
6265                 $conf->{$opt} = $new;
6266                 print "$prefix update disk '$opt' information.\n";
6267             }
6268         }
6269     }
6270
6271     # remove 'unusedX' entry if volume is used
6272     foreach my $opt (keys %$conf) {
6273         next if $opt !~ m/^unused\d+$/;
6274         my $volid = $conf->{$opt};
6275         my $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
6276         if ($referenced->{$volid} || ($path && $referencedpath->{$path})) {
6277             print "$prefix remove entry '$opt', its volume '$volid' is in use.\n";
6278             $changes = 1;
6279             delete $conf->{$opt};
6280         }
6281
6282         $referenced->{$volid} = 1;
6283         $referencedpath->{$path} = 1 if $path;
6284     }
6285
6286     foreach my $volid (sort keys %$volid_hash) {
6287         next if $volid =~ m/vm-$vmid-state-/;
6288         next if $referenced->{$volid};
6289         my $path = $volid_hash->{$volid}->{path};
6290         next if !$path; # just to be sure
6291         next if $referencedpath->{$path};
6292         $changes = 1;
6293         my $key = PVE::QemuConfig->add_unused_volume($conf, $volid);
6294         print "$prefix add unreferenced volume '$volid' as '$key' to config.\n";
6295         $referencedpath->{$path} = 1; # avoid to add more than once (aliases)
6296     }
6297
6298     return $changes;
6299 }
6300
6301 sub rescan {
6302     my ($vmid, $nolock, $dryrun) = @_;
6303
6304     my $cfg = PVE::Storage::config();
6305
6306     # FIXME: Remove once our RBD plugin can handle CT and VM on a single storage
6307     # see: https://pve.proxmox.com/pipermail/pve-devel/2018-July/032900.html
6308     foreach my $stor (keys %{$cfg->{ids}}) {
6309         delete($cfg->{ids}->{$stor}) if ! $cfg->{ids}->{$stor}->{content}->{images};
6310     }
6311
6312     print "rescan volumes...\n";
6313     my $volid_hash = scan_volids($cfg, $vmid);
6314
6315     my $updatefn =  sub {
6316         my ($vmid) = @_;
6317
6318         my $conf = PVE::QemuConfig->load_config($vmid);
6319
6320         PVE::QemuConfig->check_lock($conf);
6321
6322         my $vm_volids = {};
6323         foreach my $volid (keys %$volid_hash) {
6324             my $info = $volid_hash->{$volid};
6325             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
6326         }
6327
6328         my $changes = update_disksize($vmid, $conf, $vm_volids);
6329
6330         PVE::QemuConfig->write_config($vmid, $conf) if $changes && !$dryrun;
6331     };
6332
6333     if (defined($vmid)) {
6334         if ($nolock) {
6335             &$updatefn($vmid);
6336         } else {
6337             PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6338         }
6339     } else {
6340         my $vmlist = config_list();
6341         foreach my $vmid (keys %$vmlist) {
6342             if ($nolock) {
6343                 &$updatefn($vmid);
6344             } else {
6345                 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6346             }
6347         }
6348     }
6349 }
6350
6351 sub restore_vma_archive {
6352     my ($archive, $vmid, $user, $opts, $comp) = @_;
6353
6354     my $readfrom = $archive;
6355
6356     my $cfg = PVE::Storage::config();
6357     my $commands = [];
6358     my $bwlimit = $opts->{bwlimit};
6359
6360     my $dbg_cmdstring = '';
6361     my $add_pipe = sub {
6362         my ($cmd) = @_;
6363         push @$commands, $cmd;
6364         $dbg_cmdstring .= ' | ' if length($dbg_cmdstring);
6365         $dbg_cmdstring .= PVE::Tools::cmd2string($cmd);
6366         $readfrom = '-';
6367     };
6368
6369     my $input = undef;
6370     if ($archive eq '-') {
6371         $input = '<&STDIN';
6372     } else {
6373         # If we use a backup from a PVE defined storage we also consider that
6374         # storage's rate limit:
6375         my (undef, $volid) = PVE::Storage::path_to_volume_id($cfg, $archive);
6376         if (defined($volid)) {
6377             my ($sid, undef) = PVE::Storage::parse_volume_id($volid);
6378             my $readlimit = PVE::Storage::get_bandwidth_limit('restore', [$sid], $bwlimit);
6379             if ($readlimit) {
6380                 print STDERR "applying read rate limit: $readlimit\n";
6381                 my $cstream = ['cstream', '-t', $readlimit*1024, '--', $readfrom];
6382                 $add_pipe->($cstream);
6383             }
6384         }
6385     }
6386
6387     if ($comp) {
6388         my $cmd;
6389         if ($comp eq 'gzip') {
6390             $cmd = ['zcat', $readfrom];
6391         } elsif ($comp eq 'lzop') {
6392             $cmd = ['lzop', '-d', '-c', $readfrom];
6393         } else {
6394             die "unknown compression method '$comp'\n";
6395         }
6396         $add_pipe->($cmd);
6397     }
6398
6399     my $tmpdir = "/var/tmp/vzdumptmp$$";
6400     rmtree $tmpdir;
6401
6402     # disable interrupts (always do cleanups)
6403     local $SIG{INT} =
6404         local $SIG{TERM} =
6405         local $SIG{QUIT} =
6406         local $SIG{HUP} = sub { warn "got interrupt - ignored\n"; };
6407
6408     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
6409     POSIX::mkfifo($mapfifo, 0600);
6410     my $fifofh;
6411
6412     my $openfifo = sub {
6413         open($fifofh, '>', $mapfifo) || die $!;
6414     };
6415
6416     $add_pipe->(['vma', 'extract', '-v', '-r', $mapfifo, $readfrom, $tmpdir]);
6417
6418     my $oldtimeout;
6419     my $timeout = 5;
6420
6421     my $devinfo = {};
6422
6423     my $rpcenv = PVE::RPCEnvironment::get();
6424
6425     my $conffile = PVE::QemuConfig->config_file($vmid);
6426     my $tmpfn = "$conffile.$$.tmp";
6427
6428     # Note: $oldconf is undef if VM does not exists
6429     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
6430     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
6431
6432     my %storage_limits;
6433
6434     my $print_devmap = sub {
6435         my $virtdev_hash = {};
6436
6437         my $cfgfn = "$tmpdir/qemu-server.conf";
6438
6439         # we can read the config - that is already extracted
6440         my $fh = IO::File->new($cfgfn, "r") ||
6441             "unable to read qemu-server.conf - $!\n";
6442
6443         my $fwcfgfn = "$tmpdir/qemu-server.fw";
6444         if (-f $fwcfgfn) {
6445             my $pve_firewall_dir = '/etc/pve/firewall';
6446             mkdir $pve_firewall_dir; # make sure the dir exists
6447             PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
6448         }
6449
6450         while (defined(my $line = <$fh>)) {
6451             if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
6452                 my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
6453                 die "archive does not contain data for drive '$virtdev'\n"
6454                     if !$devinfo->{$devname};
6455                 if (defined($opts->{storage})) {
6456                     $storeid = $opts->{storage} || 'local';
6457                 } elsif (!$storeid) {
6458                     $storeid = 'local';
6459                 }
6460                 $format = 'raw' if !$format;
6461                 $devinfo->{$devname}->{devname} = $devname;
6462                 $devinfo->{$devname}->{virtdev} = $virtdev;
6463                 $devinfo->{$devname}->{format} = $format;
6464                 $devinfo->{$devname}->{storeid} = $storeid;
6465
6466                 # check permission on storage
6467                 my $pool = $opts->{pool}; # todo: do we need that?
6468                 if ($user ne 'root@pam') {
6469                     $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
6470                 }
6471
6472                 $storage_limits{$storeid} = $bwlimit;
6473
6474                 $virtdev_hash->{$virtdev} = $devinfo->{$devname};
6475             } elsif ($line =~ m/^((?:ide|sata|scsi)\d+):\s*(.*)\s*$/) {
6476                 my $virtdev = $1;
6477                 my $drive = parse_drive($virtdev, $2);
6478                 if (drive_is_cloudinit($drive)) {
6479                     my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6480                     my $scfg = PVE::Storage::storage_config($cfg, $storeid);
6481                     my $format = qemu_img_format($scfg, $volname); # has 'raw' fallback
6482
6483                     my $d = {
6484                         format => $format,
6485                         storeid => $opts->{storage} // $storeid,
6486                         size => PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE,
6487                         file => $drive->{file}, # to make drive_is_cloudinit check possible
6488                         name => "vm-$vmid-cloudinit",
6489                         is_cloudinit => 1,
6490                     };
6491                     $virtdev_hash->{$virtdev} = $d;
6492                 }
6493             }
6494         }
6495
6496         foreach my $key (keys %storage_limits) {
6497             my $limit = PVE::Storage::get_bandwidth_limit('restore', [$key], $bwlimit);
6498             next if !$limit;
6499             print STDERR "rate limit for storage $key: $limit KiB/s\n";
6500             $storage_limits{$key} = $limit * 1024;
6501         }
6502
6503         foreach my $devname (keys %$devinfo) {
6504             die "found no device mapping information for device '$devname'\n"
6505                 if !$devinfo->{$devname}->{virtdev};
6506         }
6507
6508         # create empty/temp config
6509         if ($oldconf) {
6510             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
6511             foreach_drive($oldconf, sub {
6512                 my ($ds, $drive) = @_;
6513
6514                 return if drive_is_cdrom($drive, 1);
6515
6516                 my $volid = $drive->{file};
6517                 return if !$volid || $volid =~ m|^/|;
6518
6519                 my ($path, $owner) = PVE::Storage::path($cfg, $volid);
6520                 return if !$path || !$owner || ($owner != $vmid);
6521
6522                 # Note: only delete disk we want to restore
6523                 # other volumes will become unused
6524                 if ($virtdev_hash->{$ds}) {
6525                     eval { PVE::Storage::vdisk_free($cfg, $volid); };
6526                     if (my $err = $@) {
6527                         warn $err;
6528                     }
6529                 }
6530             });
6531
6532             # delete vmstate files, after the restore we have no snapshots anymore
6533             foreach my $snapname (keys %{$oldconf->{snapshots}}) {
6534                 my $snap = $oldconf->{snapshots}->{$snapname};
6535                 if ($snap->{vmstate}) {
6536                     eval { PVE::Storage::vdisk_free($cfg, $snap->{vmstate}); };
6537                     if (my $err = $@) {
6538                         warn $err;
6539                     }
6540                 }
6541             }
6542         }
6543
6544         my $map = {};
6545         foreach my $virtdev (sort keys %$virtdev_hash) {
6546             my $d = $virtdev_hash->{$virtdev};
6547             my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
6548             my $storeid = $d->{storeid};
6549             my $scfg = PVE::Storage::storage_config($cfg, $storeid);
6550
6551             my $map_opts = '';
6552             if (my $limit = $storage_limits{$storeid}) {
6553                 $map_opts .= "throttling.bps=$limit:throttling.group=$storeid:";
6554             }
6555
6556             # test if requested format is supported
6557             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($cfg, $storeid);
6558             my $supported = grep { $_ eq $d->{format} } @$validFormats;
6559             $d->{format} = $defFormat if !$supported;
6560
6561             my $name;
6562             if ($d->{is_cloudinit}) {
6563                 $name = $d->{name};
6564                 $name .= ".$d->{format}" if $d->{format} ne 'raw';
6565             }
6566
6567             my $volid = PVE::Storage::vdisk_alloc($cfg, $storeid, $vmid, $d->{format}, $name, $alloc_size);
6568             print STDERR "new volume ID is '$volid'\n";
6569             $d->{volid} = $volid;
6570
6571             PVE::Storage::activate_volumes($cfg, [$volid]);
6572
6573             my $write_zeros = 1;
6574             if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
6575                 $write_zeros = 0;
6576             }
6577
6578             if (!$d->{is_cloudinit}) {
6579                 my $path = PVE::Storage::path($cfg, $volid);
6580
6581                 print $fifofh "${map_opts}format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
6582
6583                 print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
6584             }
6585             $map->{$virtdev} = $volid;
6586         }
6587
6588         $fh->seek(0, 0) || die "seek failed - $!\n";
6589
6590         my $outfd = new IO::File ($tmpfn, "w") ||
6591             die "unable to write config for VM $vmid\n";
6592
6593         my $cookie = { netcount => 0 };
6594         while (defined(my $line = <$fh>)) {
6595             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
6596         }
6597
6598         $fh->close();
6599         $outfd->close();
6600     };
6601
6602     eval {
6603         # enable interrupts
6604         local $SIG{INT} =
6605             local $SIG{TERM} =
6606             local $SIG{QUIT} =
6607             local $SIG{HUP} =
6608             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6609         local $SIG{ALRM} = sub { die "got timeout\n"; };
6610
6611         $oldtimeout = alarm($timeout);
6612
6613         my $parser = sub {
6614             my $line = shift;
6615
6616             print "$line\n";
6617
6618             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
6619                 my ($dev_id, $size, $devname) = ($1, $2, $3);
6620                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
6621             } elsif ($line =~ m/^CTIME: /) {
6622                 # we correctly received the vma config, so we can disable
6623                 # the timeout now for disk allocation (set to 10 minutes, so
6624                 # that we always timeout if something goes wrong)
6625                 alarm(600);
6626                 &$print_devmap();
6627                 print $fifofh "done\n";
6628                 my $tmp = $oldtimeout || 0;
6629                 $oldtimeout = undef;
6630                 alarm($tmp);
6631                 close($fifofh);
6632             }
6633         };
6634
6635         print "restore vma archive: $dbg_cmdstring\n";
6636         run_command($commands, input => $input, outfunc => $parser, afterfork => $openfifo);
6637     };
6638     my $err = $@;
6639
6640     alarm($oldtimeout) if $oldtimeout;
6641
6642     my $vollist = [];
6643     foreach my $devname (keys %$devinfo) {
6644         my $volid = $devinfo->{$devname}->{volid};
6645         push @$vollist, $volid if $volid;
6646     }
6647
6648     PVE::Storage::deactivate_volumes($cfg, $vollist);
6649
6650     unlink $mapfifo;
6651
6652     if ($err) {
6653         rmtree $tmpdir;
6654         unlink $tmpfn;
6655
6656         foreach my $devname (keys %$devinfo) {
6657             my $volid = $devinfo->{$devname}->{volid};
6658             next if !$volid;
6659             eval {
6660                 if ($volid =~ m|^/|) {
6661                     unlink $volid || die 'unlink failed\n';
6662                 } else {
6663                     PVE::Storage::vdisk_free($cfg, $volid);
6664                 }
6665                 print STDERR "temporary volume '$volid' sucessfuly removed\n";
6666             };
6667             print STDERR "unable to cleanup '$volid' - $@" if $@;
6668         }
6669         die $err;
6670     }
6671
6672     rmtree $tmpdir;
6673
6674     rename($tmpfn, $conffile) ||
6675         die "unable to commit configuration file '$conffile'\n";
6676
6677     PVE::Cluster::cfs_update(); # make sure we read new file
6678
6679     eval { rescan($vmid, 1); };
6680     warn $@ if $@;
6681 }
6682
6683 sub restore_tar_archive {
6684     my ($archive, $vmid, $user, $opts) = @_;
6685
6686     if ($archive ne '-') {
6687         my $firstfile = tar_archive_read_firstfile($archive);
6688         die "ERROR: file '$archive' dos not lock like a QemuServer vzdump backup\n"
6689             if $firstfile ne 'qemu-server.conf';
6690     }
6691
6692     my $storecfg = PVE::Storage::config();
6693
6694     # destroy existing data - keep empty config
6695     my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
6696     destroy_vm($storecfg, $vmid, 1) if -f $vmcfgfn;
6697
6698     my $tocmd = "/usr/lib/qemu-server/qmextract";
6699
6700     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
6701     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
6702     $tocmd .= ' --prealloc' if $opts->{prealloc};
6703     $tocmd .= ' --info' if $opts->{info};
6704
6705     # tar option "xf" does not autodetect compression when read from STDIN,
6706     # so we pipe to zcat
6707     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
6708         PVE::Tools::shellquote("--to-command=$tocmd");
6709
6710     my $tmpdir = "/var/tmp/vzdumptmp$$";
6711     mkpath $tmpdir;
6712
6713     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
6714     local $ENV{VZDUMP_VMID} = $vmid;
6715     local $ENV{VZDUMP_USER} = $user;
6716
6717     my $conffile = PVE::QemuConfig->config_file($vmid);
6718     my $tmpfn = "$conffile.$$.tmp";
6719
6720     # disable interrupts (always do cleanups)
6721     local $SIG{INT} =
6722         local $SIG{TERM} =
6723         local $SIG{QUIT} =
6724         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
6725
6726     eval {
6727         # enable interrupts
6728         local $SIG{INT} =
6729             local $SIG{TERM} =
6730             local $SIG{QUIT} =
6731             local $SIG{HUP} =
6732             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6733
6734         if ($archive eq '-') {
6735             print "extracting archive from STDIN\n";
6736             run_command($cmd, input => "<&STDIN");
6737         } else {
6738             print "extracting archive '$archive'\n";
6739             run_command($cmd);
6740         }
6741
6742         return if $opts->{info};
6743
6744         # read new mapping
6745         my $map = {};
6746         my $statfile = "$tmpdir/qmrestore.stat";
6747         if (my $fd = IO::File->new($statfile, "r")) {
6748             while (defined (my $line = <$fd>)) {
6749                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
6750                     $map->{$1} = $2 if $1;
6751                 } else {
6752                     print STDERR "unable to parse line in statfile - $line\n";
6753                 }
6754             }
6755             $fd->close();
6756         }
6757
6758         my $confsrc = "$tmpdir/qemu-server.conf";
6759
6760         my $srcfd = new IO::File($confsrc, "r") ||
6761             die "unable to open file '$confsrc'\n";
6762
6763         my $outfd = new IO::File ($tmpfn, "w") ||
6764             die "unable to write config for VM $vmid\n";
6765
6766         my $cookie = { netcount => 0 };
6767         while (defined (my $line = <$srcfd>)) {
6768             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
6769         }
6770
6771         $srcfd->close();
6772         $outfd->close();
6773     };
6774     my $err = $@;
6775
6776     if ($err) {
6777
6778         unlink $tmpfn;
6779
6780         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
6781
6782         die $err;
6783     }
6784
6785     rmtree $tmpdir;
6786
6787     rename $tmpfn, $conffile ||
6788         die "unable to commit configuration file '$conffile'\n";
6789
6790     PVE::Cluster::cfs_update(); # make sure we read new file
6791
6792     eval { rescan($vmid, 1); };
6793     warn $@ if $@;
6794 };
6795
6796 sub foreach_storage_used_by_vm {
6797     my ($conf, $func) = @_;
6798
6799     my $sidhash = {};
6800
6801     foreach_drive($conf, sub {
6802         my ($ds, $drive) = @_;
6803         return if drive_is_cdrom($drive);
6804
6805         my $volid = $drive->{file};
6806
6807         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
6808         $sidhash->{$sid} = $sid if $sid;
6809     });
6810
6811     foreach my $sid (sort keys %$sidhash) {
6812         &$func($sid);
6813     }
6814 }
6815
6816 sub do_snapshots_with_qemu {
6817     my ($storecfg, $volid) = @_;
6818
6819     my $storage_name = PVE::Storage::parse_volume_id($volid);
6820     my $scfg = $storecfg->{ids}->{$storage_name};
6821
6822     if ($qemu_snap_storage->{$scfg->{type}} && !$scfg->{krbd}){
6823         return 1;
6824     }
6825
6826     if ($volid =~ m/\.(qcow2|qed)$/){
6827         return 1;
6828     }
6829
6830     return undef;
6831 }
6832
6833 sub qga_check_running {
6834     my ($vmid, $nowarn) = @_;
6835
6836     eval { vm_mon_cmd($vmid, "guest-ping", timeout => 3); };
6837     if ($@) {
6838         warn "Qemu Guest Agent is not running - $@" if !$nowarn;
6839         return 0;
6840     }
6841     return 1;
6842 }
6843
6844 sub template_create {
6845     my ($vmid, $conf, $disk) = @_;
6846
6847     my $storecfg = PVE::Storage::config();
6848
6849     foreach_drive($conf, sub {
6850         my ($ds, $drive) = @_;
6851
6852         return if drive_is_cdrom($drive);
6853         return if $disk && $ds ne $disk;
6854
6855         my $volid = $drive->{file};
6856         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
6857
6858         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
6859         $drive->{file} = $voliddst;
6860         $conf->{$ds} = print_drive($vmid, $drive);
6861         PVE::QemuConfig->write_config($vmid, $conf);
6862     });
6863 }
6864
6865 sub convert_iscsi_path {
6866     my ($path) = @_;
6867
6868     if ($path =~ m|^iscsi://([^/]+)/([^/]+)/(.+)$|) {
6869         my $portal = $1;
6870         my $target = $2;
6871         my $lun = $3;
6872
6873         my $initiator_name = get_initiator_name();
6874
6875         return "file.driver=iscsi,file.transport=tcp,file.initiator-name=$initiator_name,".
6876                "file.portal=$portal,file.target=$target,file.lun=$lun,driver=raw";
6877     }
6878
6879     die "cannot convert iscsi path '$path', unkown format\n";
6880 }
6881
6882 sub qemu_img_convert {
6883     my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized) = @_;
6884
6885     my $storecfg = PVE::Storage::config();
6886     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
6887     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
6888
6889     die "destination '$dst_volid' is not a valid volid form qemu-img convert\n" if !$dst_storeid;
6890
6891     my $cachemode;
6892     my $src_path;
6893     my $src_is_iscsi = 0;
6894     my $src_format = 'raw';
6895
6896     if ($src_storeid) {
6897         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
6898         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
6899         $src_format = qemu_img_format($src_scfg, $src_volname);
6900         $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
6901         $src_is_iscsi = ($src_path =~ m|^iscsi://|);
6902         $cachemode = 'none' if $src_scfg->{type} eq 'zfspool';
6903     } elsif (-f $src_volid) {
6904         $src_path = $src_volid;
6905         if ($src_path =~ m/\.($QEMU_FORMAT_RE)$/) {
6906             $src_format = $1;
6907         }
6908     }
6909
6910     die "source '$src_volid' is not a valid volid nor path for qemu-img convert\n" if !$src_path;
6911
6912     my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6913     my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
6914     my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6915     my $dst_is_iscsi = ($dst_path =~ m|^iscsi://|);
6916
6917     my $cmd = [];
6918     push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
6919     push @$cmd, '-l', "snapshot.name=$snapname" if($snapname && $src_format eq "qcow2");
6920     push @$cmd, '-t', 'none' if $dst_scfg->{type} eq 'zfspool';
6921     push @$cmd, '-T', $cachemode if defined($cachemode);
6922
6923     if ($src_is_iscsi) {
6924         push @$cmd, '--image-opts';
6925         $src_path = convert_iscsi_path($src_path);
6926     } else {
6927         push @$cmd, '-f', $src_format;
6928     }
6929
6930     if ($dst_is_iscsi) {
6931         push @$cmd, '--target-image-opts';
6932         $dst_path = convert_iscsi_path($dst_path);
6933     } else {
6934         push @$cmd, '-O', $dst_format;
6935     }
6936
6937     push @$cmd, $src_path;
6938
6939     if (!$dst_is_iscsi && $is_zero_initialized) {
6940         push @$cmd, "zeroinit:$dst_path";
6941     } else {
6942         push @$cmd, $dst_path;
6943     }
6944
6945     my $parser = sub {
6946         my $line = shift;
6947         if($line =~ m/\((\S+)\/100\%\)/){
6948             my $percent = $1;
6949             my $transferred = int($size * $percent / 100);
6950             my $remaining = $size - $transferred;
6951
6952             print "transferred: $transferred bytes remaining: $remaining bytes total: $size bytes progression: $percent %\n";
6953         }
6954
6955     };
6956
6957     eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
6958     my $err = $@;
6959     die "copy failed: $err" if $err;
6960 }
6961
6962 sub qemu_img_format {
6963     my ($scfg, $volname) = @_;
6964
6965     if ($scfg->{path} && $volname =~ m/\.($QEMU_FORMAT_RE)$/) {
6966         return $1;
6967     } else {
6968         return "raw";
6969     }
6970 }
6971
6972 sub qemu_drive_mirror {
6973     my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $skipcomplete, $qga, $bwlimit) = @_;
6974
6975     $jobs = {} if !$jobs;
6976
6977     my $qemu_target;
6978     my $format;
6979     $jobs->{"drive-$drive"} = {};
6980
6981     if ($dst_volid =~ /^nbd:/) {
6982         $qemu_target = $dst_volid;
6983         $format = "nbd";
6984     } else {
6985         my $storecfg = PVE::Storage::config();
6986         my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
6987
6988         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6989
6990         $format = qemu_img_format($dst_scfg, $dst_volname);
6991
6992         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6993
6994         $qemu_target = $is_zero_initialized ? "zeroinit:$dst_path" : $dst_path;
6995     }
6996
6997     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $qemu_target };
6998     $opts->{format} = $format if $format;
6999
7000     if (defined($bwlimit)) {
7001         $opts->{speed} = $bwlimit * 1024;
7002         print "drive mirror is starting for drive-$drive with bandwidth limit: ${bwlimit} KB/s\n";
7003     } else {
7004         print "drive mirror is starting for drive-$drive\n";
7005     }
7006
7007     # if a job already runs for this device we get an error, catch it for cleanup
7008     eval { vm_mon_cmd($vmid, "drive-mirror", %$opts); };
7009     if (my $err = $@) {
7010         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
7011         warn "$@\n" if $@;
7012         die "mirroring error: $err\n";
7013     }
7014
7015     qemu_drive_mirror_monitor ($vmid, $vmiddst, $jobs, $skipcomplete, $qga);
7016 }
7017
7018 sub qemu_drive_mirror_monitor {
7019     my ($vmid, $vmiddst, $jobs, $skipcomplete, $qga) = @_;
7020
7021     eval {
7022         my $err_complete = 0;
7023
7024         while (1) {
7025             die "storage migration timed out\n" if $err_complete > 300;
7026
7027             my $stats = vm_mon_cmd($vmid, "query-block-jobs");
7028
7029             my $running_mirror_jobs = {};
7030             foreach my $stat (@$stats) {
7031                 next if $stat->{type} ne 'mirror';
7032                 $running_mirror_jobs->{$stat->{device}} = $stat;
7033             }
7034
7035             my $readycounter = 0;
7036
7037             foreach my $job (keys %$jobs) {
7038
7039                 if(defined($jobs->{$job}->{complete}) && !defined($running_mirror_jobs->{$job})) {
7040                     print "$job : finished\n";
7041                     delete $jobs->{$job};
7042                     next;
7043                 }
7044
7045                 die "$job: mirroring has been cancelled\n" if !defined($running_mirror_jobs->{$job});
7046
7047                 my $busy = $running_mirror_jobs->{$job}->{busy};
7048                 my $ready = $running_mirror_jobs->{$job}->{ready};
7049                 if (my $total = $running_mirror_jobs->{$job}->{len}) {
7050                     my $transferred = $running_mirror_jobs->{$job}->{offset} || 0;
7051                     my $remaining = $total - $transferred;
7052                     my $percent = sprintf "%.2f", ($transferred * 100 / $total);
7053
7054                     print "$job: transferred: $transferred bytes remaining: $remaining bytes total: $total bytes progression: $percent % busy: $busy ready: $ready \n";
7055                 }
7056
7057                 $readycounter++ if $running_mirror_jobs->{$job}->{ready};
7058             }
7059
7060             last if scalar(keys %$jobs) == 0;
7061
7062             if ($readycounter == scalar(keys %$jobs)) {
7063                 print "all mirroring jobs are ready \n";
7064                 last if $skipcomplete; #do the complete later
7065
7066                 if ($vmiddst && $vmiddst != $vmid) {
7067                     my $agent_running = $qga && qga_check_running($vmid);
7068                     if ($agent_running) {
7069                         print "freeze filesystem\n";
7070                         eval { PVE::QemuServer::vm_mon_cmd($vmid, "guest-fsfreeze-freeze"); };
7071                     } else {
7072                         print "suspend vm\n";
7073                         eval { PVE::QemuServer::vm_suspend($vmid, 1); };
7074                     }
7075
7076                     # if we clone a disk for a new target vm, we don't switch the disk
7077                     PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs);
7078
7079                     if ($agent_running) {
7080                         print "unfreeze filesystem\n";
7081                         eval { PVE::QemuServer::vm_mon_cmd($vmid, "guest-fsfreeze-thaw"); };
7082                     } else {
7083                         print "resume vm\n";
7084                         eval {  PVE::QemuServer::vm_resume($vmid, 1, 1); };
7085                     }
7086
7087                     last;
7088                 } else {
7089
7090                     foreach my $job (keys %$jobs) {
7091                         # try to switch the disk if source and destination are on the same guest
7092                         print "$job: Completing block job...\n";
7093
7094                         eval { vm_mon_cmd($vmid, "block-job-complete", device => $job) };
7095                         if ($@ =~ m/cannot be completed/) {
7096                             print "$job: Block job cannot be completed, try again.\n";
7097                             $err_complete++;
7098                         }else {
7099                             print "$job: Completed successfully.\n";
7100                             $jobs->{$job}->{complete} = 1;
7101                         }
7102                     }
7103                 }
7104             }
7105             sleep 1;
7106         }
7107     };
7108     my $err = $@;
7109
7110     if ($err) {
7111         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
7112         die "mirroring error: $err";
7113     }
7114
7115 }
7116
7117 sub qemu_blockjobs_cancel {
7118     my ($vmid, $jobs) = @_;
7119
7120     foreach my $job (keys %$jobs) {
7121         print "$job: Cancelling block job\n";
7122         eval { vm_mon_cmd($vmid, "block-job-cancel", device => $job); };
7123         $jobs->{$job}->{cancel} = 1;
7124     }
7125
7126     while (1) {
7127         my $stats = vm_mon_cmd($vmid, "query-block-jobs");
7128
7129         my $running_jobs = {};
7130         foreach my $stat (@$stats) {
7131             $running_jobs->{$stat->{device}} = $stat;
7132         }
7133
7134         foreach my $job (keys %$jobs) {
7135
7136             if (defined($jobs->{$job}->{cancel}) && !defined($running_jobs->{$job})) {
7137                 print "$job: Done.\n";
7138                 delete $jobs->{$job};
7139             }
7140         }
7141
7142         last if scalar(keys %$jobs) == 0;
7143
7144         sleep 1;
7145     }
7146 }
7147
7148 sub clone_disk {
7149     my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
7150         $newvmid, $storage, $format, $full, $newvollist, $jobs, $skipcomplete, $qga, $bwlimit) = @_;
7151
7152     my $newvolid;
7153
7154     if (!$full) {
7155         print "create linked clone of drive $drivename ($drive->{file})\n";
7156         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
7157         push @$newvollist, $newvolid;
7158     } else {
7159
7160         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
7161         $storeid = $storage if $storage;
7162
7163         my $dst_format = resolve_dst_disk_format($storecfg, $storeid, $volname, $format);
7164         my ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 3);
7165
7166         print "create full clone of drive $drivename ($drive->{file})\n";
7167         my $name = undef;
7168         $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $newvmid, $dst_format, $name, ($size/1024));
7169         push @$newvollist, $newvolid;
7170
7171         PVE::Storage::activate_volumes($storecfg, [$newvolid]);
7172
7173         my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
7174         if (!$running || $snapname) {
7175             # TODO: handle bwlimits
7176             qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit);
7177         } else {
7178
7179             my $kvmver = get_running_qemu_version ($vmid);
7180             if (!qemu_machine_feature_enabled (undef, $kvmver, 2, 7)) {
7181                 die "drive-mirror with iothread requires qemu version 2.7 or higher\n"
7182                     if $drive->{iothread};
7183             }
7184
7185             qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid, $sparseinit, $jobs, $skipcomplete, $qga, $bwlimit);
7186         }
7187     }
7188
7189     my ($size) = PVE::Storage::volume_size_info($storecfg, $newvolid, 3);
7190
7191     my $disk = $drive;
7192     $disk->{format} = undef;
7193     $disk->{file} = $newvolid;
7194     $disk->{size} = $size;
7195
7196     return $disk;
7197 }
7198
7199 # this only works if VM is running
7200 sub get_current_qemu_machine {
7201     my ($vmid) = @_;
7202
7203     my $cmd = { execute => 'query-machines', arguments => {} };
7204     my $res = vm_qmp_command($vmid, $cmd);
7205
7206     my ($current, $default);
7207     foreach my $e (@$res) {
7208         $default = $e->{name} if $e->{'is-default'};
7209         $current = $e->{name} if $e->{'is-current'};
7210     }
7211
7212     # fallback to the default machine if current is not supported by qemu
7213     return $current || $default || 'pc';
7214 }
7215
7216 sub get_running_qemu_version {
7217     my ($vmid) = @_;
7218     my $cmd = { execute => 'query-version', arguments => {} };
7219     my $res = vm_qmp_command($vmid, $cmd);
7220     return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
7221 }
7222
7223 sub qemu_machine_feature_enabled {
7224     my ($machine, $kvmver, $version_major, $version_minor) = @_;
7225
7226     my $current_major;
7227     my $current_minor;
7228
7229     if ($machine && $machine =~ m/^((?:pc(-i440fx|-q35)?|virt)-(\d+)\.(\d+))/) {
7230
7231         $current_major = $3;
7232         $current_minor = $4;
7233
7234     } elsif ($kvmver =~ m/^(\d+)\.(\d+)/) {
7235
7236         $current_major = $1;
7237         $current_minor = $2;
7238     }
7239
7240     return 1 if $current_major > $version_major ||
7241                 ($current_major == $version_major &&
7242                  $current_minor >= $version_minor);
7243 }
7244
7245 sub qemu_machine_pxe {
7246     my ($vmid, $conf) = @_;
7247
7248     my $machine =  PVE::QemuServer::get_current_qemu_machine($vmid);
7249
7250     if ($conf->{machine} && $conf->{machine} =~ m/\.pxe$/) {
7251         $machine .= '.pxe';
7252     }
7253
7254     return $machine;
7255 }
7256
7257 sub qemu_use_old_bios_files {
7258     my ($machine_type) = @_;
7259
7260     return if !$machine_type;
7261
7262     my $use_old_bios_files = undef;
7263
7264     if ($machine_type =~ m/^(\S+)\.pxe$/) {
7265         $machine_type = $1;
7266         $use_old_bios_files = 1;
7267     } else {
7268         my $kvmver = kvm_user_version();
7269         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
7270         # load new efi bios files on migration. So this hack is required to allow
7271         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
7272         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
7273         $use_old_bios_files = !qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 4);
7274     }
7275
7276     return ($use_old_bios_files, $machine_type);
7277 }
7278
7279 sub create_efidisk($$$$$) {
7280     my ($storecfg, $storeid, $vmid, $fmt, $arch) = @_;
7281
7282     my (undef, $ovmf_vars) = get_ovmf_files($arch);
7283     die "EFI vars default image not found\n" if ! -f $ovmf_vars;
7284
7285     my $vars_size_b = -s $ovmf_vars;
7286     my $vars_size = PVE::Tools::convert_size($vars_size_b, 'b' => 'kb');
7287     my $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $fmt, undef, $vars_size);
7288     PVE::Storage::activate_volumes($storecfg, [$volid]);
7289
7290     qemu_img_convert($ovmf_vars, $volid, $vars_size_b, undef, 0);
7291
7292     return ($volid, $vars_size);
7293 }
7294
7295 sub vm_iothreads_list {
7296     my ($vmid) = @_;
7297
7298     my $res = vm_mon_cmd($vmid, 'query-iothreads');
7299
7300     my $iothreads = {};
7301     foreach my $iothread (@$res) {
7302         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
7303     }
7304
7305     return $iothreads;
7306 }
7307
7308 sub scsihw_infos {
7309     my ($conf, $drive) = @_;
7310
7311     my $maxdev = 0;
7312
7313     if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)) {
7314         $maxdev = 7;
7315     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
7316         $maxdev = 1;
7317     } else {
7318         $maxdev = 256;
7319     }
7320
7321     my $controller = int($drive->{index} / $maxdev);
7322     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single') ? "virtioscsi" : "scsihw";
7323
7324     return ($maxdev, $controller, $controller_prefix);
7325 }
7326
7327 sub add_hyperv_enlightenments {
7328     my ($cpuFlags, $winversion, $machine_type, $kvmver, $bios, $gpu_passthrough, $hv_vendor_id) = @_;
7329
7330     return if $winversion < 6;
7331     return if $bios && $bios eq 'ovmf' && $winversion < 8;
7332
7333     if ($gpu_passthrough || defined($hv_vendor_id)) {
7334         $hv_vendor_id //= 'proxmox';
7335         push @$cpuFlags , "hv_vendor_id=$hv_vendor_id";
7336     }
7337
7338     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
7339         push @$cpuFlags , 'hv_spinlocks=0x1fff';
7340         push @$cpuFlags , 'hv_vapic';
7341         push @$cpuFlags , 'hv_time';
7342     } else {
7343         push @$cpuFlags , 'hv_spinlocks=0xffff';
7344     }
7345
7346     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 6)) {
7347         push @$cpuFlags , 'hv_reset';
7348         push @$cpuFlags , 'hv_vpindex';
7349         push @$cpuFlags , 'hv_runtime';
7350     }
7351
7352     if ($winversion >= 7) {
7353         push @$cpuFlags , 'hv_relaxed';
7354
7355         if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 12)) {
7356             push @$cpuFlags , 'hv_synic';
7357             push @$cpuFlags , 'hv_stimer';
7358         }
7359
7360         if (qemu_machine_feature_enabled ($machine_type, $kvmver, 3, 1)) {
7361             push @$cpuFlags , 'hv_ipi';
7362         }
7363     }
7364 }
7365
7366 sub windows_version {
7367     my ($ostype) = @_;
7368
7369     return 0 if !$ostype;
7370
7371     my $winversion = 0;
7372
7373     if($ostype eq 'wxp' || $ostype eq 'w2k3' || $ostype eq 'w2k') {
7374         $winversion = 5;
7375     } elsif($ostype eq 'w2k8' || $ostype eq 'wvista') {
7376         $winversion = 6;
7377     } elsif ($ostype =~ m/^win(\d+)$/) {
7378         $winversion = $1;
7379     }
7380
7381     return $winversion;
7382 }
7383
7384 sub resolve_dst_disk_format {
7385         my ($storecfg, $storeid, $src_volname, $format) = @_;
7386         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
7387
7388         if (!$format) {
7389             # if no target format is specified, use the source disk format as hint
7390             if ($src_volname) {
7391                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
7392                 $format = qemu_img_format($scfg, $src_volname);
7393             } else {
7394                 return $defFormat;
7395             }
7396         }
7397
7398         # test if requested format is supported - else use default
7399         my $supported = grep { $_ eq $format } @$validFormats;
7400         $format = $defFormat if !$supported;
7401         return $format;
7402 }
7403
7404 sub resolve_first_disk {
7405     my $conf = shift;
7406     my @disks = PVE::QemuServer::valid_drive_names();
7407     my $firstdisk;
7408     foreach my $ds (reverse @disks) {
7409         next if !$conf->{$ds};
7410         my $disk = PVE::QemuServer::parse_drive($ds, $conf->{$ds});
7411         next if PVE::QemuServer::drive_is_cdrom($disk);
7412         $firstdisk = $ds;
7413     }
7414     return $firstdisk;
7415 }
7416
7417 sub generate_uuid {
7418     my ($uuid, $uuid_str);
7419     UUID::generate($uuid);
7420     UUID::unparse($uuid, $uuid_str);
7421     return $uuid_str;
7422 }
7423
7424 sub generate_smbios1_uuid {
7425     return "uuid=".generate_uuid();
7426 }
7427
7428 sub nbd_stop {
7429     my ($vmid) = @_;
7430
7431     vm_mon_cmd($vmid, 'nbd-server-stop');
7432 }
7433
7434 sub create_reboot_request {
7435     my ($vmid) = @_;
7436     open(my $fh, '>', "/run/qemu-server/$vmid.reboot")
7437         or die "failed to create reboot trigger file: $!\n";
7438     close($fh);
7439 }
7440
7441 sub clear_reboot_request {
7442     my ($vmid) = @_;
7443     my $path = "/run/qemu-server/$vmid.reboot";
7444     my $res = 0;
7445
7446     $res = unlink($path);
7447     die "could not remove reboot request for $vmid: $!"
7448         if !$res && $! != POSIX::ENOENT;
7449
7450     return $res;
7451 }
7452
7453 # bash completion helper
7454
7455 sub complete_backup_archives {
7456     my ($cmdname, $pname, $cvalue) = @_;
7457
7458     my $cfg = PVE::Storage::config();
7459
7460     my $storeid;
7461
7462     if ($cvalue =~ m/^([^:]+):/) {
7463         $storeid = $1;
7464     }
7465
7466     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
7467
7468     my $res = [];
7469     foreach my $id (keys %$data) {
7470         foreach my $item (@{$data->{$id}}) {
7471             next if $item->{format} !~ m/^vma\.(gz|lzo)$/;
7472             push @$res, $item->{volid} if defined($item->{volid});
7473         }
7474     }
7475
7476     return $res;
7477 }
7478
7479 my $complete_vmid_full = sub {
7480     my ($running) = @_;
7481
7482     my $idlist = vmstatus();
7483
7484     my $res = [];
7485
7486     foreach my $id (keys %$idlist) {
7487         my $d = $idlist->{$id};
7488         if (defined($running)) {
7489             next if $d->{template};
7490             next if $running && $d->{status} ne 'running';
7491             next if !$running && $d->{status} eq 'running';
7492         }
7493         push @$res, $id;
7494
7495     }
7496     return $res;
7497 };
7498
7499 sub complete_vmid {
7500     return &$complete_vmid_full();
7501 }
7502
7503 sub complete_vmid_stopped {
7504     return &$complete_vmid_full(0);
7505 }
7506
7507 sub complete_vmid_running {
7508     return &$complete_vmid_full(1);
7509 }
7510
7511 sub complete_storage {
7512
7513     my $cfg = PVE::Storage::config();
7514     my $ids = $cfg->{ids};
7515
7516     my $res = [];
7517     foreach my $sid (keys %$ids) {
7518         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
7519         next if !$ids->{$sid}->{content}->{images};
7520         push @$res, $sid;
7521     }
7522
7523     return $res;
7524 }
7525
7526 1;