PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5
   6 use Cwd 'abs_path';
   7 use Digest::SHA;
   8 use Fcntl ':flock';
   9 use Fcntl;
  10 use File::Basename;
  11 use File::Copy qw(copy);
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use IO::Dir;
  16 use IO::File;
  17 use IO::Handle;
  18 use IO::Select;
  19 use IO::Socket::UNIX;
  20 use IPC::Open3;
  21 use JSON;
  22 use MIME::Base64;
  23 use POSIX;
  24 use Storable qw(dclone);
  25 use Time::HiRes qw(gettimeofday);
  26 use URI::Escape;
  27
  28 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file cfs_lock_file);
  29 use PVE::Exception qw(raise raise_param_exc);
  30 use PVE::GuestHelpers;
  31 use PVE::INotify;
  32 use PVE::JSONSchema qw(get_standard_option);
  33 use PVE::ProcFSTools;
  34 use PVE::RPCEnvironment;
  35 use PVE::SafeSyslog;
  36 use PVE::Storage;
  37 use PVE::SysFSTools;
  38 use PVE::Systemd;
  39 use PVE::Tools qw(run_command lock_file lock_file_full file_read_firstline dir_glob_foreach $IPV6RE);
  40
  41 use PVE::QMPClient;
  42 use PVE::QemuConfig;
  43 use PVE::QemuServer::Cloudinit;
  44 use PVE::QemuServer::Memory;
  45 use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr print_pcie_root_port);
  46 use PVE::QemuServer::USB qw(parse_usb_device);
  47
  48 my $EDK2_FW_BASE = '/usr/share/pve-edk2-firmware/';
  49 my $OVMF = {
  50     x86_64 => [
  51         "$EDK2_FW_BASE/OVMF_CODE.fd",
  52         "$EDK2_FW_BASE/OVMF_VARS.fd"
  53     ],
  54     aarch64 => [
  55         "$EDK2_FW_BASE/AAVMF_CODE.fd",
  56         "$EDK2_FW_BASE/AAVMF_VARS.fd"
  57     ],
  58 };
  59
  60 my $qemu_snap_storage = { rbd => 1 };
  61
  62 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  63
  64 my $QEMU_FORMAT_RE = qr/raw|cow|qcow|qcow2|qed|vmdk|cloop/;
  65
  66 # Note about locking: we use flock on the config file protect
  67 # against concurent actions.
  68 # Aditionaly, we have a 'lock' setting in the config file. This
  69 # can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
  70 # allowed when such lock is set. But you can ignore this kind of
  71 # lock with the --skiplock flag.
  72
  73 cfs_register_file('/qemu-server/',
  74                   \&parse_vm_config,
  75                   \&write_vm_config);
  76
  77 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
  78     description => "Some command save/restore state from this location.",
  79     type => 'string',
  80     maxLength => 128,
  81     optional => 1,
  82 });
  83
  84 PVE::JSONSchema::register_standard_option('pve-qm-image-format', {
  85     type => 'string',
  86     enum => [qw(raw cow qcow qed qcow2 vmdk cloop)],
  87     description => "The drive's backing file's data format.",
  88     optional => 1,
  89 });
  90
  91 PVE::JSONSchema::register_standard_option('pve-qemu-machine', {
  92         description => "Specifies the Qemu machine type.",
  93         type => 'string',
  94         pattern => '(pc|pc(-i440fx)?-\d+\.\d+(\.pxe)?|q35|pc-q35-\d+\.\d+(\.pxe)?|virt(?:-\d+\.\d+)?)',
  95         maxLength => 40,
  96         optional => 1,
  97 });
  98
  99 #no warnings 'redefine';
 100
 101 sub cgroups_write {
 102    my ($controller, $vmid, $option, $value) = @_;
 103
 104    my $path = "/sys/fs/cgroup/$controller/qemu.slice/$vmid.scope/$option";
 105    PVE::ProcFSTools::write_proc_entry($path, $value);
 106
 107 }
 108
 109 my $nodename = PVE::INotify::nodename();
 110
 111 mkdir "/etc/pve/nodes/$nodename";
 112 my $confdir = "/etc/pve/nodes/$nodename/qemu-server";
 113 mkdir $confdir;
 114
 115 my $var_run_tmpdir = "/var/run/qemu-server";
 116 mkdir $var_run_tmpdir;
 117
 118 my $lock_dir = "/var/lock/qemu-server";
 119 mkdir $lock_dir;
 120
 121 my $cpu_vendor_list = {
 122     # Intel CPUs
 123     486 => 'GenuineIntel',
 124     pentium => 'GenuineIntel',
 125     pentium2  => 'GenuineIntel',
 126     pentium3  => 'GenuineIntel',
 127     coreduo => 'GenuineIntel',
 128     core2duo => 'GenuineIntel',
 129     Conroe  => 'GenuineIntel',
 130     Penryn  => 'GenuineIntel',
 131     Nehalem  => 'GenuineIntel',
 132     'Nehalem-IBRS'  => 'GenuineIntel',
 133     Westmere => 'GenuineIntel',
 134     'Westmere-IBRS' => 'GenuineIntel',
 135     SandyBridge => 'GenuineIntel',
 136     'SandyBridge-IBRS' => 'GenuineIntel',
 137     IvyBridge => 'GenuineIntel',
 138     'IvyBridge-IBRS' => 'GenuineIntel',
 139     Haswell => 'GenuineIntel',
 140     'Haswell-IBRS' => 'GenuineIntel',
 141     'Haswell-noTSX' => 'GenuineIntel',
 142     'Haswell-noTSX-IBRS' => 'GenuineIntel',
 143     Broadwell => 'GenuineIntel',
 144     'Broadwell-IBRS' => 'GenuineIntel',
 145     'Broadwell-noTSX' => 'GenuineIntel',
 146     'Broadwell-noTSX-IBRS' => 'GenuineIntel',
 147     'Skylake-Client' => 'GenuineIntel',
 148     'Skylake-Client-IBRS' => 'GenuineIntel',
 149     'Skylake-Server' => 'GenuineIntel',
 150     'Skylake-Server-IBRS' => 'GenuineIntel',
 151     'Cascadelake-Server' => 'GenuineIntel',
 152     KnightsMill => 'GenuineIntel',
 153
 154
 155     # AMD CPUs
 156     athlon => 'AuthenticAMD',
 157     phenom  => 'AuthenticAMD',
 158     Opteron_G1  => 'AuthenticAMD',
 159     Opteron_G2  => 'AuthenticAMD',
 160     Opteron_G3  => 'AuthenticAMD',
 161     Opteron_G4  => 'AuthenticAMD',
 162     Opteron_G5  => 'AuthenticAMD',
 163     EPYC => 'AuthenticAMD',
 164     'EPYC-IBPB' => 'AuthenticAMD',
 165
 166     # generic types, use vendor from host node
 167     host => 'default',
 168     kvm32 => 'default',
 169     kvm64 => 'default',
 170     qemu32 => 'default',
 171     qemu64 => 'default',
 172     max => 'default',
 173 };
 174
 175 my @supported_cpu_flags = (
 176     'pcid',
 177     'spec-ctrl',
 178     'ibpb',
 179     'ssbd',
 180     'virt-ssbd',
 181     'amd-ssbd',
 182     'amd-no-ssb',
 183     'pdpe1gb',
 184     'md-clear',
 185     'hv-tlbflush',
 186     'hv-evmcs',
 187     'aes'
 188 );
 189 my $cpu_flag = qr/[+-](@{[join('|', @supported_cpu_flags)]})/;
 190
 191 my $cpu_fmt = {
 192     cputype => {
 193         description => "Emulated CPU type.",
 194         type => 'string',
 195         enum => [ sort { "\L$a" cmp "\L$b" } keys %$cpu_vendor_list ],
 196         default => 'kvm64',
 197         default_key => 1,
 198     },
 199     hidden => {
 200         description => "Do not identify as a KVM virtual machine.",
 201         type => 'boolean',
 202         optional => 1,
 203         default => 0
 204     },
 205     'hv-vendor-id' => {
 206         type => 'string',
 207         pattern => qr/[a-zA-Z0-9]{1,12}/,
 208         format_description => 'vendor-id',
 209         description => 'The Hyper-V vendor ID. Some drivers or programs inside Windows guests need a specific ID.',
 210         optional => 1,
 211     },
 212     flags => {
 213         description => "List of additional CPU flags separated by ';'."
 214                      . " Use '+FLAG' to enable, '-FLAG' to disable a flag."
 215                      . " Currently supported flags: @{[join(', ', @supported_cpu_flags)]}.",
 216         format_description => '+FLAG[;-FLAG...]',
 217         type => 'string',
 218         pattern => qr/$cpu_flag(;$cpu_flag)*/,
 219         optional => 1,
 220     },
 221 };
 222
 223 my $watchdog_fmt = {
 224     model => {
 225         default_key => 1,
 226         type => 'string',
 227         enum => [qw(i6300esb ib700)],
 228         description => "Watchdog type to emulate.",
 229         default => 'i6300esb',
 230         optional => 1,
 231     },
 232     action => {
 233         type => 'string',
 234         enum => [qw(reset shutdown poweroff pause debug none)],
 235         description => "The action to perform if after activation the guest fails to poll the watchdog in time.",
 236         optional => 1,
 237     },
 238 };
 239 PVE::JSONSchema::register_format('pve-qm-watchdog', $watchdog_fmt);
 240
 241 my $agent_fmt = {
 242     enabled => {
 243         description => "Enable/disable Qemu GuestAgent.",
 244         type => 'boolean',
 245         default => 0,
 246         default_key => 1,
 247     },
 248     fstrim_cloned_disks => {
 249         description => "Run fstrim after cloning/moving a disk.",
 250         type => 'boolean',
 251         optional => 1,
 252         default => 0
 253     },
 254 };
 255
 256 my $vga_fmt = {
 257     type => {
 258         description => "Select the VGA type.",
 259         type => 'string',
 260         default => 'std',
 261         optional => 1,
 262         default_key => 1,
 263         enum => [qw(cirrus qxl qxl2 qxl3 qxl4 none serial0 serial1 serial2 serial3 std virtio vmware)],
 264     },
 265     memory => {
 266         description => "Sets the VGA memory (in MiB). Has no effect with serial display.",
 267         type => 'integer',
 268         optional => 1,
 269         minimum => 4,
 270         maximum => 512,
 271     },
 272 };
 273
 274 my $ivshmem_fmt = {
 275     size => {
 276         type => 'integer',
 277         minimum => 1,
 278         description => "The size of the file in MB.",
 279     },
 280     name => {
 281         type => 'string',
 282         pattern => '[a-zA-Z0-9\-]+',
 283         optional => 1,
 284         format_description => 'string',
 285         description => "The name of the file. Will be prefixed with 'pve-shm-'. Default is the VMID. Will be deleted when the VM is stopped.",
 286     },
 287 };
 288
 289 my $audio_fmt = {
 290     device => {
 291         type => 'string',
 292         enum => [qw(ich9-intel-hda intel-hda AC97)],
 293         description =>  "Configure an audio device."
 294     },
 295     driver =>  {
 296         type => 'string',
 297         enum => ['spice'],
 298         default => 'spice',
 299         optional => 1,
 300         description => "Driver backend for the audio device."
 301     },
 302 };
 303
 304 my $spice_enhancements_fmt = {
 305     foldersharing => {
 306         type => 'boolean',
 307         optional => 1,
 308         default => '0',
 309         description =>  "Enable folder sharing via SPICE. Needs Spice-WebDAV daemon installed in the VM."
 310     },
 311     videostreaming =>  {
 312         type => 'string',
 313         enum => ['off', 'all', 'filter'],
 314         default => 'off',
 315         optional => 1,
 316         description => "Enable video streaming. Uses compression for detected video streams."
 317     },
 318 };
 319
 320 my $confdesc = {
 321     onboot => {
 322         optional => 1,
 323         type => 'boolean',
 324         description => "Specifies whether a VM will be started during system bootup.",
 325         default => 0,
 326     },
 327     autostart => {
 328         optional => 1,
 329         type => 'boolean',
 330         description => "Automatic restart after crash (currently ignored).",
 331         default => 0,
 332     },
 333     hotplug => {
 334         optional => 1,
 335         type => 'string', format => 'pve-hotplug-features',
 336         description => "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
 337         default => 'network,disk,usb',
 338     },
 339     reboot => {
 340         optional => 1,
 341         type => 'boolean',
 342         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 343         default => 1,
 344     },
 345     lock => {
 346         optional => 1,
 347         type => 'string',
 348         description => "Lock/unlock the VM.",
 349         enum => [qw(backup clone create migrate rollback snapshot snapshot-delete suspending suspended)],
 350     },
 351     cpulimit => {
 352         optional => 1,
 353         type => 'number',
 354         description => "Limit of CPU usage.",
 355         verbose_description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
 356         minimum => 0,
 357         maximum => 128,
 358         default => 0,
 359     },
 360     cpuunits => {
 361         optional => 1,
 362         type => 'integer',
 363         description => "CPU weight for a VM.",
 364         verbose_description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.",
 365         minimum => 2,
 366         maximum => 262144,
 367         default => 1024,
 368     },
 369     memory => {
 370         optional => 1,
 371         type => 'integer',
 372         description => "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
 373         minimum => 16,
 374         default => 512,
 375     },
 376     balloon => {
 377         optional => 1,
 378         type => 'integer',
 379         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 380         minimum => 0,
 381     },
 382     shares => {
 383         optional => 1,
 384         type => 'integer',
 385         description => "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning. Auto-ballooning is done by pvestatd.",
 386         minimum => 0,
 387         maximum => 50000,
 388         default => 1000,
 389     },
 390     keyboard => {
 391         optional => 1,
 392         type => 'string',
 393         description => "Keybord layout for vnc server. Default is read from the '/etc/pve/datacenter.cfg' configuration file.".
 394                        "It should not be necessary to set it.",
 395         enum => PVE::Tools::kvmkeymaplist(),
 396         default => undef,
 397     },
 398     name => {
 399         optional => 1,
 400         type => 'string', format => 'dns-name',
 401         description => "Set a name for the VM. Only used on the configuration web interface.",
 402     },
 403     scsihw => {
 404         optional => 1,
 405         type => 'string',
 406         description => "SCSI controller model",
 407         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 408         default => 'lsi',
 409     },
 410     description => {
 411         optional => 1,
 412         type => 'string',
 413         description => "Description for the VM. Only used on the configuration web interface. This is saved as comment inside the configuration file.",
 414     },
 415     ostype => {
 416         optional => 1,
 417         type => 'string',
 418         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 l24 l26 solaris)],
 419         description => "Specify guest operating system.",
 420         verbose_description => <<EODESC,
 421 Specify guest operating system. This is used to enable special
 422 optimization/features for specific operating systems:
 423
 424 [horizontal]
 425 other;; unspecified OS
 426 wxp;; Microsoft Windows XP
 427 w2k;; Microsoft Windows 2000
 428 w2k3;; Microsoft Windows 2003
 429 w2k8;; Microsoft Windows 2008
 430 wvista;; Microsoft Windows Vista
 431 win7;; Microsoft Windows 7
 432 win8;; Microsoft Windows 8/2012/2012r2
 433 win10;; Microsoft Windows 10/2016
 434 l24;; Linux 2.4 Kernel
 435 l26;; Linux 2.6/3.X Kernel
 436 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
 437 EODESC
 438     },
 439     boot => {
 440         optional => 1,
 441         type => 'string',
 442         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n).",
 443         pattern => '[acdn]{1,4}',
 444         default => 'cdn',
 445     },
 446     bootdisk => {
 447         optional => 1,
 448         type => 'string', format => 'pve-qm-bootdisk',
 449         description => "Enable booting from specified disk.",
 450         pattern => '(ide|sata|scsi|virtio)\d+',
 451     },
 452     smp => {
 453         optional => 1,
 454         type => 'integer',
 455         description => "The number of CPUs. Please use option -sockets instead.",
 456         minimum => 1,
 457         default => 1,
 458     },
 459     sockets => {
 460         optional => 1,
 461         type => 'integer',
 462         description => "The number of CPU sockets.",
 463         minimum => 1,
 464         default => 1,
 465     },
 466     cores => {
 467         optional => 1,
 468         type => 'integer',
 469         description => "The number of cores per socket.",
 470         minimum => 1,
 471         default => 1,
 472     },
 473     numa => {
 474         optional => 1,
 475         type => 'boolean',
 476         description => "Enable/disable NUMA.",
 477         default => 0,
 478     },
 479     hugepages => {
 480         optional => 1,
 481         type => 'string',
 482         description => "Enable/disable hugepages memory.",
 483         enum => [qw(any 2 1024)],
 484     },
 485     vcpus => {
 486         optional => 1,
 487         type => 'integer',
 488         description => "Number of hotplugged vcpus.",
 489         minimum => 1,
 490         default => 0,
 491     },
 492     acpi => {
 493         optional => 1,
 494         type => 'boolean',
 495         description => "Enable/disable ACPI.",
 496         default => 1,
 497     },
 498     agent => {
 499         optional => 1,
 500         description => "Enable/disable Qemu GuestAgent and its properties.",
 501         type => 'string',
 502         format => $agent_fmt,
 503     },
 504     kvm => {
 505         optional => 1,
 506         type => 'boolean',
 507         description => "Enable/disable KVM hardware virtualization.",
 508         default => 1,
 509     },
 510     tdf => {
 511         optional => 1,
 512         type => 'boolean',
 513         description => "Enable/disable time drift fix.",
 514         default => 0,
 515     },
 516     localtime => {
 517         optional => 1,
 518         type => 'boolean',
 519         description => "Set the real time clock to local time. This is enabled by default if ostype indicates a Microsoft OS.",
 520     },
 521     freeze => {
 522         optional => 1,
 523         type => 'boolean',
 524         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 525     },
 526     vga => {
 527         optional => 1,
 528         type => 'string', format => $vga_fmt,
 529         description => "Configure the VGA hardware.",
 530         verbose_description => "Configure the VGA Hardware. If you want to use ".
 531             "high resolution modes (>= 1280x1024x16) you may need to increase " .
 532             "the vga memory option. Since QEMU 2.9 the default VGA display type " .
 533             "is 'std' for all OS types besides some Windows versions (XP and " .
 534             "older) which use 'cirrus'. The 'qxl' option enables the SPICE " .
 535             "display server. For win* OS you can select how many independent " .
 536             "displays you want, Linux guests can add displays them self.\n".
 537             "You can also run without any graphic card, using a serial device as terminal.",
 538     },
 539     watchdog => {
 540         optional => 1,
 541         type => 'string', format => 'pve-qm-watchdog',
 542         description => "Create a virtual hardware watchdog device.",
 543         verbose_description => "Create a virtual hardware watchdog device. Once enabled" .
 544             " (by a guest action), the watchdog must be periodically polled " .
 545             "by an agent inside the guest or else the watchdog will reset " .
 546             "the guest (or execute the respective action specified)",
 547     },
 548     startdate => {
 549         optional => 1,
 550         type => 'string',
 551         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 552         description => "Set the initial date of the real time clock. Valid format for date are: 'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 553         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 554         default => 'now',
 555     },
 556     startup =>  get_standard_option('pve-startup-order'),
 557     template => {
 558         optional => 1,
 559         type => 'boolean',
 560         description => "Enable/disable Template.",
 561         default => 0,
 562     },
 563     args => {
 564         optional => 1,
 565         type => 'string',
 566         description => "Arbitrary arguments passed to kvm.",
 567         verbose_description => <<EODESCR,
 568 Arbitrary arguments passed to kvm, for example:
 569
 570 args: -no-reboot -no-hpet
 571
 572 NOTE: this option is for experts only.
 573 EODESCR
 574     },
 575     tablet => {
 576         optional => 1,
 577         type => 'boolean',
 578         default => 1,
 579         description => "Enable/disable the USB tablet device.",
 580         verbose_description => "Enable/disable the USB tablet device. This device is " .
 581             "usually needed to allow absolute mouse positioning with VNC. " .
 582             "Else the mouse runs out of sync with normal VNC clients. " .
 583             "If you're running lots of console-only guests on one host, " .
 584             "you may consider disabling this to save some context switches. " .
 585             "This is turned off by default if you use spice (-vga=qxl).",
 586     },
 587     migrate_speed => {
 588         optional => 1,
 589         type => 'integer',
 590         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 591         minimum => 0,
 592         default => 0,
 593     },
 594     migrate_downtime => {
 595         optional => 1,
 596         type => 'number',
 597         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 598         minimum => 0,
 599         default => 0.1,
 600     },
 601     cdrom => {
 602         optional => 1,
 603         type => 'string', format => 'pve-qm-ide',
 604         typetext => '<volume>',
 605         description => "This is an alias for option -ide2",
 606     },
 607     cpu => {
 608         optional => 1,
 609         description => "Emulated CPU type.",
 610         type => 'string',
 611         format => $cpu_fmt,
 612     },
 613     parent => get_standard_option('pve-snapshot-name', {
 614         optional => 1,
 615         description => "Parent snapshot name. This is used internally, and should not be modified.",
 616     }),
 617     snaptime => {
 618         optional => 1,
 619         description => "Timestamp for snapshots.",
 620         type => 'integer',
 621         minimum => 0,
 622     },
 623     vmstate => {
 624         optional => 1,
 625         type => 'string', format => 'pve-volume-id',
 626         description => "Reference to a volume which stores the VM state. This is used internally for snapshots.",
 627     },
 628     vmstatestorage => get_standard_option('pve-storage-id', {
 629         description => "Default storage for VM state volumes/files.",
 630         optional => 1,
 631     }),
 632     runningmachine => get_standard_option('pve-qemu-machine', {
 633         description => "Specifies the Qemu machine type of the running vm. This is used internally for snapshots.",
 634     }),
 635     machine => get_standard_option('pve-qemu-machine'),
 636     arch => {
 637         description => "Virtual processor architecture. Defaults to the host.",
 638         optional => 1,
 639         type => 'string',
 640         enum => [qw(x86_64 aarch64)],
 641     },
 642     smbios1 => {
 643         description => "Specify SMBIOS type 1 fields.",
 644         type => 'string', format => 'pve-qm-smbios1',
 645         maxLength => 512,
 646         optional => 1,
 647     },
 648     protection => {
 649         optional => 1,
 650         type => 'boolean',
 651         description => "Sets the protection flag of the VM. This will disable the remove VM and remove disk operations.",
 652         default => 0,
 653     },
 654     bios => {
 655         optional => 1,
 656         type => 'string',
 657         enum => [ qw(seabios ovmf) ],
 658         description => "Select BIOS implementation.",
 659         default => 'seabios',
 660     },
 661     vmgenid => {
 662         type => 'string',
 663         pattern => '(?:[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}|[01])',
 664         format_description => 'UUID',
 665         description => "Set VM Generation ID. Use '1' to autogenerate on create or update, pass '0' to disable explicitly.",
 666         verbose_description => "The VM generation ID (vmgenid) device exposes a".
 667             " 128-bit integer value identifier to the guest OS. This allows to".
 668             " notify the guest operating system when the virtual machine is".
 669             " executed with a different configuration (e.g. snapshot execution".
 670             " or creation from a template). The guest operating system notices".
 671             " the change, and is then able to react as appropriate by marking".
 672             " its copies of distributed databases as dirty, re-initializing its".
 673             " random number generator, etc.\n".
 674             "Note that auto-creation only works when done throug API/CLI create".
 675             " or update methods, but not when manually editing the config file.",
 676         default => "1 (autogenerated)",
 677         optional => 1,
 678     },
 679     hookscript => {
 680         type => 'string',
 681         format => 'pve-volume-id',
 682         optional => 1,
 683         description => "Script that will be executed during various steps in the vms lifetime.",
 684     },
 685     ivshmem => {
 686         type => 'string',
 687         format => $ivshmem_fmt,
 688         description => "Inter-VM shared memory. Useful for direct communication between VMs, or to the host.",
 689         optional => 1,
 690     },
 691     audio0 => {
 692         type => 'string',
 693         format => $audio_fmt,
 694         description => "Configure a audio device, useful in combination with QXL/Spice.",
 695         optional => 1
 696     },
 697     spice_enhancements => {
 698         type => 'string',
 699         format => $spice_enhancements_fmt,
 700         description => "Configure additional enhancements for SPICE.",
 701         optional => 1
 702     },
 703 };
 704
 705 my $cicustom_fmt = {
 706     meta => {
 707         type => 'string',
 708         optional => 1,
 709         description => 'Specify a custom file containing all meta data passed to the VM via cloud-init. This is provider specific meaning configdrive2 and nocloud differ.',
 710         format => 'pve-volume-id',
 711         format_description => 'volume',
 712     },
 713     network => {
 714         type => 'string',
 715         optional => 1,
 716         description => 'Specify a custom file containing all network data passed to the VM via cloud-init.',
 717         format => 'pve-volume-id',
 718         format_description => 'volume',
 719     },
 720     user => {
 721         type => 'string',
 722         optional => 1,
 723         description => 'Specify a custom file containing all user data passed to the VM via cloud-init.',
 724         format => 'pve-volume-id',
 725         format_description => 'volume',
 726     },
 727 };
 728 PVE::JSONSchema::register_format('pve-qm-cicustom', $cicustom_fmt);
 729
 730 my $confdesc_cloudinit = {
 731     citype => {
 732         optional => 1,
 733         type => 'string',
 734         description => 'Specifies the cloud-init configuration format. The default depends on the configured operating system type (`ostype`. We use the `nocloud` format for Linux, and `configdrive2` for windows.',
 735         enum => ['configdrive2', 'nocloud'],
 736     },
 737     ciuser => {
 738         optional => 1,
 739         type => 'string',
 740         description => "cloud-init: User name to change ssh keys and password for instead of the image's configured default user.",
 741     },
 742     cipassword => {
 743         optional => 1,
 744         type => 'string',
 745         description => 'cloud-init: Password to assign the user. Using this is generally not recommended. Use ssh keys instead. Also note that older cloud-init versions do not support hashed passwords.',
 746     },
 747     cicustom => {
 748         optional => 1,
 749         type => 'string',
 750         description => 'cloud-init: Specify custom files to replace the automatically generated ones at start.',
 751         format => 'pve-qm-cicustom',
 752     },
 753     searchdomain => {
 754         optional => 1,
 755         type => 'string',
 756         description => "cloud-init: Sets DNS search domains for a container. Create will automatically use the setting from the host if neither searchdomain nor nameserver are set.",
 757     },
 758     nameserver => {
 759         optional => 1,
 760         type => 'string', format => 'address-list',
 761         description => "cloud-init: Sets DNS server IP address for a container. Create will automatically use the setting from the host if neither searchdomain nor nameserver are set.",
 762     },
 763     sshkeys => {
 764         optional => 1,
 765         type => 'string',
 766         format => 'urlencoded',
 767         description => "cloud-init: Setup public SSH keys (one key per line, OpenSSH format).",
 768     },
 769 };
 770
 771 # what about other qemu settings ?
 772 #cpu => 'string',
 773 #machine => 'string',
 774 #fda => 'file',
 775 #fdb => 'file',
 776 #mtdblock => 'file',
 777 #sd => 'file',
 778 #pflash => 'file',
 779 #snapshot => 'bool',
 780 #bootp => 'file',
 781 ##tftp => 'dir',
 782 ##smb => 'dir',
 783 #kernel => 'file',
 784 #append => 'string',
 785 #initrd => 'file',
 786 ##soundhw => 'string',
 787
 788 while (my ($k, $v) = each %$confdesc) {
 789     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 790 }
 791
 792 my $MAX_IDE_DISKS = 4;
 793 my $MAX_SCSI_DISKS = 14;
 794 my $MAX_VIRTIO_DISKS = 16;
 795 my $MAX_SATA_DISKS = 6;
 796 my $MAX_USB_DEVICES = 5;
 797 my $MAX_NETS = 32;
 798 my $MAX_UNUSED_DISKS = 256;
 799 my $MAX_HOSTPCI_DEVICES = 16;
 800 my $MAX_SERIAL_PORTS = 4;
 801 my $MAX_PARALLEL_PORTS = 3;
 802 my $MAX_NUMA = 8;
 803
 804 my $numa_fmt = {
 805     cpus => {
 806         type => "string",
 807         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 808         description => "CPUs accessing this NUMA node.",
 809         format_description => "id[-id];...",
 810     },
 811     memory => {
 812         type => "number",
 813         description => "Amount of memory this NUMA node provides.",
 814         optional => 1,
 815     },
 816     hostnodes => {
 817         type => "string",
 818         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 819         description => "Host NUMA nodes to use.",
 820         format_description => "id[-id];...",
 821         optional => 1,
 822     },
 823     policy => {
 824         type => 'string',
 825         enum => [qw(preferred bind interleave)],
 826         description => "NUMA allocation policy.",
 827         optional => 1,
 828     },
 829 };
 830 PVE::JSONSchema::register_format('pve-qm-numanode', $numa_fmt);
 831 my $numadesc = {
 832     optional => 1,
 833     type => 'string', format => $numa_fmt,
 834     description => "NUMA topology.",
 835 };
 836 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 837
 838 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 839     $confdesc->{"numa$i"} = $numadesc;
 840 }
 841
 842 my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000',  'pcnet',  'virtio',
 843                       'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3',
 844                       'e1000-82540em', 'e1000-82544gc', 'e1000-82545em'];
 845 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 846
 847 my $net_fmt_bridge_descr = <<__EOD__;
 848 Bridge to attach the network device to. The Proxmox VE standard bridge
 849 is called 'vmbr0'.
 850
 851 If you do not specify a bridge, we create a kvm user (NATed) network
 852 device, which provides DHCP and DNS services. The following addresses
 853 are used:
 854
 855  10.0.2.2   Gateway
 856  10.0.2.3   DNS Server
 857  10.0.2.4   SMB Server
 858
 859 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 860 __EOD__
 861
 862 my $net_fmt = {
 863     macaddr  => get_standard_option('mac-addr', {
 864         description => "MAC address. That address must be unique withing your network. This is automatically generated if not specified.",
 865     }),
 866     model => {
 867         type => 'string',
 868         description => "Network Card Model. The 'virtio' model provides the best performance with very low CPU overhead. If your guest does not support this driver, it is usually best to use 'e1000'.",
 869         enum => $nic_model_list,
 870         default_key => 1,
 871     },
 872     (map { $_ => { keyAlias => 'model', alias => 'macaddr' }} @$nic_model_list),
 873     bridge => {
 874         type => 'string',
 875         description => $net_fmt_bridge_descr,
 876         format_description => 'bridge',
 877         optional => 1,
 878     },
 879     queues => {
 880         type => 'integer',
 881         minimum => 0, maximum => 16,
 882         description => 'Number of packet queues to be used on the device.',
 883         optional => 1,
 884     },
 885     rate => {
 886         type => 'number',
 887         minimum => 0,
 888         description => "Rate limit in mbps (megabytes per second) as floating point number.",
 889         optional => 1,
 890     },
 891     tag => {
 892         type => 'integer',
 893         minimum => 1, maximum => 4094,
 894         description => 'VLAN tag to apply to packets on this interface.',
 895         optional => 1,
 896     },
 897     trunks => {
 898         type => 'string',
 899         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 900         description => 'VLAN trunks to pass through this interface.',
 901         format_description => 'vlanid[;vlanid...]',
 902         optional => 1,
 903     },
 904     firewall => {
 905         type => 'boolean',
 906         description => 'Whether this interface should be protected by the firewall.',
 907         optional => 1,
 908     },
 909     link_down => {
 910         type => 'boolean',
 911         description => 'Whether this interface should be disconnected (like pulling the plug).',
 912         optional => 1,
 913     },
 914 };
 915
 916 my $netdesc = {
 917     optional => 1,
 918     type => 'string', format => $net_fmt,
 919     description => "Specify network devices.",
 920 };
 921
 922 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 923
 924 my $ipconfig_fmt = {
 925     ip => {
 926         type => 'string',
 927         format => 'pve-ipv4-config',
 928         format_description => 'IPv4Format/CIDR',
 929         description => 'IPv4 address in CIDR format.',
 930         optional => 1,
 931         default => 'dhcp',
 932     },
 933     gw => {
 934         type => 'string',
 935         format => 'ipv4',
 936         format_description => 'GatewayIPv4',
 937         description => 'Default gateway for IPv4 traffic.',
 938         optional => 1,
 939         requires => 'ip',
 940     },
 941     ip6 => {
 942         type => 'string',
 943         format => 'pve-ipv6-config',
 944         format_description => 'IPv6Format/CIDR',
 945         description => 'IPv6 address in CIDR format.',
 946         optional => 1,
 947         default => 'dhcp',
 948     },
 949     gw6 => {
 950         type => 'string',
 951         format => 'ipv6',
 952         format_description => 'GatewayIPv6',
 953         description => 'Default gateway for IPv6 traffic.',
 954         optional => 1,
 955         requires => 'ip6',
 956     },
 957 };
 958 PVE::JSONSchema::register_format('pve-qm-ipconfig', $ipconfig_fmt);
 959 my $ipconfigdesc = {
 960     optional => 1,
 961     type => 'string', format => 'pve-qm-ipconfig',
 962     description => <<'EODESCR',
 963 cloud-init: Specify IP addresses and gateways for the corresponding interface.
 964
 965 IP addresses use CIDR notation, gateways are optional but need an IP of the same type specified.
 966
 967 The special string 'dhcp' can be used for IP addresses to use DHCP, in which case no explicit gateway should be provided.
 968 For IPv6 the special string 'auto' can be used to use stateless autoconfiguration.
 969
 970 If cloud-init is enabled and neither an IPv4 nor an IPv6 address is specified, it defaults to using dhcp on IPv4.
 971 EODESCR
 972 };
 973 PVE::JSONSchema::register_standard_option("pve-qm-ipconfig", $netdesc);
 974
 975 for (my $i = 0; $i < $MAX_NETS; $i++)  {
 976     $confdesc->{"net$i"} = $netdesc;
 977     $confdesc_cloudinit->{"ipconfig$i"} = $ipconfigdesc;
 978 }
 979
 980 foreach my $key (keys %$confdesc_cloudinit) {
 981     $confdesc->{$key} = $confdesc_cloudinit->{$key};
 982 }
 983
 984 PVE::JSONSchema::register_format('pve-volume-id-or-qm-path', \&verify_volume_id_or_qm_path);
 985 sub verify_volume_id_or_qm_path {
 986     my ($volid, $noerr) = @_;
 987
 988     if ($volid eq 'none' || $volid eq 'cdrom' || $volid =~ m|^/|) {
 989         return $volid;
 990     }
 991
 992     # if its neither 'none' nor 'cdrom' nor a path, check if its a volume-id
 993     $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') };
 994     if ($@) {
 995         return undef if $noerr;
 996         die $@;
 997     }
 998     return $volid;
 999 }
1000
1001 my $drivename_hash;
1002
1003 my %drivedesc_base = (
1004     volume => { alias => 'file' },
1005     file => {
1006         type => 'string',
1007         format => 'pve-volume-id-or-qm-path',
1008         default_key => 1,
1009         format_description => 'volume',
1010         description => "The drive's backing volume.",
1011     },
1012     media => {
1013         type => 'string',
1014         enum => [qw(cdrom disk)],
1015         description => "The drive's media type.",
1016         default => 'disk',
1017         optional => 1
1018     },
1019     cyls => {
1020         type => 'integer',
1021         description => "Force the drive's physical geometry to have a specific cylinder count.",
1022         optional => 1
1023     },
1024     heads => {
1025         type => 'integer',
1026         description => "Force the drive's physical geometry to have a specific head count.",
1027         optional => 1
1028     },
1029     secs => {
1030         type => 'integer',
1031         description => "Force the drive's physical geometry to have a specific sector count.",
1032         optional => 1
1033     },
1034     trans => {
1035         type => 'string',
1036         enum => [qw(none lba auto)],
1037         description => "Force disk geometry bios translation mode.",
1038         optional => 1,
1039     },
1040     snapshot => {
1041         type => 'boolean',
1042         description => "Controls qemu's snapshot mode feature."
1043             . " If activated, changes made to the disk are temporary and will"
1044             . " be discarded when the VM is shutdown.",
1045         optional => 1,
1046     },
1047     cache => {
1048         type => 'string',
1049         enum => [qw(none writethrough writeback unsafe directsync)],
1050         description => "The drive's cache mode",
1051         optional => 1,
1052     },
1053     format => get_standard_option('pve-qm-image-format'),
1054     size => {
1055         type => 'string',
1056         format => 'disk-size',
1057         format_description => 'DiskSize',
1058         description => "Disk size. This is purely informational and has no effect.",
1059         optional => 1,
1060     },
1061     backup => {
1062         type => 'boolean',
1063         description => "Whether the drive should be included when making backups.",
1064         optional => 1,
1065     },
1066     replicate => {
1067         type => 'boolean',
1068         description => 'Whether the drive should considered for replication jobs.',
1069         optional => 1,
1070         default => 1,
1071     },
1072     rerror => {
1073         type => 'string',
1074         enum => [qw(ignore report stop)],
1075         description => 'Read error action.',
1076         optional => 1,
1077     },
1078     werror => {
1079         type => 'string',
1080         enum => [qw(enospc ignore report stop)],
1081         description => 'Write error action.',
1082         optional => 1,
1083     },
1084     aio => {
1085         type => 'string',
1086         enum => [qw(native threads)],
1087         description => 'AIO type to use.',
1088         optional => 1,
1089     },
1090     discard => {
1091         type => 'string',
1092         enum => [qw(ignore on)],
1093         description => 'Controls whether to pass discard/trim requests to the underlying storage.',
1094         optional => 1,
1095     },
1096     detect_zeroes => {
1097         type => 'boolean',
1098         description => 'Controls whether to detect and try to optimize writes of zeroes.',
1099         optional => 1,
1100     },
1101     serial => {
1102         type => 'string',
1103         format => 'urlencoded',
1104         format_description => 'serial',
1105         maxLength => 20*3, # *3 since it's %xx url enoded
1106         description => "The drive's reported serial number, url-encoded, up to 20 bytes long.",
1107         optional => 1,
1108     },
1109     shared => {
1110         type => 'boolean',
1111         description => 'Mark this locally-managed volume as available on all nodes',
1112         verbose_description => "Mark this locally-managed volume as available on all nodes.\n\nWARNING: This option does not share the volume automatically, it assumes it is shared already!",
1113         optional => 1,
1114         default => 0,
1115     }
1116 );
1117
1118 my %iothread_fmt = ( iothread => {
1119         type => 'boolean',
1120         description => "Whether to use iothreads for this drive",
1121         optional => 1,
1122 });
1123
1124 my %model_fmt = (
1125     model => {
1126         type => 'string',
1127         format => 'urlencoded',
1128         format_description => 'model',
1129         maxLength => 40*3, # *3 since it's %xx url enoded
1130         description => "The drive's reported model name, url-encoded, up to 40 bytes long.",
1131         optional => 1,
1132     },
1133 );
1134
1135 my %queues_fmt = (
1136     queues => {
1137         type => 'integer',
1138         description => "Number of queues.",
1139         minimum => 2,
1140         optional => 1
1141     }
1142 );
1143
1144 my %scsiblock_fmt = (
1145     scsiblock => {
1146         type => 'boolean',
1147         description => "whether to use scsi-block for full passthrough of host block device\n\nWARNING: can lead to I/O errors in combination with low memory or high memory fragmentation on host",
1148         optional => 1,
1149         default => 0,
1150     },
1151 );
1152
1153 my %ssd_fmt = (
1154     ssd => {
1155         type => 'boolean',
1156         description => "Whether to expose this drive as an SSD, rather than a rotational hard disk.",
1157         optional => 1,
1158     },
1159 );
1160
1161 my %wwn_fmt = (
1162     wwn => {
1163         type => 'string',
1164         pattern => qr/^(0x)[0-9a-fA-F]{16}/,
1165         format_description => 'wwn',
1166         description => "The drive's worldwide name, encoded as 16 bytes hex string, prefixed by '0x'.",
1167         optional => 1,
1168     },
1169 );
1170
1171 my $add_throttle_desc = sub {
1172     my ($key, $type, $what, $unit, $longunit, $minimum) = @_;
1173     my $d = {
1174         type => $type,
1175         format_description => $unit,
1176         description => "Maximum $what in $longunit.",
1177         optional => 1,
1178     };
1179     $d->{minimum} = $minimum if defined($minimum);
1180     $drivedesc_base{$key} = $d;
1181 };
1182 # throughput: (leaky bucket)
1183 $add_throttle_desc->('bps',     'integer', 'r/w speed',   'bps',  'bytes per second');
1184 $add_throttle_desc->('bps_rd',  'integer', 'read speed',  'bps',  'bytes per second');
1185 $add_throttle_desc->('bps_wr',  'integer', 'write speed', 'bps',  'bytes per second');
1186 $add_throttle_desc->('mbps',    'number',  'r/w speed',   'mbps', 'megabytes per second');
1187 $add_throttle_desc->('mbps_rd', 'number',  'read speed',  'mbps', 'megabytes per second');
1188 $add_throttle_desc->('mbps_wr', 'number',  'write speed', 'mbps', 'megabytes per second');
1189 $add_throttle_desc->('iops',    'integer', 'r/w I/O',     'iops', 'operations per second');
1190 $add_throttle_desc->('iops_rd', 'integer', 'read I/O',    'iops', 'operations per second');
1191 $add_throttle_desc->('iops_wr', 'integer', 'write I/O',   'iops', 'operations per second');
1192
1193 # pools: (pool of IO before throttling starts taking effect)
1194 $add_throttle_desc->('mbps_max',    'number',  'unthrottled r/w pool',       'mbps', 'megabytes per second');
1195 $add_throttle_desc->('mbps_rd_max', 'number',  'unthrottled read pool',      'mbps', 'megabytes per second');
1196 $add_throttle_desc->('mbps_wr_max', 'number',  'unthrottled write pool',     'mbps', 'megabytes per second');
1197 $add_throttle_desc->('iops_max',    'integer', 'unthrottled r/w I/O pool',   'iops', 'operations per second');
1198 $add_throttle_desc->('iops_rd_max', 'integer', 'unthrottled read I/O pool',  'iops', 'operations per second');
1199 $add_throttle_desc->('iops_wr_max', 'integer', 'unthrottled write I/O pool', 'iops', 'operations per second');
1200
1201 # burst lengths
1202 $add_throttle_desc->('bps_max_length',     'integer', 'length of I/O bursts',       'seconds', 'seconds', 1);
1203 $add_throttle_desc->('bps_rd_max_length',  'integer', 'length of read I/O bursts',  'seconds', 'seconds', 1);
1204 $add_throttle_desc->('bps_wr_max_length',  'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
1205 $add_throttle_desc->('iops_max_length',    'integer', 'length of I/O bursts',       'seconds', 'seconds', 1);
1206 $add_throttle_desc->('iops_rd_max_length', 'integer', 'length of read I/O bursts',  'seconds', 'seconds', 1);
1207 $add_throttle_desc->('iops_wr_max_length', 'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
1208
1209 # legacy support
1210 $drivedesc_base{'bps_rd_length'} = { alias => 'bps_rd_max_length' };
1211 $drivedesc_base{'bps_wr_length'} = { alias => 'bps_wr_max_length' };
1212 $drivedesc_base{'iops_rd_length'} = { alias => 'iops_rd_max_length' };
1213 $drivedesc_base{'iops_wr_length'} = { alias => 'iops_wr_max_length' };
1214
1215 my $ide_fmt = {
1216     %drivedesc_base,
1217     %model_fmt,
1218     %ssd_fmt,
1219     %wwn_fmt,
1220 };
1221 PVE::JSONSchema::register_format("pve-qm-ide", $ide_fmt);
1222
1223 my $idedesc = {
1224     optional => 1,
1225     type => 'string', format => $ide_fmt,
1226     description => "Use volume as IDE hard disk or CD-ROM (n is 0 to " .($MAX_IDE_DISKS -1) . ").",
1227 };
1228 PVE::JSONSchema::register_standard_option("pve-qm-ide", $idedesc);
1229
1230 my $scsi_fmt = {
1231     %drivedesc_base,
1232     %iothread_fmt,
1233     %queues_fmt,
1234     %scsiblock_fmt,
1235     %ssd_fmt,
1236     %wwn_fmt,
1237 };
1238 my $scsidesc = {
1239     optional => 1,
1240     type => 'string', format => $scsi_fmt,
1241     description => "Use volume as SCSI hard disk or CD-ROM (n is 0 to " . ($MAX_SCSI_DISKS - 1) . ").",
1242 };
1243 PVE::JSONSchema::register_standard_option("pve-qm-scsi", $scsidesc);
1244
1245 my $sata_fmt = {
1246     %drivedesc_base,
1247     %ssd_fmt,
1248     %wwn_fmt,
1249 };
1250 my $satadesc = {
1251     optional => 1,
1252     type => 'string', format => $sata_fmt,
1253     description => "Use volume as SATA hard disk or CD-ROM (n is 0 to " . ($MAX_SATA_DISKS - 1). ").",
1254 };
1255 PVE::JSONSchema::register_standard_option("pve-qm-sata", $satadesc);
1256
1257 my $virtio_fmt = {
1258     %drivedesc_base,
1259     %iothread_fmt,
1260 };
1261 my $virtiodesc = {
1262     optional => 1,
1263     type => 'string', format => $virtio_fmt,
1264     description => "Use volume as VIRTIO hard disk (n is 0 to " . ($MAX_VIRTIO_DISKS - 1) . ").",
1265 };
1266 PVE::JSONSchema::register_standard_option("pve-qm-virtio", $virtiodesc);
1267
1268 my $alldrive_fmt = {
1269     %drivedesc_base,
1270     %iothread_fmt,
1271     %model_fmt,
1272     %queues_fmt,
1273     %scsiblock_fmt,
1274     %ssd_fmt,
1275     %wwn_fmt,
1276 };
1277
1278 my $efidisk_fmt = {
1279     volume => { alias => 'file' },
1280     file => {
1281         type => 'string',
1282         format => 'pve-volume-id-or-qm-path',
1283         default_key => 1,
1284         format_description => 'volume',
1285         description => "The drive's backing volume.",
1286     },
1287     format => get_standard_option('pve-qm-image-format'),
1288     size => {
1289         type => 'string',
1290         format => 'disk-size',
1291         format_description => 'DiskSize',
1292         description => "Disk size. This is purely informational and has no effect.",
1293         optional => 1,
1294     },
1295 };
1296
1297 my $efidisk_desc = {
1298     optional => 1,
1299     type => 'string', format => $efidisk_fmt,
1300     description => "Configure a Disk for storing EFI vars",
1301 };
1302
1303 PVE::JSONSchema::register_standard_option("pve-qm-efidisk", $efidisk_desc);
1304
1305 my $usb_fmt = {
1306     host => {
1307         default_key => 1,
1308         type => 'string', format => 'pve-qm-usb-device',
1309         format_description => 'HOSTUSBDEVICE|spice',
1310         description => <<EODESCR,
1311 The Host USB device or port or the value 'spice'. HOSTUSBDEVICE syntax is:
1312
1313  'bus-port(.port)*' (decimal numbers) or
1314  'vendor_id:product_id' (hexadeciaml numbers) or
1315  'spice'
1316
1317 You can use the 'lsusb -t' command to list existing usb devices.
1318
1319 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1320
1321 The value 'spice' can be used to add a usb redirection devices for spice.
1322 EODESCR
1323     },
1324     usb3 => {
1325         optional => 1,
1326         type => 'boolean',
1327         description => "Specifies whether if given host option is a USB3 device or port.",
1328         default => 0,
1329     },
1330 };
1331
1332 my $usbdesc = {
1333     optional => 1,
1334     type => 'string', format => $usb_fmt,
1335     description => "Configure an USB device (n is 0 to 4).",
1336 };
1337 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
1338
1339 my $PCIRE = qr/[a-f0-9]{2}:[a-f0-9]{2}(?:\.[a-f0-9])?/;
1340 my $hostpci_fmt = {
1341     host => {
1342         default_key => 1,
1343         type => 'string',
1344         pattern => qr/$PCIRE(;$PCIRE)*/,
1345         format_description => 'HOSTPCIID[;HOSTPCIID2...]',
1346         description => <<EODESCR,
1347 Host PCI device pass through. The PCI ID of a host's PCI device or a list
1348 of PCI virtual functions of the host. HOSTPCIID syntax is:
1349
1350 'bus:dev.func' (hexadecimal numbers)
1351
1352 You can us the 'lspci' command to list existing PCI devices.
1353 EODESCR
1354     },
1355     rombar => {
1356         type => 'boolean',
1357         description =>  "Specify whether or not the device's ROM will be visible in the guest's memory map.",
1358         optional => 1,
1359         default => 1,
1360     },
1361     romfile => {
1362         type => 'string',
1363         pattern => '[^,;]+',
1364         format_description => 'string',
1365         description => "Custom pci device rom filename (must be located in /usr/share/kvm/).",
1366         optional => 1,
1367     },
1368     pcie => {
1369         type => 'boolean',
1370         description =>  "Choose the PCI-express bus (needs the 'q35' machine model).",
1371         optional => 1,
1372         default => 0,
1373     },
1374     'x-vga' => {
1375         type => 'boolean',
1376         description =>  "Enable vfio-vga device support.",
1377         optional => 1,
1378         default => 0,
1379     },
1380     'mdev' => {
1381         type => 'string',
1382         format_description => 'string',
1383         pattern => '[^/\.:]+',
1384         optional => 1,
1385         description => <<EODESCR
1386 The type of mediated device to use.
1387 An instance of this type will be created on startup of the VM and
1388 will be cleaned up when the VM stops.
1389 EODESCR
1390     }
1391 };
1392 PVE::JSONSchema::register_format('pve-qm-hostpci', $hostpci_fmt);
1393
1394 my $hostpcidesc = {
1395         optional => 1,
1396         type => 'string', format => 'pve-qm-hostpci',
1397         description => "Map host PCI devices into guest.",
1398         verbose_description =>  <<EODESCR,
1399 Map host PCI devices into guest.
1400
1401 NOTE: This option allows direct access to host hardware. So it is no longer
1402 possible to migrate such machines - use with special care.
1403
1404 CAUTION: Experimental! User reported problems with this option.
1405 EODESCR
1406 };
1407 PVE::JSONSchema::register_standard_option("pve-qm-hostpci", $hostpcidesc);
1408
1409 my $serialdesc = {
1410         optional => 1,
1411         type => 'string',
1412         pattern => '(/dev/.+|socket)',
1413         description =>  "Create a serial device inside the VM (n is 0 to 3)",
1414         verbose_description =>  <<EODESCR,
1415 Create a serial device inside the VM (n is 0 to 3), and pass through a
1416 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1417 host side (use 'qm terminal' to open a terminal connection).
1418
1419 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines - use with special care.
1420
1421 CAUTION: Experimental! User reported problems with this option.
1422 EODESCR
1423 };
1424
1425 my $paralleldesc= {
1426         optional => 1,
1427         type => 'string',
1428         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
1429         description =>  "Map host parallel devices (n is 0 to 2).",
1430         verbose_description =>  <<EODESCR,
1431 Map host parallel devices (n is 0 to 2).
1432
1433 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1434
1435 CAUTION: Experimental! User reported problems with this option.
1436 EODESCR
1437 };
1438
1439 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
1440     $confdesc->{"parallel$i"} = $paralleldesc;
1441 }
1442
1443 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
1444     $confdesc->{"serial$i"} = $serialdesc;
1445 }
1446
1447 for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
1448     $confdesc->{"hostpci$i"} = $hostpcidesc;
1449 }
1450
1451 for (my $i = 0; $i < $MAX_IDE_DISKS; $i++)  {
1452     $drivename_hash->{"ide$i"} = 1;
1453     $confdesc->{"ide$i"} = $idedesc;
1454 }
1455
1456 for (my $i = 0; $i < $MAX_SATA_DISKS; $i++)  {
1457     $drivename_hash->{"sata$i"} = 1;
1458     $confdesc->{"sata$i"} = $satadesc;
1459 }
1460
1461 for (my $i = 0; $i < $MAX_SCSI_DISKS; $i++)  {
1462     $drivename_hash->{"scsi$i"} = 1;
1463     $confdesc->{"scsi$i"} = $scsidesc ;
1464 }
1465
1466 for (my $i = 0; $i < $MAX_VIRTIO_DISKS; $i++)  {
1467     $drivename_hash->{"virtio$i"} = 1;
1468     $confdesc->{"virtio$i"} = $virtiodesc;
1469 }
1470
1471 $drivename_hash->{efidisk0} = 1;
1472 $confdesc->{efidisk0} = $efidisk_desc;
1473
1474 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
1475     $confdesc->{"usb$i"} = $usbdesc;
1476 }
1477
1478 my $unuseddesc = {
1479     optional => 1,
1480     type => 'string', format => 'pve-volume-id',
1481     description => "Reference to unused volumes. This is used internally, and should not be modified manually.",
1482 };
1483
1484 for (my $i = 0; $i < $MAX_UNUSED_DISKS; $i++)  {
1485     $confdesc->{"unused$i"} = $unuseddesc;
1486 }
1487
1488 my $kvm_api_version = 0;
1489
1490 sub kvm_version {
1491     return $kvm_api_version if $kvm_api_version;
1492
1493     open my $fh, '<', '/dev/kvm'
1494         or return undef;
1495
1496     # 0xae00 => KVM_GET_API_VERSION
1497     $kvm_api_version = ioctl($fh, 0xae00, 0);
1498
1499     return $kvm_api_version;
1500 }
1501
1502 my $kvm_user_version = {};
1503 my $kvm_mtime = {};
1504
1505 sub kvm_user_version {
1506     my ($binary) = @_;
1507
1508     $binary //= get_command_for_arch(get_host_arch()); # get the native arch by default
1509     my $st = stat($binary);
1510
1511     my $cachedmtime = $kvm_mtime->{$binary} // -1;
1512     return $kvm_user_version->{$binary} if $kvm_user_version->{$binary} &&
1513         $cachedmtime == $st->mtime;
1514
1515     $kvm_user_version->{$binary} = 'unknown';
1516     $kvm_mtime->{$binary} = $st->mtime;
1517
1518     my $code = sub {
1519         my $line = shift;
1520         if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1521             $kvm_user_version->{$binary} = $2;
1522         }
1523     };
1524
1525     eval { run_command([$binary, '--version'], outfunc => $code); };
1526     warn $@ if $@;
1527
1528     return $kvm_user_version->{$binary};
1529
1530 }
1531
1532 sub kernel_has_vhost_net {
1533     return -c '/dev/vhost-net';
1534 }
1535
1536 sub valid_drive_names {
1537     # order is important - used to autoselect boot disk
1538     return ((map { "ide$_" } (0 .. ($MAX_IDE_DISKS - 1))),
1539             (map { "scsi$_" } (0 .. ($MAX_SCSI_DISKS - 1))),
1540             (map { "virtio$_" } (0 .. ($MAX_VIRTIO_DISKS - 1))),
1541             (map { "sata$_" } (0 .. ($MAX_SATA_DISKS - 1))),
1542             'efidisk0');
1543 }
1544
1545 sub is_valid_drivename {
1546     my $dev = shift;
1547
1548     return defined($drivename_hash->{$dev});
1549 }
1550
1551 sub option_exists {
1552     my $key = shift;
1553     return defined($confdesc->{$key});
1554 }
1555
1556 sub nic_models {
1557     return $nic_model_list;
1558 }
1559
1560 sub os_list_description {
1561
1562     return {
1563         other => 'Other',
1564         wxp => 'Windows XP',
1565         w2k => 'Windows 2000',
1566         w2k3 =>, 'Windows 2003',
1567         w2k8 => 'Windows 2008',
1568         wvista => 'Windows Vista',
1569         win7 => 'Windows 7',
1570         win8 => 'Windows 8/2012',
1571         win10 => 'Windows 10/2016',
1572         l24 => 'Linux 2.4',
1573         l26 => 'Linux 2.6',
1574     };
1575 }
1576
1577 my $cdrom_path;
1578
1579 sub get_cdrom_path {
1580
1581     return  $cdrom_path if $cdrom_path;
1582
1583     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
1584     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
1585     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
1586 }
1587
1588 sub get_iso_path {
1589     my ($storecfg, $vmid, $cdrom) = @_;
1590
1591     if ($cdrom eq 'cdrom') {
1592         return get_cdrom_path();
1593     } elsif ($cdrom eq 'none') {
1594         return '';
1595     } elsif ($cdrom =~ m|^/|) {
1596         return $cdrom;
1597     } else {
1598         return PVE::Storage::path($storecfg, $cdrom);
1599     }
1600 }
1601
1602 # try to convert old style file names to volume IDs
1603 sub filename_to_volume_id {
1604     my ($vmid, $file, $media) = @_;
1605
1606      if (!($file eq 'none' || $file eq 'cdrom' ||
1607           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1608
1609         return undef if $file =~ m|/|;
1610
1611         if ($media && $media eq 'cdrom') {
1612             $file = "local:iso/$file";
1613         } else {
1614             $file = "local:$vmid/$file";
1615         }
1616     }
1617
1618     return $file;
1619 }
1620
1621 sub verify_media_type {
1622     my ($opt, $vtype, $media) = @_;
1623
1624     return if !$media;
1625
1626     my $etype;
1627     if ($media eq 'disk') {
1628         $etype = 'images';
1629     } elsif ($media eq 'cdrom') {
1630         $etype = 'iso';
1631     } else {
1632         die "internal error";
1633     }
1634
1635     return if ($vtype eq $etype);
1636
1637     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1638 }
1639
1640 sub cleanup_drive_path {
1641     my ($opt, $storecfg, $drive) = @_;
1642
1643     # try to convert filesystem paths to volume IDs
1644
1645     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1646         ($drive->{file} !~ m|^/dev/.+|) &&
1647         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1648         ($drive->{file} !~ m/^\d+$/)) {
1649         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1650         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"}) if !$vtype;
1651         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1652         verify_media_type($opt, $vtype, $drive->{media});
1653         $drive->{file} = $volid;
1654     }
1655
1656     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1657 }
1658
1659 sub parse_hotplug_features {
1660     my ($data) = @_;
1661
1662     my $res = {};
1663
1664     return $res if $data eq '0';
1665
1666     $data = $confdesc->{hotplug}->{default} if $data eq '1';
1667
1668     foreach my $feature (PVE::Tools::split_list($data)) {
1669         if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
1670             $res->{$1} = 1;
1671         } else {
1672             die "invalid hotplug feature '$feature'\n";
1673         }
1674     }
1675     return $res;
1676 }
1677
1678 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1679 sub pve_verify_hotplug_features {
1680     my ($value, $noerr) = @_;
1681
1682     return $value if parse_hotplug_features($value);
1683
1684     return undef if $noerr;
1685
1686     die "unable to parse hotplug option\n";
1687 }
1688
1689 # ideX = [volume=]volume-id[,media=d][,cyls=c,heads=h,secs=s[,trans=t]]
1690 #        [,snapshot=on|off][,cache=on|off][,format=f][,backup=yes|no]
1691 #        [,rerror=ignore|report|stop][,werror=enospc|ignore|report|stop]
1692 #        [,aio=native|threads][,discard=ignore|on][,detect_zeroes=on|off]
1693 #        [,iothread=on][,serial=serial][,model=model]
1694
1695 sub parse_drive {
1696     my ($key, $data) = @_;
1697
1698     my ($interface, $index);
1699
1700     if ($key =~ m/^([^\d]+)(\d+)$/) {
1701         $interface = $1;
1702         $index = $2;
1703     } else {
1704         return undef;
1705     }
1706
1707     my $desc = $key =~ /^unused\d+$/ ? $alldrive_fmt
1708                                      : $confdesc->{$key}->{format};
1709     if (!$desc) {
1710         warn "invalid drive key: $key\n";
1711         return undef;
1712     }
1713     my $res = eval { PVE::JSONSchema::parse_property_string($desc, $data) };
1714     return undef if !$res;
1715     $res->{interface} = $interface;
1716     $res->{index} = $index;
1717
1718     my $error = 0;
1719     foreach my $opt (qw(bps bps_rd bps_wr)) {
1720         if (my $bps = defined(delete $res->{$opt})) {
1721             if (defined($res->{"m$opt"})) {
1722                 warn "both $opt and m$opt specified\n";
1723                 ++$error;
1724                 next;
1725             }
1726             $res->{"m$opt"} = sprintf("%.3f", $bps / (1024*1024.0));
1727         }
1728     }
1729
1730     # can't use the schema's 'requires' because of the mbps* => bps* "transforming aliases"
1731     for my $requirement (
1732         [mbps_max => 'mbps'],
1733         [mbps_rd_max => 'mbps_rd'],
1734         [mbps_wr_max => 'mbps_wr'],
1735         [miops_max => 'miops'],
1736         [miops_rd_max => 'miops_rd'],
1737         [miops_wr_max => 'miops_wr'],
1738         [bps_max_length => 'mbps_max'],
1739         [bps_rd_max_length => 'mbps_rd_max'],
1740         [bps_wr_max_length => 'mbps_wr_max'],
1741         [iops_max_length => 'iops_max'],
1742         [iops_rd_max_length => 'iops_rd_max'],
1743         [iops_wr_max_length => 'iops_wr_max']) {
1744         my ($option, $requires) = @$requirement;
1745         if ($res->{$option} && !$res->{$requires}) {
1746             warn "$option requires $requires\n";
1747             ++$error;
1748         }
1749     }
1750
1751     return undef if $error;
1752
1753     return undef if $res->{mbps_rd} && $res->{mbps};
1754     return undef if $res->{mbps_wr} && $res->{mbps};
1755     return undef if $res->{iops_rd} && $res->{iops};
1756     return undef if $res->{iops_wr} && $res->{iops};
1757
1758     if ($res->{media} && ($res->{media} eq 'cdrom')) {
1759         return undef if $res->{snapshot} || $res->{trans} || $res->{format};
1760         return undef if $res->{heads} || $res->{secs} || $res->{cyls};
1761         return undef if $res->{interface} eq 'virtio';
1762     }
1763
1764     if (my $size = $res->{size}) {
1765         return undef if !defined($res->{size} = PVE::JSONSchema::parse_size($size));
1766     }
1767
1768     return $res;
1769 }
1770
1771 sub print_drive {
1772     my ($vmid, $drive) = @_;
1773     my $data = { %$drive };
1774     delete $data->{$_} for qw(index interface);
1775     return PVE::JSONSchema::print_property_string($data, $alldrive_fmt);
1776 }
1777
1778 sub scsi_inquiry {
1779     my($fh, $noerr) = @_;
1780
1781     my $SG_IO = 0x2285;
1782     my $SG_GET_VERSION_NUM = 0x2282;
1783
1784     my $versionbuf = "\x00" x 8;
1785     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1786     if (!$ret) {
1787         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1788         return undef;
1789     }
1790     my $version = unpack("I", $versionbuf);
1791     if ($version < 30000) {
1792         die "scsi generic interface too old\n"  if !$noerr;
1793         return undef;
1794     }
1795
1796     my $buf = "\x00" x 36;
1797     my $sensebuf = "\x00" x 8;
1798     my $cmd = pack("C x3 C x1", 0x12, 36);
1799
1800     # see /usr/include/scsi/sg.h
1801     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1802
1803     my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1804                       length($sensebuf), 0, length($buf), $buf,
1805                       $cmd, $sensebuf, 6000);
1806
1807     $ret = ioctl($fh, $SG_IO, $packet);
1808     if (!$ret) {
1809         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1810         return undef;
1811     }
1812
1813     my @res = unpack($sg_io_hdr_t, $packet);
1814     if ($res[17] || $res[18]) {
1815         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1816         return undef;
1817     }
1818
1819     my $res = {};
1820     (my $byte0, my $byte1, $res->{vendor},
1821      $res->{product}, $res->{revision}) = unpack("C C x6 A8 A16 A4", $buf);
1822
1823     $res->{removable} = $byte1 & 128 ? 1 : 0;
1824     $res->{type} = $byte0 & 31;
1825
1826     return $res;
1827 }
1828
1829 sub path_is_scsi {
1830     my ($path) = @_;
1831
1832     my $fh = IO::File->new("+<$path") || return undef;
1833     my $res = scsi_inquiry($fh, 1);
1834     close($fh);
1835
1836     return $res;
1837 }
1838
1839 sub machine_type_is_q35 {
1840     my ($conf) = @_;
1841
1842     return $conf->{machine} && ($conf->{machine} =~ m/q35/) ? 1 : 0;
1843 }
1844
1845 sub print_tabletdevice_full {
1846     my ($conf, $arch) = @_;
1847
1848     my $q35 = machine_type_is_q35($conf);
1849
1850     # we use uhci for old VMs because tablet driver was buggy in older qemu
1851     my $usbbus;
1852     if (machine_type_is_q35($conf) || $arch eq 'aarch64') {
1853         $usbbus = 'ehci';
1854     } else {
1855         $usbbus = 'uhci';
1856     }
1857
1858     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1859 }
1860
1861 sub print_keyboarddevice_full {
1862     my ($conf, $arch, $machine) = @_;
1863
1864     return undef if $arch ne 'aarch64';
1865
1866     return "usb-kbd,id=keyboard,bus=ehci.0,port=2";
1867 }
1868
1869 sub print_drivedevice_full {
1870     my ($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type) = @_;
1871
1872     my $device = '';
1873     my $maxdev = 0;
1874
1875     if ($drive->{interface} eq 'virtio') {
1876         my $pciaddr = print_pci_addr("$drive->{interface}$drive->{index}", $bridges, $arch, $machine_type);
1877         $device = "virtio-blk-pci,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}$pciaddr";
1878         $device .= ",iothread=iothread-$drive->{interface}$drive->{index}" if $drive->{iothread};
1879     } elsif ($drive->{interface} eq 'scsi') {
1880
1881         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1882         my $unit = $drive->{index} % $maxdev;
1883         my $devicetype = 'hd';
1884         my $path = '';
1885         if (drive_is_cdrom($drive)) {
1886             $devicetype = 'cd';
1887         } else {
1888             if ($drive->{file} =~ m|^/|) {
1889                 $path = $drive->{file};
1890                 if (my $info = path_is_scsi($path)) {
1891                     if ($info->{type} == 0 && $drive->{scsiblock}) {
1892                         $devicetype = 'block';
1893                     } elsif ($info->{type} == 1) { # tape
1894                         $devicetype = 'generic';
1895                     }
1896                 }
1897             } else {
1898                  $path = PVE::Storage::path($storecfg, $drive->{file});
1899             }
1900
1901             if($path =~ m/^iscsi\:\/\//){
1902                 $devicetype = 'generic';
1903             }
1904         }
1905
1906         if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)){
1907             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1908         } else {
1909             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0,lun=$drive->{index},drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1910         }
1911
1912         if ($drive->{ssd} && ($devicetype eq 'block' || $devicetype eq 'hd')) {
1913             $device .= ",rotation_rate=1";
1914         }
1915         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1916
1917     } elsif ($drive->{interface} eq 'ide' || $drive->{interface} eq 'sata') {
1918         my $maxdev = ($drive->{interface} eq 'sata') ? $MAX_SATA_DISKS : 2;
1919         my $controller = int($drive->{index} / $maxdev);
1920         my $unit = $drive->{index} % $maxdev;
1921         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1922
1923         $device = "ide-$devicetype";
1924         if ($drive->{interface} eq 'ide') {
1925             $device .= ",bus=ide.$controller,unit=$unit";
1926         } else {
1927             $device .= ",bus=ahci$controller.$unit";
1928         }
1929         $device .= ",drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1930
1931         if ($devicetype eq 'hd') {
1932             if (my $model = $drive->{model}) {
1933                 $model = URI::Escape::uri_unescape($model);
1934                 $device .= ",model=$model";
1935             }
1936             if ($drive->{ssd}) {
1937                 $device .= ",rotation_rate=1";
1938             }
1939         }
1940         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1941     } elsif ($drive->{interface} eq 'usb') {
1942         die "implement me";
1943         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1944     } else {
1945         die "unsupported interface type";
1946     }
1947
1948     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1949
1950     if (my $serial = $drive->{serial}) {
1951         $serial = URI::Escape::uri_unescape($serial);
1952         $device .= ",serial=$serial";
1953     }
1954
1955
1956     return $device;
1957 }
1958
1959 sub get_initiator_name {
1960     my $initiator;
1961
1962     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return undef;
1963     while (defined(my $line = <$fh>)) {
1964         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1965         $initiator = $1;
1966         last;
1967     }
1968     $fh->close();
1969
1970     return $initiator;
1971 }
1972
1973 sub print_drive_full {
1974     my ($storecfg, $vmid, $drive) = @_;
1975
1976     my $path;
1977     my $volid = $drive->{file};
1978     my $format;
1979
1980     if (drive_is_cdrom($drive)) {
1981         $path = get_iso_path($storecfg, $vmid, $volid);
1982     } else {
1983         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1984         if ($storeid) {
1985             $path = PVE::Storage::path($storecfg, $volid);
1986             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
1987             $format = qemu_img_format($scfg, $volname);
1988         } else {
1989             $path = $volid;
1990             $format = "raw";
1991         }
1992    }
1993
1994     my $opts = '';
1995     my @qemu_drive_options = qw(heads secs cyls trans media format cache rerror werror aio discard);
1996     foreach my $o (@qemu_drive_options) {
1997         $opts .= ",$o=$drive->{$o}" if defined($drive->{$o});
1998     }
1999
2000     # snapshot only accepts on|off
2001     if (defined($drive->{snapshot})) {
2002         my $v = $drive->{snapshot} ? 'on' : 'off';
2003         $opts .= ",snapshot=$v";
2004     }
2005
2006     foreach my $type (['', '-total'], [_rd => '-read'], [_wr => '-write']) {
2007         my ($dir, $qmpname) = @$type;
2008         if (my $v = $drive->{"mbps$dir"}) {
2009             $opts .= ",throttling.bps$qmpname=".int($v*1024*1024);
2010         }
2011         if (my $v = $drive->{"mbps${dir}_max"}) {
2012             $opts .= ",throttling.bps$qmpname-max=".int($v*1024*1024);
2013         }
2014         if (my $v = $drive->{"bps${dir}_max_length"}) {
2015             $opts .= ",throttling.bps$qmpname-max-length=$v";
2016         }
2017         if (my $v = $drive->{"iops${dir}"}) {
2018             $opts .= ",throttling.iops$qmpname=$v";
2019         }
2020         if (my $v = $drive->{"iops${dir}_max"}) {
2021             $opts .= ",throttling.iops$qmpname-max=$v";
2022         }
2023         if (my $v = $drive->{"iops${dir}_max_length"}) {
2024             $opts .= ",throttling.iops$qmpname-max-length=$v";
2025         }
2026     }
2027
2028     $opts .= ",format=$format" if $format && !$drive->{format};
2029
2030     my $cache_direct = 0;
2031
2032     if (my $cache = $drive->{cache}) {
2033         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
2034     } elsif (!drive_is_cdrom($drive)) {
2035         $opts .= ",cache=none";
2036         $cache_direct = 1;
2037     }
2038
2039     # aio native works only with O_DIRECT
2040     if (!$drive->{aio}) {
2041         if($cache_direct) {
2042             $opts .= ",aio=native";
2043         } else {
2044             $opts .= ",aio=threads";
2045         }
2046     }
2047
2048     if (!drive_is_cdrom($drive)) {
2049         my $detectzeroes;
2050         if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
2051             $detectzeroes = 'off';
2052         } elsif ($drive->{discard}) {
2053             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
2054         } else {
2055             # This used to be our default with discard not being specified:
2056             $detectzeroes = 'on';
2057         }
2058         $opts .= ",detect-zeroes=$detectzeroes" if $detectzeroes;
2059     }
2060
2061     my $pathinfo = $path ? "file=$path," : '';
2062
2063     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
2064 }
2065
2066 sub print_netdevice_full {
2067     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files, $arch, $machine_type) = @_;
2068
2069     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
2070
2071     my $device = $net->{model};
2072     if ($net->{model} eq 'virtio') {
2073          $device = 'virtio-net-pci';
2074      };
2075
2076     my $pciaddr = print_pci_addr("$netid", $bridges, $arch, $machine_type);
2077     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
2078     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
2079         #Consider we have N queues, the number of vectors needed is 2*N + 2 (plus one config interrupt and control vq)
2080         my $vectors = $net->{queues} * 2 + 2;
2081         $tmpstr .= ",vectors=$vectors,mq=on";
2082     }
2083     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
2084
2085     if ($use_old_bios_files) {
2086         my $romfile;
2087         if ($device eq 'virtio-net-pci') {
2088             $romfile = 'pxe-virtio.rom';
2089         } elsif ($device eq 'e1000') {
2090             $romfile = 'pxe-e1000.rom';
2091         } elsif ($device eq 'ne2k') {
2092             $romfile = 'pxe-ne2k_pci.rom';
2093         } elsif ($device eq 'pcnet') {
2094             $romfile = 'pxe-pcnet.rom';
2095         } elsif ($device eq 'rtl8139') {
2096             $romfile = 'pxe-rtl8139.rom';
2097         }
2098         $tmpstr .= ",romfile=$romfile" if $romfile;
2099     }
2100
2101     return $tmpstr;
2102 }
2103
2104 sub print_netdev_full {
2105     my ($vmid, $conf, $arch, $net, $netid, $hotplug) = @_;
2106
2107     my $i = '';
2108     if ($netid =~ m/^net(\d+)$/) {
2109         $i = int($1);
2110     }
2111
2112     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
2113
2114     my $ifname = "tap${vmid}i$i";
2115
2116     # kvm uses TUNSETIFF ioctl, and that limits ifname length
2117     die "interface name '$ifname' is too long (max 15 character)\n"
2118         if length($ifname) >= 16;
2119
2120     my $vhostparam = '';
2121     if (is_native($arch)) {
2122         $vhostparam = ',vhost=on' if kernel_has_vhost_net() && $net->{model} eq 'virtio';
2123     }
2124
2125     my $vmname = $conf->{name} || "vm$vmid";
2126
2127     my $netdev = "";
2128     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
2129
2130     if ($net->{bridge}) {
2131         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script,downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
2132     } else {
2133         $netdev = "type=user,id=$netid,hostname=$vmname";
2134     }
2135
2136     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
2137
2138     return $netdev;
2139 }
2140
2141
2142 sub print_cpu_device {
2143     my ($conf, $id) = @_;
2144
2145     my $kvm = $conf->{kvm} // 1;
2146     my $cpu = $kvm ? "kvm64" : "qemu64";
2147     if (my $cputype = $conf->{cpu}) {
2148         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
2149             or die "Cannot parse cpu description: $cputype\n";
2150         $cpu = $cpuconf->{cputype};
2151     }
2152
2153     my $cores = $conf->{cores} || 1;
2154
2155     my $current_core = ($id - 1) % $cores;
2156     my $current_socket = int(($id - 1 - $current_core)/$cores);
2157
2158     return "$cpu-x86_64-cpu,id=cpu$id,socket-id=$current_socket,core-id=$current_core,thread-id=0";
2159 }
2160
2161 my $vga_map = {
2162     'cirrus' => 'cirrus-vga',
2163     'std' => 'VGA',
2164     'vmware' => 'vmware-svga',
2165     'virtio' => 'virtio-vga',
2166 };
2167
2168 sub print_vga_device {
2169     my ($conf, $vga, $arch, $machine, $id, $qxlnum, $bridges) = @_;
2170
2171     my $type = $vga_map->{$vga->{type}};
2172     if ($arch eq 'aarch64' && defined($type) && $type eq 'virtio-vga') {
2173         $type = 'virtio-gpu';
2174     }
2175     my $vgamem_mb = $vga->{memory};
2176     if ($qxlnum) {
2177         $type = $id ? 'qxl' : 'qxl-vga';
2178     }
2179     die "no devicetype for $vga->{type}\n" if !$type;
2180
2181     my $memory = "";
2182     if ($vgamem_mb) {
2183         if ($vga->{type} eq 'virtio') {
2184             my $bytes = PVE::Tools::convert_size($vgamem_mb, "mb" => "b");
2185             $memory = ",max_hostmem=$bytes";
2186         } elsif ($qxlnum) {
2187             # from https://www.spice-space.org/multiple-monitors.html
2188             $memory = ",vgamem_mb=$vga->{memory}";
2189             my $ram = $vgamem_mb * 4;
2190             my $vram = $vgamem_mb * 2;
2191             $memory .= ",ram_size_mb=$ram,vram_size_mb=$vram";
2192         } else {
2193             $memory = ",vgamem_mb=$vga->{memory}";
2194         }
2195     } elsif ($qxlnum && $id) {
2196         $memory = ",ram_size=67108864,vram_size=33554432";
2197     }
2198
2199     my $q35 = machine_type_is_q35($conf);
2200     my $vgaid = "vga" . ($id // '');
2201     my $pciaddr;
2202
2203     if ($q35 && $vgaid eq 'vga') {
2204         # the first display uses pcie.0 bus on q35 machines
2205         $pciaddr = print_pcie_addr($vgaid, $bridges, $arch, $machine);
2206     } else {
2207         $pciaddr = print_pci_addr($vgaid, $bridges, $arch, $machine);
2208     }
2209
2210     return "$type,id=${vgaid}${memory}${pciaddr}";
2211 }
2212
2213 sub drive_is_cloudinit {
2214     my ($drive) = @_;
2215     return $drive->{file} =~ m@[:/]vm-\d+-cloudinit(?:\.$QEMU_FORMAT_RE)?$@;
2216 }
2217
2218 sub drive_is_cdrom {
2219     my ($drive, $exclude_cloudinit) = @_;
2220
2221     return 0 if $exclude_cloudinit && drive_is_cloudinit($drive);
2222
2223     return $drive && $drive->{media} && ($drive->{media} eq 'cdrom');
2224
2225 }
2226
2227 sub parse_number_sets {
2228     my ($set) = @_;
2229     my $res = [];
2230     foreach my $part (split(/;/, $set)) {
2231         if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
2232             die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
2233             push @$res, [ $1, $2 ];
2234         } else {
2235             die "invalid range: $part\n";
2236         }
2237     }
2238     return $res;
2239 }
2240
2241 sub parse_numa {
2242     my ($data) = @_;
2243
2244     my $res = PVE::JSONSchema::parse_property_string($numa_fmt, $data);
2245     $res->{cpus} = parse_number_sets($res->{cpus}) if defined($res->{cpus});
2246     $res->{hostnodes} = parse_number_sets($res->{hostnodes}) if defined($res->{hostnodes});
2247     return $res;
2248 }
2249
2250 sub parse_hostpci {
2251     my ($value) = @_;
2252
2253     return undef if !$value;
2254
2255     my $res = PVE::JSONSchema::parse_property_string($hostpci_fmt, $value);
2256
2257     my @idlist = split(/;/, $res->{host});
2258     delete $res->{host};
2259     foreach my $id (@idlist) {
2260         if ($id =~ m/\./) { # full id 00:00.1
2261             push @{$res->{pciid}}, {
2262                 id => $id,
2263             };
2264         } else { # partial id 00:00
2265             $res->{pciid} = PVE::SysFSTools::lspci($id);
2266         }
2267     }
2268     return $res;
2269 }
2270
2271 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
2272 sub parse_net {
2273     my ($data) = @_;
2274
2275     my $res = eval { PVE::JSONSchema::parse_property_string($net_fmt, $data) };
2276     if ($@) {
2277         warn $@;
2278         return undef;
2279     }
2280     if (!defined($res->{macaddr})) {
2281         my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
2282         $res->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix});
2283     }
2284     return $res;
2285 }
2286
2287 # ipconfigX ip=cidr,gw=ip,ip6=cidr,gw6=ip
2288 sub parse_ipconfig {
2289     my ($data) = @_;
2290
2291     my $res = eval { PVE::JSONSchema::parse_property_string($ipconfig_fmt, $data) };
2292     if ($@) {
2293         warn $@;
2294         return undef;
2295     }
2296
2297     if ($res->{gw} && !$res->{ip}) {
2298         warn 'gateway specified without specifying an IP address';
2299         return undef;
2300     }
2301     if ($res->{gw6} && !$res->{ip6}) {
2302         warn 'IPv6 gateway specified without specifying an IPv6 address';
2303         return undef;
2304     }
2305     if ($res->{gw} && $res->{ip} eq 'dhcp') {
2306         warn 'gateway specified together with DHCP';
2307         return undef;
2308     }
2309     if ($res->{gw6} && $res->{ip6} !~ /^$IPV6RE/) {
2310         # gw6 + auto/dhcp
2311         warn "IPv6 gateway specified together with $res->{ip6} address";
2312         return undef;
2313     }
2314
2315     if (!$res->{ip} && !$res->{ip6}) {
2316         return { ip => 'dhcp', ip6 => 'dhcp' };
2317     }
2318
2319     return $res;
2320 }
2321
2322 sub print_net {
2323     my $net = shift;
2324
2325     return PVE::JSONSchema::print_property_string($net, $net_fmt);
2326 }
2327
2328 sub add_random_macs {
2329     my ($settings) = @_;
2330
2331     foreach my $opt (keys %$settings) {
2332         next if $opt !~ m/^net(\d+)$/;
2333         my $net = parse_net($settings->{$opt});
2334         next if !$net;
2335         $settings->{$opt} = print_net($net);
2336     }
2337 }
2338
2339 sub vm_is_volid_owner {
2340     my ($storecfg, $vmid, $volid) = @_;
2341
2342     if ($volid !~  m|^/|) {
2343         my ($path, $owner);
2344         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
2345         if ($owner && ($owner == $vmid)) {
2346             return 1;
2347         }
2348     }
2349
2350     return undef;
2351 }
2352
2353 sub split_flagged_list {
2354     my $text = shift || '';
2355     $text =~ s/[,;]/ /g;
2356     $text =~ s/^\s+//;
2357     return { map { /^(!?)(.*)$/ && ($2, $1) } ($text =~ /\S+/g) };
2358 }
2359
2360 sub join_flagged_list {
2361     my ($how, $lst) = @_;
2362     join $how, map { $lst->{$_} . $_ } keys %$lst;
2363 }
2364
2365 sub vmconfig_delete_pending_option {
2366     my ($conf, $key, $force) = @_;
2367
2368     delete $conf->{pending}->{$key};
2369     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
2370     $pending_delete_hash->{$key} = $force ? '!' : '';
2371     $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
2372 }
2373
2374 sub vmconfig_undelete_pending_option {
2375     my ($conf, $key) = @_;
2376
2377     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
2378     delete $pending_delete_hash->{$key};
2379
2380     if (%$pending_delete_hash) {
2381         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
2382     } else {
2383         delete $conf->{pending}->{delete};
2384     }
2385 }
2386
2387 sub vmconfig_register_unused_drive {
2388     my ($storecfg, $vmid, $conf, $drive) = @_;
2389
2390     if (drive_is_cloudinit($drive)) {
2391         eval { PVE::Storage::vdisk_free($storecfg, $drive->{file}) };
2392         warn $@ if $@;
2393     } elsif (!drive_is_cdrom($drive)) {
2394         my $volid = $drive->{file};
2395         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
2396             PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
2397         }
2398     }
2399 }
2400
2401 sub vmconfig_cleanup_pending {
2402     my ($conf) = @_;
2403
2404     # remove pending changes when nothing changed
2405     my $changes;
2406     foreach my $opt (keys %{$conf->{pending}}) {
2407         if (defined($conf->{$opt}) && ($conf->{pending}->{$opt} eq  $conf->{$opt})) {
2408             $changes = 1;
2409             delete $conf->{pending}->{$opt};
2410         }
2411     }
2412
2413     my $current_delete_hash = split_flagged_list($conf->{pending}->{delete});
2414     my $pending_delete_hash = {};
2415     while (my ($opt, $force) = each %$current_delete_hash) {
2416         if (defined($conf->{$opt})) {
2417             $pending_delete_hash->{$opt} = $force;
2418         } else {
2419             $changes = 1;
2420         }
2421     }
2422
2423     if (%$pending_delete_hash) {
2424         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
2425     } else {
2426         delete $conf->{pending}->{delete};
2427     }
2428
2429     return $changes;
2430 }
2431
2432 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str][,base64=bool]
2433 my $smbios1_fmt = {
2434     uuid => {
2435         type => 'string',
2436         pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
2437         format_description => 'UUID',
2438         description => "Set SMBIOS1 UUID.",
2439         optional => 1,
2440     },
2441     version => {
2442         type => 'string',
2443         pattern => '[A-Za-z0-9+\/]+={0,2}',
2444         format_description => 'Base64 encoded string',
2445         description => "Set SMBIOS1 version.",
2446         optional => 1,
2447     },
2448     serial => {
2449         type => 'string',
2450         pattern => '[A-Za-z0-9+\/]+={0,2}',
2451         format_description => 'Base64 encoded string',
2452         description => "Set SMBIOS1 serial number.",
2453         optional => 1,
2454     },
2455     manufacturer => {
2456         type => 'string',
2457         pattern => '[A-Za-z0-9+\/]+={0,2}',
2458         format_description => 'Base64 encoded string',
2459         description => "Set SMBIOS1 manufacturer.",
2460         optional => 1,
2461     },
2462     product => {
2463         type => 'string',
2464         pattern => '[A-Za-z0-9+\/]+={0,2}',
2465         format_description => 'Base64 encoded string',
2466         description => "Set SMBIOS1 product ID.",
2467         optional => 1,
2468     },
2469     sku => {
2470         type => 'string',
2471         pattern => '[A-Za-z0-9+\/]+={0,2}',
2472         format_description => 'Base64 encoded string',
2473         description => "Set SMBIOS1 SKU string.",
2474         optional => 1,
2475     },
2476     family => {
2477         type => 'string',
2478         pattern => '[A-Za-z0-9+\/]+={0,2}',
2479         format_description => 'Base64 encoded string',
2480         description => "Set SMBIOS1 family string.",
2481         optional => 1,
2482     },
2483     base64 => {
2484         type => 'boolean',
2485         description => 'Flag to indicate that the SMBIOS values are base64 encoded',
2486         optional => 1,
2487     },
2488 };
2489
2490 sub parse_smbios1 {
2491     my ($data) = @_;
2492
2493     my $res = eval { PVE::JSONSchema::parse_property_string($smbios1_fmt, $data) };
2494     warn $@ if $@;
2495     return $res;
2496 }
2497
2498 sub print_smbios1 {
2499     my ($smbios1) = @_;
2500     return PVE::JSONSchema::print_property_string($smbios1, $smbios1_fmt);
2501 }
2502
2503 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_fmt);
2504
2505 PVE::JSONSchema::register_format('pve-qm-bootdisk', \&verify_bootdisk);
2506 sub verify_bootdisk {
2507     my ($value, $noerr) = @_;
2508
2509     return $value if is_valid_drivename($value);
2510
2511     return undef if $noerr;
2512
2513     die "invalid boot disk '$value'\n";
2514 }
2515
2516 sub parse_watchdog {
2517     my ($value) = @_;
2518
2519     return undef if !$value;
2520
2521     my $res = eval { PVE::JSONSchema::parse_property_string($watchdog_fmt, $value) };
2522     warn $@ if $@;
2523     return $res;
2524 }
2525
2526 sub parse_guest_agent {
2527     my ($value) = @_;
2528
2529     return {} if !defined($value->{agent});
2530
2531     my $res = eval { PVE::JSONSchema::parse_property_string($agent_fmt, $value->{agent}) };
2532     warn $@ if $@;
2533
2534     # if the agent is disabled ignore the other potentially set properties
2535     return {} if !$res->{enabled};
2536     return $res;
2537 }
2538
2539 sub parse_vga {
2540     my ($value) = @_;
2541
2542     return {} if !$value;
2543     my $res = eval { PVE::JSONSchema::parse_property_string($vga_fmt, $value) };
2544     warn $@ if $@;
2545     return $res;
2546 }
2547
2548 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
2549 sub verify_usb_device {
2550     my ($value, $noerr) = @_;
2551
2552     return $value if parse_usb_device($value);
2553
2554     return undef if $noerr;
2555
2556     die "unable to parse usb device\n";
2557 }
2558
2559 # add JSON properties for create and set function
2560 sub json_config_properties {
2561     my $prop = shift;
2562
2563     foreach my $opt (keys %$confdesc) {
2564         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate' || $opt eq 'runningmachine';
2565         $prop->{$opt} = $confdesc->{$opt};
2566     }
2567
2568     return $prop;
2569 }
2570
2571 # return copy of $confdesc_cloudinit to generate documentation
2572 sub cloudinit_config_properties {
2573
2574     return dclone($confdesc_cloudinit);
2575 }
2576
2577 sub check_type {
2578     my ($key, $value) = @_;
2579
2580     die "unknown setting '$key'\n" if !$confdesc->{$key};
2581
2582     my $type = $confdesc->{$key}->{type};
2583
2584     if (!defined($value)) {
2585         die "got undefined value\n";
2586     }
2587
2588     if ($value =~ m/[\n\r]/) {
2589         die "property contains a line feed\n";
2590     }
2591
2592     if ($type eq 'boolean') {
2593         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
2594         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
2595         die "type check ('boolean') failed - got '$value'\n";
2596     } elsif ($type eq 'integer') {
2597         return int($1) if $value =~ m/^(\d+)$/;
2598         die "type check ('integer') failed - got '$value'\n";
2599     } elsif ($type eq 'number') {
2600         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
2601         die "type check ('number') failed - got '$value'\n";
2602     } elsif ($type eq 'string') {
2603         if (my $fmt = $confdesc->{$key}->{format}) {
2604             PVE::JSONSchema::check_format($fmt, $value);
2605             return $value;
2606         }
2607         $value =~ s/^\"(.*)\"$/$1/;
2608         return $value;
2609     } else {
2610         die "internal error"
2611     }
2612 }
2613
2614 sub touch_config {
2615     my ($vmid) = @_;
2616
2617     my $conf = PVE::QemuConfig->config_file($vmid);
2618     utime undef, undef, $conf;
2619 }
2620
2621 sub destroy_vm {
2622     my ($storecfg, $vmid, $keep_empty_config, $skiplock) = @_;
2623
2624     my $conf = PVE::QemuConfig->load_config($vmid);
2625
2626     PVE::QemuConfig->check_lock($conf) if !$skiplock;
2627
2628     if ($conf->{template}) {
2629         # check if any base image is still used by a linked clone
2630         foreach_drive($conf, sub {
2631                 my ($ds, $drive) = @_;
2632
2633                 return if drive_is_cdrom($drive);
2634
2635                 my $volid = $drive->{file};
2636
2637                 return if !$volid || $volid =~ m|^/|;
2638
2639                 die "base volume '$volid' is still in use by linked cloned\n"
2640                     if PVE::Storage::volume_is_base_and_used($storecfg, $volid);
2641
2642         });
2643     }
2644
2645     # only remove disks owned by this VM
2646     foreach_drive($conf, sub {
2647         my ($ds, $drive) = @_;
2648
2649         return if drive_is_cdrom($drive, 1);
2650
2651         my $volid = $drive->{file};
2652
2653         return if !$volid || $volid =~ m|^/|;
2654
2655         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
2656         return if !$path || !$owner || ($owner != $vmid);
2657
2658         eval {
2659             PVE::Storage::vdisk_free($storecfg, $volid);
2660         };
2661         warn "Could not remove disk '$volid', check manually: $@" if $@;
2662
2663     });
2664
2665     if ($keep_empty_config) {
2666         PVE::QemuConfig->write_config($vmid, { memory => 128 });
2667     } else {
2668         PVE::QemuConfig->destroy_config($vmid);
2669     }
2670
2671     # also remove unused disk
2672     eval {
2673         my $dl = PVE::Storage::vdisk_list($storecfg, undef, $vmid);
2674
2675         eval {
2676             PVE::Storage::foreach_volid($dl, sub {
2677                 my ($volid, $sid, $volname, $d) = @_;
2678                 PVE::Storage::vdisk_free($storecfg, $volid);
2679             });
2680         };
2681         warn $@ if $@;
2682
2683     };
2684     warn $@ if $@;
2685 }
2686
2687 sub parse_vm_config {
2688     my ($filename, $raw) = @_;
2689
2690     return undef if !defined($raw);
2691
2692     my $res = {
2693         digest => Digest::SHA::sha1_hex($raw),
2694         snapshots => {},
2695         pending => {},
2696     };
2697
2698     $filename =~ m|/qemu-server/(\d+)\.conf$|
2699         || die "got strange filename '$filename'";
2700
2701     my $vmid = $1;
2702
2703     my $conf = $res;
2704     my $descr;
2705     my $section = '';
2706
2707     my @lines = split(/\n/, $raw);
2708     foreach my $line (@lines) {
2709         next if $line =~ m/^\s*$/;
2710
2711         if ($line =~ m/^\[PENDING\]\s*$/i) {
2712             $section = 'pending';
2713             if (defined($descr)) {
2714                 $descr =~ s/\s+$//;
2715                 $conf->{description} = $descr;
2716             }
2717             $descr = undef;
2718             $conf = $res->{$section} = {};
2719             next;
2720
2721         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2722             $section = $1;
2723             if (defined($descr)) {
2724                 $descr =~ s/\s+$//;
2725                 $conf->{description} = $descr;
2726             }
2727             $descr = undef;
2728             $conf = $res->{snapshots}->{$section} = {};
2729             next;
2730         }
2731
2732         if ($line =~ m/^\#(.*)\s*$/) {
2733             $descr = '' if !defined($descr);
2734             $descr .= PVE::Tools::decode_text($1) . "\n";
2735             next;
2736         }
2737
2738         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2739             $descr = '' if !defined($descr);
2740             $descr .= PVE::Tools::decode_text($2);
2741         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2742             $conf->{snapstate} = $1;
2743         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2744             my $key = $1;
2745             my $value = $2;
2746             $conf->{$key} = $value;
2747         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2748             my $value = $1;
2749             if ($section eq 'pending') {
2750                 $conf->{delete} = $value; # we parse this later
2751             } else {
2752                 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
2753             }
2754         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(.+?)\s*$/) {
2755             my $key = $1;
2756             my $value = $2;
2757             eval { $value = check_type($key, $value); };
2758             if ($@) {
2759                 warn "vm $vmid - unable to parse value of '$key' - $@";
2760             } else {
2761                 $key = 'ide2' if $key eq 'cdrom';
2762                 my $fmt = $confdesc->{$key}->{format};
2763                 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2764                     my $v = parse_drive($key, $value);
2765                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2766                         $v->{file} = $volid;
2767                         $value = print_drive($vmid, $v);
2768                     } else {
2769                         warn "vm $vmid - unable to parse value of '$key'\n";
2770                         next;
2771                     }
2772                 }
2773
2774                 $conf->{$key} = $value;
2775             }
2776         }
2777     }
2778
2779     if (defined($descr)) {
2780         $descr =~ s/\s+$//;
2781         $conf->{description} = $descr;
2782     }
2783     delete $res->{snapstate}; # just to be sure
2784
2785     return $res;
2786 }
2787
2788 sub write_vm_config {
2789     my ($filename, $conf) = @_;
2790
2791     delete $conf->{snapstate}; # just to be sure
2792
2793     if ($conf->{cdrom}) {
2794         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2795         $conf->{ide2} = $conf->{cdrom};
2796         delete $conf->{cdrom};
2797     }
2798
2799     # we do not use 'smp' any longer
2800     if ($conf->{sockets}) {
2801         delete $conf->{smp};
2802     } elsif ($conf->{smp}) {
2803         $conf->{sockets} = $conf->{smp};
2804         delete $conf->{cores};
2805         delete $conf->{smp};
2806     }
2807
2808     my $used_volids = {};
2809
2810     my $cleanup_config = sub {
2811         my ($cref, $pending, $snapname) = @_;
2812
2813         foreach my $key (keys %$cref) {
2814             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2815                 $key eq 'snapstate' || $key eq 'pending';
2816             my $value = $cref->{$key};
2817             if ($key eq 'delete') {
2818                 die "propertry 'delete' is only allowed in [PENDING]\n"
2819                     if !$pending;
2820                 # fixme: check syntax?
2821                 next;
2822             }
2823             eval { $value = check_type($key, $value); };
2824             die "unable to parse value of '$key' - $@" if $@;
2825
2826             $cref->{$key} = $value;
2827
2828             if (!$snapname && is_valid_drivename($key)) {
2829                 my $drive = parse_drive($key, $value);
2830                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2831             }
2832         }
2833     };
2834
2835     &$cleanup_config($conf);
2836
2837     &$cleanup_config($conf->{pending}, 1);
2838
2839     foreach my $snapname (keys %{$conf->{snapshots}}) {
2840         die "internal error" if $snapname eq 'pending';
2841         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2842     }
2843
2844     # remove 'unusedX' settings if we re-add a volume
2845     foreach my $key (keys %$conf) {
2846         my $value = $conf->{$key};
2847         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2848             delete $conf->{$key};
2849         }
2850     }
2851
2852     my $generate_raw_config = sub {
2853         my ($conf, $pending) = @_;
2854
2855         my $raw = '';
2856
2857         # add description as comment to top of file
2858         if (defined(my $descr = $conf->{description})) {
2859             if ($descr) {
2860                 foreach my $cl (split(/\n/, $descr)) {
2861                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2862                 }
2863             } else {
2864                 $raw .= "#\n" if $pending;
2865             }
2866         }
2867
2868         foreach my $key (sort keys %$conf) {
2869             next if $key eq 'digest' || $key eq 'description' || $key eq 'pending' || $key eq 'snapshots';
2870             $raw .= "$key: $conf->{$key}\n";
2871         }
2872         return $raw;
2873     };
2874
2875     my $raw = &$generate_raw_config($conf);
2876
2877     if (scalar(keys %{$conf->{pending}})){
2878         $raw .= "\n[PENDING]\n";
2879         $raw .= &$generate_raw_config($conf->{pending}, 1);
2880     }
2881
2882     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2883         $raw .= "\n[$snapname]\n";
2884         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2885     }
2886
2887     return $raw;
2888 }
2889
2890 sub load_defaults {
2891
2892     my $res = {};
2893
2894     # we use static defaults from our JSON schema configuration
2895     foreach my $key (keys %$confdesc) {
2896         if (defined(my $default = $confdesc->{$key}->{default})) {
2897             $res->{$key} = $default;
2898         }
2899     }
2900
2901     return $res;
2902 }
2903
2904 sub config_list {
2905     my $vmlist = PVE::Cluster::get_vmlist();
2906     my $res = {};
2907     return $res if !$vmlist || !$vmlist->{ids};
2908     my $ids = $vmlist->{ids};
2909
2910     foreach my $vmid (keys %$ids) {
2911         my $d = $ids->{$vmid};
2912         next if !$d->{node} || $d->{node} ne $nodename;
2913         next if !$d->{type} || $d->{type} ne 'qemu';
2914         $res->{$vmid}->{exists} = 1;
2915     }
2916     return $res;
2917 }
2918
2919 # test if VM uses local resources (to prevent migration)
2920 sub check_local_resources {
2921     my ($conf, $noerr) = @_;
2922
2923     my @loc_res = ();
2924
2925     push @loc_res, "hostusb" if $conf->{hostusb}; # old syntax
2926     push @loc_res, "hostpci" if $conf->{hostpci}; # old syntax
2927
2928     push @loc_res, "ivshmem" if $conf->{ivshmem};
2929
2930     foreach my $k (keys %$conf) {
2931         next if $k =~ m/^usb/ && ($conf->{$k} =~ m/^spice(?![^,])/);
2932         # sockets are safe: they will recreated be on the target side post-migrate
2933         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2934         push @loc_res, $k if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2935     }
2936
2937     die "VM uses local resources\n" if scalar @loc_res && !$noerr;
2938
2939     return \@loc_res;
2940 }
2941
2942 # check if used storages are available on all nodes (use by migrate)
2943 sub check_storage_availability {
2944     my ($storecfg, $conf, $node) = @_;
2945
2946     foreach_drive($conf, sub {
2947         my ($ds, $drive) = @_;
2948
2949         my $volid = $drive->{file};
2950         return if !$volid;
2951
2952         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2953         return if !$sid;
2954
2955         # check if storage is available on both nodes
2956         my $scfg = PVE::Storage::storage_check_node($storecfg, $sid);
2957         PVE::Storage::storage_check_node($storecfg, $sid, $node);
2958    });
2959 }
2960
2961 # list nodes where all VM images are available (used by has_feature API)
2962 sub shared_nodes {
2963     my ($conf, $storecfg) = @_;
2964
2965     my $nodelist = PVE::Cluster::get_nodelist();
2966     my $nodehash = { map { $_ => 1 } @$nodelist };
2967     my $nodename = PVE::INotify::nodename();
2968
2969     foreach_drive($conf, sub {
2970         my ($ds, $drive) = @_;
2971
2972         my $volid = $drive->{file};
2973         return if !$volid;
2974
2975         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2976         if ($storeid) {
2977             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2978             if ($scfg->{disable}) {
2979                 $nodehash = {};
2980             } elsif (my $avail = $scfg->{nodes}) {
2981                 foreach my $node (keys %$nodehash) {
2982                     delete $nodehash->{$node} if !$avail->{$node};
2983                 }
2984             } elsif (!$scfg->{shared}) {
2985                 foreach my $node (keys %$nodehash) {
2986                     delete $nodehash->{$node} if $node ne $nodename
2987                 }
2988             }
2989         }
2990     });
2991
2992     return $nodehash
2993 }
2994
2995 sub check_local_storage_availability {
2996     my ($conf, $storecfg) = @_;
2997
2998     my $nodelist = PVE::Cluster::get_nodelist();
2999     my $nodehash = { map { $_ => {} } @$nodelist };
3000
3001     foreach_drive($conf, sub {
3002         my ($ds, $drive) = @_;
3003
3004         my $volid = $drive->{file};
3005         return if !$volid;
3006
3007         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
3008         if ($storeid) {
3009             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
3010
3011             if ($scfg->{disable}) {
3012                 foreach my $node (keys %$nodehash) {
3013                     $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
3014                 }
3015             } elsif (my $avail = $scfg->{nodes}) {
3016                 foreach my $node (keys %$nodehash) {
3017                     if (!$avail->{$node}) {
3018                         $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
3019                     }
3020                 }
3021             }
3022         }
3023     });
3024
3025     foreach my $node (values %$nodehash) {
3026         if (my $unavail = $node->{unavailable_storages}) {
3027             $node->{unavailable_storages} = [ sort keys %$unavail ];
3028         }
3029     }
3030
3031     return $nodehash
3032 }
3033
3034 sub check_cmdline {
3035     my ($pidfile, $pid) = @_;
3036
3037     my $fh = IO::File->new("/proc/$pid/cmdline", "r");
3038     if (defined($fh)) {
3039         my $line = <$fh>;
3040         $fh->close;
3041         return undef if !$line;
3042         my @param = split(/\0/, $line);
3043
3044         my $cmd = $param[0];
3045         return if !$cmd || ($cmd !~ m|kvm$| && $cmd !~ m@(?:^|/)qemu-system-[^/]+$@);
3046
3047         for (my $i = 0; $i < scalar (@param); $i++) {
3048             my $p = $param[$i];
3049             next if !$p;
3050             if (($p eq '-pidfile') || ($p eq '--pidfile')) {
3051                 my $p = $param[$i+1];
3052                 return 1 if $p && ($p eq $pidfile);
3053                 return undef;
3054             }
3055         }
3056     }
3057     return undef;
3058 }
3059
3060 sub check_running {
3061     my ($vmid, $nocheck, $node) = @_;
3062
3063     my $filename = PVE::QemuConfig->config_file($vmid, $node);
3064
3065     die "unable to find configuration file for VM $vmid - no such machine\n"
3066         if !$nocheck && ! -f $filename;
3067
3068     my $pidfile = pidfile_name($vmid);
3069
3070     if (my $fd = IO::File->new("<$pidfile")) {
3071         my $st = stat($fd);
3072         my $line = <$fd>;
3073         close($fd);
3074
3075         my $mtime = $st->mtime;
3076         if ($mtime > time()) {
3077             warn "file '$filename' modified in future\n";
3078         }
3079
3080         if ($line =~ m/^(\d+)$/) {
3081             my $pid = $1;
3082             if (check_cmdline($pidfile, $pid)) {
3083                 if (my $pinfo = PVE::ProcFSTools::check_process_running($pid)) {
3084                     return $pid;
3085                 }
3086             }
3087         }
3088     }
3089
3090     return undef;
3091 }
3092
3093 sub vzlist {
3094
3095     my $vzlist = config_list();
3096
3097     my $fd = IO::Dir->new($var_run_tmpdir) || return $vzlist;
3098
3099     while (defined(my $de = $fd->read)) {
3100         next if $de !~ m/^(\d+)\.pid$/;
3101         my $vmid = $1;
3102         next if !defined($vzlist->{$vmid});
3103         if (my $pid = check_running($vmid)) {
3104             $vzlist->{$vmid}->{pid} = $pid;
3105         }
3106     }
3107
3108     return $vzlist;
3109 }
3110
3111 sub disksize {
3112     my ($storecfg, $conf) = @_;
3113
3114     my $bootdisk = $conf->{bootdisk};
3115     return undef if !$bootdisk;
3116     return undef if !is_valid_drivename($bootdisk);
3117
3118     return undef if !$conf->{$bootdisk};
3119
3120     my $drive = parse_drive($bootdisk, $conf->{$bootdisk});
3121     return undef if !defined($drive);
3122
3123     return undef if drive_is_cdrom($drive);
3124
3125     my $volid = $drive->{file};
3126     return undef if !$volid;
3127
3128     return $drive->{size};
3129 }
3130
3131 our $vmstatus_return_properties = {
3132     vmid => get_standard_option('pve-vmid'),
3133     status => {
3134         description => "Qemu process status.",
3135         type => 'string',
3136         enum => ['stopped', 'running'],
3137     },
3138     maxmem => {
3139         description => "Maximum memory in bytes.",
3140         type => 'integer',
3141         optional => 1,
3142         renderer => 'bytes',
3143     },
3144     maxdisk => {
3145         description => "Root disk size in bytes.",
3146         type => 'integer',
3147         optional => 1,
3148         renderer => 'bytes',
3149     },
3150     name => {
3151         description => "VM name.",
3152         type => 'string',
3153         optional => 1,
3154     },
3155     qmpstatus => {
3156         description => "Qemu QMP agent status.",
3157         type => 'string',
3158         optional => 1,
3159     },
3160     pid => {
3161         description => "PID of running qemu process.",
3162         type => 'integer',
3163         optional => 1,
3164     },
3165     uptime => {
3166         description => "Uptime.",
3167         type => 'integer',
3168         optional => 1,
3169         renderer => 'duration',
3170     },
3171     cpus => {
3172         description => "Maximum usable CPUs.",
3173         type => 'number',
3174         optional => 1,
3175     },
3176     lock => {
3177         description => "The current config lock, if any.",
3178         type => 'string',
3179         optional => 1,
3180     }
3181 };
3182
3183 my $last_proc_pid_stat;
3184
3185 # get VM status information
3186 # This must be fast and should not block ($full == false)
3187 # We only query KVM using QMP if $full == true (this can be slow)
3188 sub vmstatus {
3189     my ($opt_vmid, $full) = @_;
3190
3191     my $res = {};
3192
3193     my $storecfg = PVE::Storage::config();
3194
3195     my $list = vzlist();
3196     my $defaults = load_defaults();
3197
3198     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
3199
3200     my $cpucount = $cpuinfo->{cpus} || 1;
3201
3202     foreach my $vmid (keys %$list) {
3203         next if $opt_vmid && ($vmid ne $opt_vmid);
3204
3205         my $cfspath = PVE::QemuConfig->cfs_config_path($vmid);
3206         my $conf = PVE::Cluster::cfs_read_file($cfspath) || {};
3207
3208         my $d = { vmid => $vmid };
3209         $d->{pid} = $list->{$vmid}->{pid};
3210
3211         # fixme: better status?
3212         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
3213
3214         my $size = disksize($storecfg, $conf);
3215         if (defined($size)) {
3216             $d->{disk} = 0; # no info available
3217             $d->{maxdisk} = $size;
3218         } else {
3219             $d->{disk} = 0;
3220             $d->{maxdisk} = 0;
3221         }
3222
3223         $d->{cpus} = ($conf->{sockets} || $defaults->{sockets})
3224             * ($conf->{cores} || $defaults->{cores});
3225         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
3226         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
3227
3228         $d->{name} = $conf->{name} || "VM $vmid";
3229         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024)
3230             : $defaults->{memory}*(1024*1024);
3231
3232         if ($conf->{balloon}) {
3233             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
3234             $d->{shares} = defined($conf->{shares}) ? $conf->{shares}
3235                 : $defaults->{shares};
3236         }
3237
3238         $d->{uptime} = 0;
3239         $d->{cpu} = 0;
3240         $d->{mem} = 0;
3241
3242         $d->{netout} = 0;
3243         $d->{netin} = 0;
3244
3245         $d->{diskread} = 0;
3246         $d->{diskwrite} = 0;
3247
3248         $d->{template} = PVE::QemuConfig->is_template($conf);
3249
3250         $d->{serial} = 1 if conf_has_serial($conf);
3251         $d->{lock} = $conf->{lock} if $conf->{lock};
3252
3253         $res->{$vmid} = $d;
3254     }
3255
3256     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
3257     foreach my $dev (keys %$netdev) {
3258         next if $dev !~ m/^tap([1-9]\d*)i/;
3259         my $vmid = $1;
3260         my $d = $res->{$vmid};
3261         next if !$d;
3262
3263         $d->{netout} += $netdev->{$dev}->{receive};
3264         $d->{netin} += $netdev->{$dev}->{transmit};
3265
3266         if ($full) {
3267             $d->{nics}->{$dev}->{netout} = $netdev->{$dev}->{receive};
3268             $d->{nics}->{$dev}->{netin} = $netdev->{$dev}->{transmit};
3269         }
3270
3271     }
3272
3273     my $ctime = gettimeofday;
3274
3275     foreach my $vmid (keys %$list) {
3276
3277         my $d = $res->{$vmid};
3278         my $pid = $d->{pid};
3279         next if !$pid;
3280
3281         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
3282         next if !$pstat; # not running
3283
3284         my $used = $pstat->{utime} + $pstat->{stime};
3285
3286         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
3287
3288         if ($pstat->{vsize}) {
3289             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
3290         }
3291
3292         my $old = $last_proc_pid_stat->{$pid};
3293         if (!$old) {
3294             $last_proc_pid_stat->{$pid} = {
3295                 time => $ctime,
3296                 used => $used,
3297                 cpu => 0,
3298             };
3299             next;
3300         }
3301
3302         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
3303
3304         if ($dtime > 1000) {
3305             my $dutime = $used -  $old->{used};
3306
3307             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
3308             $last_proc_pid_stat->{$pid} = {
3309                 time => $ctime,
3310                 used => $used,
3311                 cpu => $d->{cpu},
3312             };
3313         } else {
3314             $d->{cpu} = $old->{cpu};
3315         }
3316     }
3317
3318     return $res if !$full;
3319
3320     my $qmpclient = PVE::QMPClient->new();
3321
3322     my $ballooncb = sub {
3323         my ($vmid, $resp) = @_;
3324
3325         my $info = $resp->{'return'};
3326         return if !$info->{max_mem};
3327
3328         my $d = $res->{$vmid};
3329
3330         # use memory assigned to VM
3331         $d->{maxmem} = $info->{max_mem};
3332         $d->{balloon} = $info->{actual};
3333
3334         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
3335             $d->{mem} = $info->{total_mem} - $info->{free_mem};
3336             $d->{freemem} = $info->{free_mem};
3337         }
3338
3339         $d->{ballooninfo} = $info;
3340     };
3341
3342     my $blockstatscb = sub {
3343         my ($vmid, $resp) = @_;
3344         my $data = $resp->{'return'} || [];
3345         my $totalrdbytes = 0;
3346         my $totalwrbytes = 0;
3347
3348         for my $blockstat (@$data) {
3349             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
3350             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
3351
3352             $blockstat->{device} =~ s/drive-//;
3353             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
3354         }
3355         $res->{$vmid}->{diskread} = $totalrdbytes;
3356         $res->{$vmid}->{diskwrite} = $totalwrbytes;
3357     };
3358
3359     my $statuscb = sub {
3360         my ($vmid, $resp) = @_;
3361
3362         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
3363         # this fails if ballon driver is not loaded, so this must be
3364         # the last commnand (following command are aborted if this fails).
3365         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
3366
3367         my $status = 'unknown';
3368         if (!defined($status = $resp->{'return'}->{status})) {
3369             warn "unable to get VM status\n";
3370             return;
3371         }
3372
3373         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
3374     };
3375
3376     foreach my $vmid (keys %$list) {
3377         next if $opt_vmid && ($vmid ne $opt_vmid);
3378         next if !$res->{$vmid}->{pid}; # not running
3379         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
3380     }
3381
3382     $qmpclient->queue_execute(undef, 2);
3383
3384     foreach my $vmid (keys %$list) {
3385         next if $opt_vmid && ($vmid ne $opt_vmid);
3386         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
3387     }
3388
3389     return $res;
3390 }
3391
3392 sub foreach_drive {
3393     my ($conf, $func, @param) = @_;
3394
3395     foreach my $ds (valid_drive_names()) {
3396         next if !defined($conf->{$ds});
3397
3398         my $drive = parse_drive($ds, $conf->{$ds});
3399         next if !$drive;
3400
3401         &$func($ds, $drive, @param);
3402     }
3403 }
3404
3405 sub foreach_volid {
3406     my ($conf, $func, @param) = @_;
3407
3408     my $volhash = {};
3409
3410     my $test_volid = sub {
3411         my ($volid, $is_cdrom, $replicate, $shared, $snapname, $size) = @_;
3412
3413         return if !$volid;
3414
3415         $volhash->{$volid}->{cdrom} //= 1;
3416         $volhash->{$volid}->{cdrom} = 0 if !$is_cdrom;
3417
3418         $volhash->{$volid}->{replicate} //= 0;
3419         $volhash->{$volid}->{replicate} = 1 if $replicate;
3420
3421         $volhash->{$volid}->{shared} //= 0;
3422         $volhash->{$volid}->{shared} = 1 if $shared;
3423
3424         $volhash->{$volid}->{referenced_in_config} //= 0;
3425         $volhash->{$volid}->{referenced_in_config} = 1 if !defined($snapname);
3426
3427         $volhash->{$volid}->{referenced_in_snapshot}->{$snapname} = 1
3428             if defined($snapname);
3429         $volhash->{$volid}->{size} = $size if $size;
3430     };
3431
3432     foreach_drive($conf, sub {
3433         my ($ds, $drive) = @_;
3434         $test_volid->($drive->{file}, drive_is_cdrom($drive), $drive->{replicate} // 1, $drive->{shared}, undef, $drive->{size});
3435     });
3436
3437     foreach my $snapname (keys %{$conf->{snapshots}}) {
3438         my $snap = $conf->{snapshots}->{$snapname};
3439         $test_volid->($snap->{vmstate}, 0, 1, $snapname);
3440         foreach_drive($snap, sub {
3441             my ($ds, $drive) = @_;
3442             $test_volid->($drive->{file}, drive_is_cdrom($drive), $drive->{replicate} // 1, $drive->{shared}, $snapname);
3443         });
3444     }
3445
3446     foreach my $volid (keys %$volhash) {
3447         &$func($volid, $volhash->{$volid}, @param);
3448     }
3449 }
3450
3451 sub conf_has_serial {
3452     my ($conf) = @_;
3453
3454     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3455         if ($conf->{"serial$i"}) {
3456             return 1;
3457         }
3458     }
3459
3460     return 0;
3461 }
3462
3463 sub conf_has_audio {
3464     my ($conf, $id) = @_;
3465
3466     $id //= 0;
3467     my $audio = $conf->{"audio$id"};
3468     return undef if !defined($audio);
3469
3470     my $audioproperties = PVE::JSONSchema::parse_property_string($audio_fmt, $audio);
3471     my $audiodriver = $audioproperties->{driver} // 'spice';
3472
3473     return {
3474         dev => $audioproperties->{device},
3475         dev_id => "audiodev$id",
3476         backend => $audiodriver,
3477         backend_id => "$audiodriver-backend${id}",
3478     };
3479 }
3480
3481 sub vga_conf_has_spice {
3482     my ($vga) = @_;
3483
3484     my $vgaconf = parse_vga($vga);
3485     my $vgatype = $vgaconf->{type};
3486     return 0 if !$vgatype || $vgatype !~ m/^qxl([234])?$/;
3487
3488     return $1 || 1;
3489 }
3490
3491 my $host_arch; # FIXME: fix PVE::Tools::get_host_arch
3492 sub get_host_arch() {
3493     $host_arch = (POSIX::uname())[4] if !$host_arch;
3494     return $host_arch;
3495 }
3496
3497 sub is_native($) {
3498     my ($arch) = @_;
3499     return get_host_arch() eq $arch;
3500 }
3501
3502 my $default_machines = {
3503     x86_64 => 'pc',
3504     aarch64 => 'virt',
3505 };
3506
3507 sub get_basic_machine_info {
3508     my ($conf, $forcemachine) = @_;
3509
3510     my $arch = $conf->{arch} // get_host_arch();
3511     my $machine = $forcemachine || $conf->{machine} || $default_machines->{$arch};
3512     return ($arch, $machine);
3513 }
3514
3515 sub get_ovmf_files($) {
3516     my ($arch) = @_;
3517
3518     my $ovmf = $OVMF->{$arch}
3519         or die "no OVMF images known for architecture '$arch'\n";
3520
3521     return @$ovmf;
3522 }
3523
3524 my $Arch2Qemu = {
3525     aarch64 => '/usr/bin/qemu-system-aarch64',
3526     x86_64 => '/usr/bin/qemu-system-x86_64',
3527 };
3528 sub get_command_for_arch($) {
3529     my ($arch) = @_;
3530     return '/usr/bin/kvm' if is_native($arch);
3531
3532     my $cmd = $Arch2Qemu->{$arch}
3533         or die "don't know how to emulate architecture '$arch'\n";
3534     return $cmd;
3535 }
3536
3537 sub get_cpu_options {
3538     my ($conf, $arch, $kvm, $machine_type, $kvm_off, $kvmver, $winversion, $gpu_passthrough) = @_;
3539
3540     my $cpuFlags = [];
3541     my $ostype = $conf->{ostype};
3542
3543     my $cpu = $kvm ? "kvm64" : "qemu64";
3544     if ($arch eq 'aarch64') {
3545         $cpu = 'cortex-a57';
3546     }
3547     my $hv_vendor_id;
3548     if (my $cputype = $conf->{cpu}) {
3549         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
3550             or die "Cannot parse cpu description: $cputype\n";
3551         $cpu = $cpuconf->{cputype};
3552         $kvm_off = 1 if $cpuconf->{hidden};
3553         $hv_vendor_id = $cpuconf->{'hv-vendor-id'};
3554
3555         if (defined(my $flags = $cpuconf->{flags})) {
3556             push @$cpuFlags, split(";", $flags);
3557         }
3558     }
3559
3560     push @$cpuFlags , '+lahf_lm' if $cpu eq 'kvm64' && $arch eq 'x86_64';
3561
3562     push @$cpuFlags , '-x2apic'
3563         if $conf->{ostype} && $conf->{ostype} eq 'solaris';
3564
3565     push @$cpuFlags, '+sep' if $cpu eq 'kvm64' || $cpu eq 'kvm32';
3566
3567     push @$cpuFlags, '-rdtscp' if $cpu =~ m/^Opteron/;
3568
3569     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3) && $arch eq 'x86_64') {
3570
3571         push @$cpuFlags , '+kvm_pv_unhalt' if $kvm;
3572         push @$cpuFlags , '+kvm_pv_eoi' if $kvm;
3573     }
3574
3575     add_hyperv_enlightenments($cpuFlags, $winversion, $machine_type, $kvmver, $conf->{bios}, $gpu_passthrough, $hv_vendor_id) if $kvm;
3576
3577     push @$cpuFlags, 'enforce' if $cpu ne 'host' && $kvm && $arch eq 'x86_64';
3578
3579     push @$cpuFlags, 'kvm=off' if $kvm_off;
3580
3581     if (my $cpu_vendor = $cpu_vendor_list->{$cpu}) {
3582         push @$cpuFlags, "vendor=${cpu_vendor}"
3583             if $cpu_vendor ne 'default';
3584     } elsif ($arch ne 'aarch64') {
3585         die "internal error"; # should not happen
3586     }
3587
3588     $cpu .= "," . join(',', @$cpuFlags) if scalar(@$cpuFlags);
3589
3590     return ('-cpu', $cpu);
3591 }
3592
3593 sub config_to_command {
3594     my ($storecfg, $vmid, $conf, $defaults, $forcemachine) = @_;
3595
3596     my $cmd = [];
3597     my $globalFlags = [];
3598     my $machineFlags = [];
3599     my $rtcFlags = [];
3600     my $devices = [];
3601     my $pciaddr = '';
3602     my $bridges = {};
3603     my $vernum = 0; # unknown
3604     my $ostype = $conf->{ostype};
3605     my $winversion = windows_version($ostype);
3606     my $kvm = $conf->{kvm};
3607
3608     my ($arch, $machine_type) = get_basic_machine_info($conf, $forcemachine);
3609     my $kvm_binary = get_command_for_arch($arch);
3610     my $kvmver = kvm_user_version($kvm_binary);
3611     $kvm //= 1 if is_native($arch);
3612
3613     if ($kvm) {
3614         die "KVM virtualisation configured, but not available. Either disable in VM configuration or enable in BIOS.\n"
3615             if !defined kvm_version();
3616     }
3617
3618     if ($kvmver =~ m/^(\d+)\.(\d+)$/) {
3619         $vernum = $1*1000000+$2*1000;
3620     } elsif ($kvmver =~ m/^(\d+)\.(\d+)\.(\d+)$/) {
3621         $vernum = $1*1000000+$2*1000+$3;
3622     }
3623
3624     die "detected old qemu-kvm binary ($kvmver)\n" if $vernum < 15000;
3625
3626     my $q35 = machine_type_is_q35($conf);
3627     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
3628     my $use_old_bios_files = undef;
3629     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3630
3631     my $cpuunits = defined($conf->{cpuunits}) ?
3632             $conf->{cpuunits} : $defaults->{cpuunits};
3633
3634     push @$cmd, $kvm_binary;
3635
3636     push @$cmd, '-id', $vmid;
3637
3638     my $vmname = $conf->{name} || "vm$vmid";
3639
3640     push @$cmd, '-name', $vmname;
3641
3642     my $use_virtio = 0;
3643
3644     my $qmpsocket = qmp_socket($vmid);
3645     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server,nowait";
3646     push @$cmd, '-mon', "chardev=qmp,mode=control";
3647
3648     if (qemu_machine_feature_enabled($machine_type, $kvmver, 2, 12)) {
3649         push @$cmd, '-chardev', "socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5";
3650         push @$cmd, '-mon', "chardev=qmp-event,mode=control";
3651     }
3652
3653     push @$cmd, '-pidfile' , pidfile_name($vmid);
3654
3655     push @$cmd, '-daemonize';
3656
3657     if ($conf->{smbios1}) {
3658         my $smbios_conf = parse_smbios1($conf->{smbios1});
3659         if ($smbios_conf->{base64}) {
3660             # Do not pass base64 flag to qemu
3661             delete $smbios_conf->{base64};
3662             my $smbios_string = "";
3663             foreach my $key (keys %$smbios_conf) {
3664                 my $value;
3665                 if ($key eq "uuid") {
3666                     $value = $smbios_conf->{uuid}
3667                 } else {
3668                     $value = decode_base64($smbios_conf->{$key});
3669                 }
3670                 # qemu accepts any binary data, only commas need escaping by double comma
3671                 $value =~ s/,/,,/g;
3672                 $smbios_string .= "," . $key . "=" . $value if $value;
3673             }
3674             push @$cmd, '-smbios', "type=1" . $smbios_string;
3675         } else {
3676             push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
3677         }
3678     }
3679
3680     if ($conf->{vmgenid}) {
3681         push @$devices, '-device', 'vmgenid,guid='.$conf->{vmgenid};
3682     }
3683
3684     my ($ovmf_code, $ovmf_vars) = get_ovmf_files($arch);
3685     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3686         die "uefi base image not found\n" if ! -f $ovmf_code;
3687
3688         my $path;
3689         my $format;
3690         if (my $efidisk = $conf->{efidisk0}) {
3691             my $d = PVE::JSONSchema::parse_property_string($efidisk_fmt, $efidisk);
3692             my ($storeid, $volname) = PVE::Storage::parse_volume_id($d->{file}, 1);
3693             $format = $d->{format};
3694             if ($storeid) {
3695                 $path = PVE::Storage::path($storecfg, $d->{file});
3696                 if (!defined($format)) {
3697                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
3698                     $format = qemu_img_format($scfg, $volname);
3699                 }
3700             } else {
3701                 $path = $d->{file};
3702                 die "efidisk format must be specified\n"
3703                     if !defined($format);
3704             }
3705         } else {
3706             warn "no efidisk configured! Using temporary efivars disk.\n";
3707             $path = "/tmp/$vmid-ovmf.fd";
3708             PVE::Tools::file_copy($ovmf_vars, $path, -s $ovmf_vars);
3709             $format = 'raw';
3710         }
3711
3712         push @$cmd, '-drive', "if=pflash,unit=0,format=raw,readonly,file=$ovmf_code";
3713         push @$cmd, '-drive', "if=pflash,unit=1,format=$format,id=drive-efidisk0,file=$path";
3714     }
3715
3716     # load q35 config
3717     if ($q35) {
3718         # we use different pcie-port hardware for qemu >= 4.0 for passthrough
3719         if (qemu_machine_feature_enabled($machine_type, $kvmver, 4, 0)) {
3720             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35-4.0.cfg';
3721         } else {
3722             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35.cfg';
3723         }
3724     }
3725
3726     # add usb controllers
3727     my @usbcontrollers = PVE::QemuServer::USB::get_usb_controllers($conf, $bridges, $arch, $machine_type, $usbdesc->{format}, $MAX_USB_DEVICES);
3728     push @$devices, @usbcontrollers if @usbcontrollers;
3729     my $vga = parse_vga($conf->{vga});
3730
3731     my $qxlnum = vga_conf_has_spice($conf->{vga});
3732     $vga->{type} = 'qxl' if $qxlnum;
3733
3734     if (!$vga->{type}) {
3735         if ($arch eq 'aarch64') {
3736             $vga->{type} = 'virtio';
3737         } elsif (qemu_machine_feature_enabled($machine_type, $kvmver, 2, 9)) {
3738             $vga->{type} = (!$winversion || $winversion >= 6) ? 'std' : 'cirrus';
3739         } else {
3740             $vga->{type} = ($winversion >= 6) ? 'std' : 'cirrus';
3741         }
3742     }
3743
3744     # enable absolute mouse coordinates (needed by vnc)
3745     my $tablet;
3746     if (defined($conf->{tablet})) {
3747         $tablet = $conf->{tablet};
3748     } else {
3749         $tablet = $defaults->{tablet};
3750         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
3751         $tablet = 0 if $vga->{type} =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
3752     }
3753
3754     if ($tablet) {
3755         push @$devices, '-device', print_tabletdevice_full($conf, $arch) if $tablet;
3756         my $kbd = print_keyboarddevice_full($conf, $arch);
3757         push @$devices, '-device', $kbd if defined($kbd);
3758     }
3759
3760     my $kvm_off = 0;
3761     my $gpu_passthrough;
3762
3763     # host pci devices
3764     for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
3765         my $id = "hostpci$i";
3766         my $d = parse_hostpci($conf->{$id});
3767         next if !$d;
3768
3769         if (my $pcie = $d->{pcie}) {
3770             die "q35 machine model is not enabled" if !$q35;
3771             # win7 wants to have the pcie devices directly on the pcie bus
3772             # instead of in the root port
3773             if ($winversion == 7) {
3774                 $pciaddr = print_pcie_addr("${id}bus0");
3775             } else {
3776                 # add more root ports if needed, 4 are present by default
3777                 # by pve-q35 cfgs, rest added here on demand.
3778                 if ($i > 3) {
3779                     push @$devices, '-device', print_pcie_root_port($i);
3780                 }
3781                 $pciaddr = print_pcie_addr($id);
3782             }
3783         } else {
3784             $pciaddr = print_pci_addr($id, $bridges, $arch, $machine_type);
3785         }
3786
3787         my $xvga = '';
3788         if ($d->{'x-vga'}) {
3789             $xvga = ',x-vga=on' if !($conf->{bios} && $conf->{bios} eq 'ovmf');
3790             $kvm_off = 1;
3791             $vga->{type} = 'none' if !defined($conf->{vga});
3792             $gpu_passthrough = 1;
3793         }
3794
3795         my $pcidevices = $d->{pciid};
3796         my $multifunction = 1 if @$pcidevices > 1;
3797
3798         my $sysfspath;
3799         if ($d->{mdev} && scalar(@$pcidevices) == 1) {
3800             my $pci_id = $pcidevices->[0]->{id};
3801             my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $i);
3802             $sysfspath = "/sys/bus/pci/devices/0000:$pci_id/$uuid";
3803         } elsif ($d->{mdev}) {
3804             warn "ignoring mediated device '$id' with multifunction device\n";
3805         }
3806
3807         my $j=0;
3808         foreach my $pcidevice (@$pcidevices) {
3809             my $devicestr = "vfio-pci";
3810
3811             if ($sysfspath) {
3812                 $devicestr .= ",sysfsdev=$sysfspath";
3813             } else {
3814                 $devicestr .= ",host=$pcidevice->{id}";
3815             }
3816
3817             my $mf_addr = $multifunction ? ".$j" : '';
3818             $devicestr .= ",id=${id}${mf_addr}${pciaddr}${mf_addr}";
3819
3820             if ($j == 0) {
3821                 $devicestr .= ',rombar=0' if defined($d->{rombar}) && !$d->{rombar};
3822                 $devicestr .= "$xvga";
3823                 $devicestr .= ",multifunction=on" if $multifunction;
3824                 $devicestr .= ",romfile=/usr/share/kvm/$d->{romfile}" if $d->{romfile};
3825             }
3826
3827             push @$devices, '-device', $devicestr;
3828             $j++;
3829         }
3830     }
3831
3832     # usb devices
3833     my $usb_dev_features = {};
3834     $usb_dev_features->{spice_usb3} = 1 if qemu_machine_feature_enabled($machine_type, $kvmver, 4, 0);
3835
3836     my @usbdevices = PVE::QemuServer::USB::get_usb_devices($conf, $usbdesc->{format}, $MAX_USB_DEVICES, $usb_dev_features);
3837     push @$devices, @usbdevices if @usbdevices;
3838     # serial devices
3839     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3840         if (my $path = $conf->{"serial$i"}) {
3841             if ($path eq 'socket') {
3842                 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
3843                 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server,nowait";
3844                 # On aarch64, serial0 is the UART device. Qemu only allows
3845                 # connecting UART devices via the '-serial' command line, as
3846                 # the device has a fixed slot on the hardware...
3847                 if ($arch eq 'aarch64' && $i == 0) {
3848                     push @$devices, '-serial', "chardev:serial$i";
3849                 } else {
3850                     push @$devices, '-device', "isa-serial,chardev=serial$i";
3851                 }
3852             } else {
3853                 die "no such serial device\n" if ! -c $path;
3854                 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
3855                 push @$devices, '-device', "isa-serial,chardev=serial$i";
3856             }
3857         }
3858     }
3859
3860     # parallel devices
3861     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
3862         if (my $path = $conf->{"parallel$i"}) {
3863             die "no such parallel device\n" if ! -c $path;
3864             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
3865             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
3866             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
3867         }
3868     }
3869
3870     if (my $audio = conf_has_audio($conf)) {
3871
3872         my $audiopciaddr = print_pci_addr("audio0", $bridges, $arch, $machine_type);
3873
3874         my $id = $audio->{dev_id};
3875         if ($audio->{dev} eq 'AC97') {
3876             push @$devices, '-device', "AC97,id=${id}${audiopciaddr}";
3877         } elsif ($audio->{dev} =~ /intel\-hda$/) {
3878             push @$devices, '-device', "$audio->{dev},id=${id}${audiopciaddr}";
3879             push @$devices, '-device', "hda-micro,id=${id}-codec0,bus=${id}.0,cad=0";
3880             push @$devices, '-device', "hda-duplex,id=${id}-codec1,bus=${id}.0,cad=1";
3881         } else {
3882             die "unkown audio device '$audio->{dev}', implement me!";
3883         }
3884
3885         push @$devices, '-audiodev', "$audio->{backend},id=$audio->{backend_id}";
3886     }
3887
3888     my $sockets = 1;
3889     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3890     $sockets = $conf->{sockets} if  $conf->{sockets};
3891
3892     my $cores = $conf->{cores} || 1;
3893
3894     my $maxcpus = $sockets * $cores;
3895
3896     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
3897
3898     my $allowed_vcpus = $cpuinfo->{cpus};
3899
3900     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n"
3901         if ($allowed_vcpus < $maxcpus);
3902
3903     if($hotplug_features->{cpu} && qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 7)) {
3904
3905         push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3906         for (my $i = 2; $i <= $vcpus; $i++)  {
3907             my $cpustr = print_cpu_device($conf,$i);
3908             push @$cmd, '-device', $cpustr;
3909         }
3910
3911     } else {
3912
3913         push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3914     }
3915     push @$cmd, '-nodefaults';
3916
3917     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
3918
3919     my $bootindex_hash = {};
3920     my $i = 1;
3921     foreach my $o (split(//, $bootorder)) {
3922         $bootindex_hash->{$o} = $i*100;
3923         $i++;
3924     }
3925
3926     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
3927
3928     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
3929
3930     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
3931
3932     if ($vga->{type} && $vga->{type} !~ m/^serial\d+$/ && $vga->{type} ne 'none'){
3933         push @$devices, '-device', print_vga_device($conf, $vga, $arch, $machine_type, undef, $qxlnum, $bridges);
3934         my $socket = vnc_socket($vmid);
3935         push @$cmd,  '-vnc', "unix:$socket,password";
3936     } else {
3937         push @$cmd, '-vga', 'none' if $vga->{type} eq 'none';
3938         push @$cmd, '-nographic';
3939     }
3940
3941     # time drift fix
3942     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
3943
3944     my $useLocaltime = $conf->{localtime};
3945
3946     if ($winversion >= 5) { # windows
3947         $useLocaltime = 1 if !defined($conf->{localtime});
3948
3949         # use time drift fix when acpi is enabled
3950         if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
3951             $tdf = 1 if !defined($conf->{tdf});
3952         }
3953     }
3954
3955     if ($winversion >= 6) {
3956         push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3957         push @$cmd, '-no-hpet';
3958     }
3959
3960     push @$rtcFlags, 'driftfix=slew' if $tdf;
3961
3962     if (!$kvm) {
3963         push @$machineFlags, 'accel=tcg';
3964     }
3965
3966     if ($machine_type) {
3967         push @$machineFlags, "type=${machine_type}";
3968     }
3969
3970     if (($conf->{startdate}) && ($conf->{startdate} ne 'now')) {
3971         push @$rtcFlags, "base=$conf->{startdate}";
3972     } elsif ($useLocaltime) {
3973         push @$rtcFlags, 'base=localtime';
3974     }
3975
3976     push @$cmd, get_cpu_options($conf, $arch, $kvm, $machine_type, $kvm_off, $kvmver, $winversion, $gpu_passthrough);
3977
3978     PVE::QemuServer::Memory::config($conf, $vmid, $sockets, $cores, $defaults, $hotplug_features, $cmd);
3979
3980     push @$cmd, '-S' if $conf->{freeze};
3981
3982     push @$cmd, '-k', $conf->{keyboard} if defined($conf->{keyboard});
3983
3984     if (parse_guest_agent($conf)->{enabled}) {
3985         my $qgasocket = qmp_socket($vmid, 1);
3986         my $pciaddr = print_pci_addr("qga0", $bridges, $arch, $machine_type);
3987         push @$devices, '-chardev', "socket,path=$qgasocket,server,nowait,id=qga0";
3988         push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3989         push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3990     }
3991
3992     my $spice_port;
3993
3994     if ($qxlnum) {
3995         if ($qxlnum > 1) {
3996             if ($winversion){
3997                 for(my $i = 1; $i < $qxlnum; $i++){
3998                     push @$devices, '-device', print_vga_device($conf, $vga, $arch, $machine_type, $i, $qxlnum, $bridges);
3999                 }
4000             } else {
4001                 # assume other OS works like Linux
4002                 my ($ram, $vram) = ("134217728", "67108864");
4003                 if ($vga->{memory}) {
4004                     $ram = PVE::Tools::convert_size($qxlnum*4*$vga->{memory}, 'mb' => 'b');
4005                     $vram = PVE::Tools::convert_size($qxlnum*2*$vga->{memory}, 'mb' => 'b');
4006                 }
4007                 push @$cmd, '-global', "qxl-vga.ram_size=$ram";
4008                 push @$cmd, '-global', "qxl-vga.vram_size=$vram";
4009             }
4010         }
4011
4012         my $pciaddr = print_pci_addr("spice", $bridges, $arch, $machine_type);
4013
4014         my $nodename = PVE::INotify::nodename();
4015         my $pfamily = PVE::Tools::get_host_address_family($nodename);
4016         my @nodeaddrs = PVE::Tools::getaddrinfo_all('localhost', family => $pfamily);
4017         die "failed to get an ip address of type $pfamily for 'localhost'\n" if !@nodeaddrs;
4018
4019         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
4020         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
4021         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
4022
4023         my $localhost = PVE::Network::addr_to_ip($nodeaddrs[0]->{addr});
4024         $spice_port = PVE::Tools::next_spice_port($pfamily, $localhost);
4025
4026         my $spice_enhancement = PVE::JSONSchema::parse_property_string($spice_enhancements_fmt, $conf->{spice_enhancements} // '');
4027         if ($spice_enhancement->{foldersharing}) {
4028             push @$devices, '-chardev', "spiceport,id=foldershare,name=org.spice-space.webdav.0";
4029             push @$devices, '-device', "virtserialport,chardev=foldershare,name=org.spice-space.webdav.0";
4030         }
4031
4032         my $spice_opts = "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";
4033         $spice_opts .= ",streaming-video=$spice_enhancement->{videostreaming}" if $spice_enhancement->{videostreaming};
4034         push @$devices, '-spice', "$spice_opts";
4035     }
4036
4037     # enable balloon by default, unless explicitly disabled
4038     if (!defined($conf->{balloon}) || $conf->{balloon}) {
4039         $pciaddr = print_pci_addr("balloon0", $bridges, $arch, $machine_type);
4040         push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
4041     }
4042
4043     if ($conf->{watchdog}) {
4044         my $wdopts = parse_watchdog($conf->{watchdog});
4045         $pciaddr = print_pci_addr("watchdog", $bridges, $arch, $machine_type);
4046         my $watchdog = $wdopts->{model} || 'i6300esb';
4047         push @$devices, '-device', "$watchdog$pciaddr";
4048         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
4049     }
4050
4051     my $vollist = [];
4052     my $scsicontroller = {};
4053     my $ahcicontroller = {};
4054     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
4055
4056     # Add iscsi initiator name if available
4057     if (my $initiator = get_initiator_name()) {
4058         push @$devices, '-iscsi', "initiator-name=$initiator";
4059     }
4060
4061     foreach_drive($conf, sub {
4062         my ($ds, $drive) = @_;
4063
4064         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
4065             push @$vollist, $drive->{file};
4066         }
4067
4068         # ignore efidisk here, already added in bios/fw handling code above
4069         return if $drive->{interface} eq 'efidisk';
4070
4071         $use_virtio = 1 if $ds =~ m/^virtio/;
4072
4073         if (drive_is_cdrom ($drive)) {
4074             if ($bootindex_hash->{d}) {
4075                 $drive->{bootindex} = $bootindex_hash->{d};
4076                 $bootindex_hash->{d} += 1;
4077             }
4078         } else {
4079             if ($bootindex_hash->{c}) {
4080                 $drive->{bootindex} = $bootindex_hash->{c} if $conf->{bootdisk} && ($conf->{bootdisk} eq $ds);
4081                 $bootindex_hash->{c} += 1;
4082             }
4083         }
4084
4085         if($drive->{interface} eq 'virtio'){
4086            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
4087         }
4088
4089         if ($drive->{interface} eq 'scsi') {
4090
4091             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
4092
4093             $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges, $arch, $machine_type);
4094             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
4095
4096             my $iothread = '';
4097             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
4098                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
4099                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
4100             } elsif ($drive->{iothread}) {
4101                 warn "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n";
4102             }
4103
4104             my $queues = '';
4105             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
4106                 $queues = ",num_queues=$drive->{queues}";
4107             }
4108
4109             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues" if !$scsicontroller->{$controller};
4110             $scsicontroller->{$controller}=1;
4111         }
4112
4113         if ($drive->{interface} eq 'sata') {
4114            my $controller = int($drive->{index} / $MAX_SATA_DISKS);
4115            $pciaddr = print_pci_addr("ahci$controller", $bridges, $arch, $machine_type);
4116            push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr" if !$ahcicontroller->{$controller};
4117            $ahcicontroller->{$controller}=1;
4118         }
4119
4120         my $drive_cmd = print_drive_full($storecfg, $vmid, $drive);
4121         push @$devices, '-drive',$drive_cmd;
4122         push @$devices, '-device', print_drivedevice_full($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type);
4123     });
4124
4125     for (my $i = 0; $i < $MAX_NETS; $i++) {
4126          next if !$conf->{"net$i"};
4127          my $d = parse_net($conf->{"net$i"});
4128          next if !$d;
4129
4130          $use_virtio = 1 if $d->{model} eq 'virtio';
4131
4132          if ($bootindex_hash->{n}) {
4133             $d->{bootindex} = $bootindex_hash->{n};
4134             $bootindex_hash->{n} += 1;
4135          }
4136
4137          my $netdevfull = print_netdev_full($vmid, $conf, $arch, $d, "net$i");
4138          push @$devices, '-netdev', $netdevfull;
4139
4140          my $netdevicefull = print_netdevice_full($vmid, $conf, $d, "net$i", $bridges, $use_old_bios_files, $arch, $machine_type);
4141          push @$devices, '-device', $netdevicefull;
4142     }
4143
4144     if ($conf->{ivshmem}) {
4145         my $ivshmem = PVE::JSONSchema::parse_property_string($ivshmem_fmt, $conf->{ivshmem});
4146
4147         my $bus;
4148         if ($q35) {
4149             $bus = print_pcie_addr("ivshmem");
4150         } else {
4151             $bus = print_pci_addr("ivshmem", $bridges, $arch, $machine_type);
4152         }
4153
4154         my $ivshmem_name = $ivshmem->{name} // $vmid;
4155         my $path = '/dev/shm/pve-shm-' . $ivshmem_name;
4156
4157         push @$devices, '-device', "ivshmem-plain,memdev=ivshmem$bus,";
4158         push @$devices, '-object', "memory-backend-file,id=ivshmem,share=on,mem-path=$path,size=$ivshmem->{size}M";
4159     }
4160
4161     if (!$q35) {
4162         # add pci bridges
4163         if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
4164            $bridges->{1} = 1;
4165            $bridges->{2} = 1;
4166         }
4167
4168         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
4169
4170         for my $k (sort {$b cmp $a} keys %$bridges) {
4171             $pciaddr = print_pci_addr("pci.$k", undef, $arch, $machine_type);
4172             unshift @$devices, '-device', "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr" if $k > 0;
4173         }
4174     }
4175
4176     push @$cmd, @$devices;
4177     push @$cmd, '-rtc', join(',', @$rtcFlags)
4178         if scalar(@$rtcFlags);
4179     push @$cmd, '-machine', join(',', @$machineFlags)
4180         if scalar(@$machineFlags);
4181     push @$cmd, '-global', join(',', @$globalFlags)
4182         if scalar(@$globalFlags);
4183
4184     if (my $vmstate = $conf->{vmstate}) {
4185         my $statepath = PVE::Storage::path($storecfg, $vmstate);
4186         push @$vollist, $statepath;
4187         push @$cmd, '-loadstate', $statepath;
4188     }
4189
4190     # add custom args
4191     if ($conf->{args}) {
4192         my $aa = PVE::Tools::split_args($conf->{args});
4193         push @$cmd, @$aa;
4194     }
4195
4196     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
4197 }
4198
4199 sub vnc_socket {
4200     my ($vmid) = @_;
4201     return "${var_run_tmpdir}/$vmid.vnc";
4202 }
4203
4204 sub spice_port {
4205     my ($vmid) = @_;
4206
4207     my $res = vm_mon_cmd($vmid, 'query-spice');
4208
4209     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
4210 }
4211
4212 sub qmp_socket {
4213     my ($vmid, $qga, $name) = @_;
4214     my $sockettype = $qga ? 'qga' : 'qmp';
4215     my $ext = $name ? '-'.$name : '';
4216     return "${var_run_tmpdir}/$vmid$ext.$sockettype";
4217 }
4218
4219 sub pidfile_name {
4220     my ($vmid) = @_;
4221     return "${var_run_tmpdir}/$vmid.pid";
4222 }
4223
4224 sub vm_devices_list {
4225     my ($vmid) = @_;
4226
4227     my $res = vm_mon_cmd($vmid, 'query-pci');
4228     my $devices_to_check = [];
4229     my $devices = {};
4230     foreach my $pcibus (@$res) {
4231         push @$devices_to_check, @{$pcibus->{devices}},
4232     }
4233
4234     while (@$devices_to_check) {
4235         my $to_check = [];
4236         for my $d (@$devices_to_check) {
4237             $devices->{$d->{'qdev_id'}} = 1 if $d->{'qdev_id'};
4238             next if !$d->{'pci_bridge'};
4239
4240             $devices->{$d->{'qdev_id'}} += scalar(@{$d->{'pci_bridge'}->{devices}});
4241             push @$to_check, @{$d->{'pci_bridge'}->{devices}};
4242         }
4243         $devices_to_check = $to_check;
4244     }
4245
4246     my $resblock = vm_mon_cmd($vmid, 'query-block');
4247     foreach my $block (@$resblock) {
4248         if($block->{device} =~ m/^drive-(\S+)/){
4249                 $devices->{$1} = 1;
4250         }
4251     }
4252
4253     my $resmice = vm_mon_cmd($vmid, 'query-mice');
4254     foreach my $mice (@$resmice) {
4255         if ($mice->{name} eq 'QEMU HID Tablet') {
4256             $devices->{tablet} = 1;
4257             last;
4258         }
4259     }
4260
4261     # for usb devices there is no query-usb
4262     # but we can iterate over the entries in
4263     # qom-list path=/machine/peripheral
4264     my $resperipheral = vm_mon_cmd($vmid, 'qom-list', path => '/machine/peripheral');
4265     foreach my $per (@$resperipheral) {
4266         if ($per->{name} =~ m/^usb\d+$/) {
4267             $devices->{$per->{name}} = 1;
4268         }
4269     }
4270
4271     return $devices;
4272 }
4273
4274 sub vm_deviceplug {
4275     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4276
4277     my $q35 = machine_type_is_q35($conf);
4278
4279     my $devices_list = vm_devices_list($vmid);
4280     return 1 if defined($devices_list->{$deviceid});
4281
4282     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid, $arch, $machine_type); # add PCI bridge if we need it for the device
4283
4284     if ($deviceid eq 'tablet') {
4285
4286         qemu_deviceadd($vmid, print_tabletdevice_full($conf, $arch));
4287
4288     } elsif ($deviceid eq 'keyboard') {
4289
4290         qemu_deviceadd($vmid, print_keyboarddevice_full($conf, $arch));
4291
4292     } elsif ($deviceid =~ m/^usb(\d+)$/) {
4293
4294         die "usb hotplug currently not reliable\n";
4295         # since we can't reliably hot unplug all added usb devices
4296         # and usb passthrough disables live migration
4297         # we disable usb hotplugging for now
4298         qemu_deviceadd($vmid, PVE::QemuServer::USB::print_usbdevice_full($conf, $deviceid, $device));
4299
4300     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4301
4302         qemu_iothread_add($vmid, $deviceid, $device);
4303
4304         qemu_driveadd($storecfg, $vmid, $device);
4305         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, $arch, $machine_type);
4306
4307         qemu_deviceadd($vmid, $devicefull);
4308         eval { qemu_deviceaddverify($vmid, $deviceid); };
4309         if (my $err = $@) {
4310             eval { qemu_drivedel($vmid, $deviceid); };
4311             warn $@ if $@;
4312             die $err;
4313         }
4314
4315     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4316
4317
4318         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
4319         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
4320         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
4321
4322         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
4323
4324         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
4325             qemu_iothread_add($vmid, $deviceid, $device);
4326             $devicefull .= ",iothread=iothread-$deviceid";
4327         }
4328
4329         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
4330             $devicefull .= ",num_queues=$device->{queues}";
4331         }
4332
4333         qemu_deviceadd($vmid, $devicefull);
4334         qemu_deviceaddverify($vmid, $deviceid);
4335
4336     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4337
4338         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device, $arch, $machine_type);
4339         qemu_driveadd($storecfg, $vmid, $device);
4340
4341         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, $arch, $machine_type);
4342         eval { qemu_deviceadd($vmid, $devicefull); };
4343         if (my $err = $@) {
4344             eval { qemu_drivedel($vmid, $deviceid); };
4345             warn $@ if $@;
4346             die $err;
4347         }
4348
4349     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4350
4351         return undef if !qemu_netdevadd($vmid, $conf, $arch, $device, $deviceid);
4352
4353         my $machine_type = PVE::QemuServer::qemu_machine_pxe($vmid, $conf);
4354         my $use_old_bios_files = undef;
4355         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
4356
4357         my $netdevicefull = print_netdevice_full($vmid, $conf, $device, $deviceid, undef, $use_old_bios_files, $arch, $machine_type);
4358         qemu_deviceadd($vmid, $netdevicefull);
4359         eval {
4360             qemu_deviceaddverify($vmid, $deviceid);
4361             qemu_set_link_status($vmid, $deviceid, !$device->{link_down});
4362         };
4363         if (my $err = $@) {
4364             eval { qemu_netdevdel($vmid, $deviceid); };
4365             warn $@ if $@;
4366             die $err;
4367         }
4368
4369     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
4370
4371         my $bridgeid = $2;
4372         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
4373         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
4374
4375         qemu_deviceadd($vmid, $devicefull);
4376         qemu_deviceaddverify($vmid, $deviceid);
4377
4378     } else {
4379         die "can't hotplug device '$deviceid'\n";
4380     }
4381
4382     return 1;
4383 }
4384
4385 # fixme: this should raise exceptions on error!
4386 sub vm_deviceunplug {
4387     my ($vmid, $conf, $deviceid) = @_;
4388
4389     my $devices_list = vm_devices_list($vmid);
4390     return 1 if !defined($devices_list->{$deviceid});
4391
4392     die "can't unplug bootdisk" if $conf->{bootdisk} && $conf->{bootdisk} eq $deviceid;
4393
4394     if ($deviceid eq 'tablet' || $deviceid eq 'keyboard') {
4395
4396         qemu_devicedel($vmid, $deviceid);
4397
4398     } elsif ($deviceid =~ m/^usb\d+$/) {
4399
4400         die "usb hotplug currently not reliable\n";
4401         # when unplugging usb devices this way,
4402         # there may be remaining usb controllers/hubs
4403         # so we disable it for now
4404         qemu_devicedel($vmid, $deviceid);
4405         qemu_devicedelverify($vmid, $deviceid);
4406
4407     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4408
4409         qemu_devicedel($vmid, $deviceid);
4410         qemu_devicedelverify($vmid, $deviceid);
4411         qemu_drivedel($vmid, $deviceid);
4412         qemu_iothread_del($conf, $vmid, $deviceid);
4413
4414     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4415
4416         qemu_devicedel($vmid, $deviceid);
4417         qemu_devicedelverify($vmid, $deviceid);
4418         qemu_iothread_del($conf, $vmid, $deviceid);
4419
4420     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4421
4422         qemu_devicedel($vmid, $deviceid);
4423         qemu_drivedel($vmid, $deviceid);
4424         qemu_deletescsihw($conf, $vmid, $deviceid);
4425
4426     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4427
4428         qemu_devicedel($vmid, $deviceid);
4429         qemu_devicedelverify($vmid, $deviceid);
4430         qemu_netdevdel($vmid, $deviceid);
4431
4432     } else {
4433         die "can't unplug device '$deviceid'\n";
4434     }
4435
4436     return 1;
4437 }
4438
4439 sub qemu_deviceadd {
4440     my ($vmid, $devicefull) = @_;
4441
4442     $devicefull = "driver=".$devicefull;
4443     my %options =  split(/[=,]/, $devicefull);
4444
4445     vm_mon_cmd($vmid, "device_add" , %options);
4446 }
4447
4448 sub qemu_devicedel {
4449     my ($vmid, $deviceid) = @_;
4450
4451     my $ret = vm_mon_cmd($vmid, "device_del", id => $deviceid);
4452 }
4453
4454 sub qemu_iothread_add {
4455     my($vmid, $deviceid, $device) = @_;
4456
4457     if ($device->{iothread}) {
4458         my $iothreads = vm_iothreads_list($vmid);
4459         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
4460     }
4461 }
4462
4463 sub qemu_iothread_del {
4464     my($conf, $vmid, $deviceid) = @_;
4465
4466     my $confid = $deviceid;
4467     if ($deviceid =~ m/^(?:virtioscsi|scsihw)(\d+)$/) {
4468         $confid = 'scsi' . $1;
4469     }
4470     my $device = parse_drive($confid, $conf->{$confid});
4471     if ($device->{iothread}) {
4472         my $iothreads = vm_iothreads_list($vmid);
4473         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
4474     }
4475 }
4476
4477 sub qemu_objectadd {
4478     my($vmid, $objectid, $qomtype) = @_;
4479
4480     vm_mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
4481
4482     return 1;
4483 }
4484
4485 sub qemu_objectdel {
4486     my($vmid, $objectid) = @_;
4487
4488     vm_mon_cmd($vmid, "object-del", id => $objectid);
4489
4490     return 1;
4491 }
4492
4493 sub qemu_driveadd {
4494     my ($storecfg, $vmid, $device) = @_;
4495
4496     my $drive = print_drive_full($storecfg, $vmid, $device);
4497     $drive =~ s/\\/\\\\/g;
4498     my $ret = vm_human_monitor_command($vmid, "drive_add auto \"$drive\"");
4499
4500     # If the command succeeds qemu prints: "OK"
4501     return 1 if $ret =~ m/OK/s;
4502
4503     die "adding drive failed: $ret\n";
4504 }
4505
4506 sub qemu_drivedel {
4507     my($vmid, $deviceid) = @_;
4508
4509     my $ret = vm_human_monitor_command($vmid, "drive_del drive-$deviceid");
4510     $ret =~ s/^\s+//;
4511
4512     return 1 if $ret eq "";
4513
4514     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
4515     return 1 if $ret =~ m/Device \'.*?\' not found/s;
4516
4517     die "deleting drive $deviceid failed : $ret\n";
4518 }
4519
4520 sub qemu_deviceaddverify {
4521     my ($vmid, $deviceid) = @_;
4522
4523     for (my $i = 0; $i <= 5; $i++) {
4524          my $devices_list = vm_devices_list($vmid);
4525          return 1 if defined($devices_list->{$deviceid});
4526          sleep 1;
4527     }
4528
4529     die "error on hotplug device '$deviceid'\n";
4530 }
4531
4532
4533 sub qemu_devicedelverify {
4534     my ($vmid, $deviceid) = @_;
4535
4536     # need to verify that the device is correctly removed as device_del
4537     # is async and empty return is not reliable
4538
4539     for (my $i = 0; $i <= 5; $i++) {
4540          my $devices_list = vm_devices_list($vmid);
4541          return 1 if !defined($devices_list->{$deviceid});
4542          sleep 1;
4543     }
4544
4545     die "error on hot-unplugging device '$deviceid'\n";
4546 }
4547
4548 sub qemu_findorcreatescsihw {
4549     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4550
4551     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4552
4553     my $scsihwid="$controller_prefix$controller";
4554     my $devices_list = vm_devices_list($vmid);
4555
4556     if(!defined($devices_list->{$scsihwid})) {
4557         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device, $arch, $machine_type);
4558     }
4559
4560     return 1;
4561 }
4562
4563 sub qemu_deletescsihw {
4564     my ($conf, $vmid, $opt) = @_;
4565
4566     my $device = parse_drive($opt, $conf->{$opt});
4567
4568     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
4569         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
4570         return 1;
4571     }
4572
4573     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4574
4575     my $devices_list = vm_devices_list($vmid);
4576     foreach my $opt (keys %{$devices_list}) {
4577         if (PVE::QemuServer::is_valid_drivename($opt)) {
4578             my $drive = PVE::QemuServer::parse_drive($opt, $conf->{$opt});
4579             if($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
4580                 return 1;
4581             }
4582         }
4583     }
4584
4585     my $scsihwid="scsihw$controller";
4586
4587     vm_deviceunplug($vmid, $conf, $scsihwid);
4588
4589     return 1;
4590 }
4591
4592 sub qemu_add_pci_bridge {
4593     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4594
4595     my $bridges = {};
4596
4597     my $bridgeid;
4598
4599     print_pci_addr($device, $bridges, $arch, $machine_type);
4600
4601     while (my ($k, $v) = each %$bridges) {
4602         $bridgeid = $k;
4603     }
4604     return 1 if !defined($bridgeid) || $bridgeid < 1;
4605
4606     my $bridge = "pci.$bridgeid";
4607     my $devices_list = vm_devices_list($vmid);
4608
4609     if (!defined($devices_list->{$bridge})) {
4610         vm_deviceplug($storecfg, $conf, $vmid, $bridge, $arch, $machine_type);
4611     }
4612
4613     return 1;
4614 }
4615
4616 sub qemu_set_link_status {
4617     my ($vmid, $device, $up) = @_;
4618
4619     vm_mon_cmd($vmid, "set_link", name => $device,
4620                up => $up ? JSON::true : JSON::false);
4621 }
4622
4623 sub qemu_netdevadd {
4624     my ($vmid, $conf, $arch, $device, $deviceid) = @_;
4625
4626     my $netdev = print_netdev_full($vmid, $conf, $arch, $device, $deviceid, 1);
4627     my %options =  split(/[=,]/, $netdev);
4628
4629     vm_mon_cmd($vmid, "netdev_add",  %options);
4630     return 1;
4631 }
4632
4633 sub qemu_netdevdel {
4634     my ($vmid, $deviceid) = @_;
4635
4636     vm_mon_cmd($vmid, "netdev_del", id => $deviceid);
4637 }
4638
4639 sub qemu_usb_hotplug {
4640     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4641
4642     return if !$device;
4643
4644     # remove the old one first
4645     vm_deviceunplug($vmid, $conf, $deviceid);
4646
4647     # check if xhci controller is necessary and available
4648     if ($device->{usb3}) {
4649
4650         my $devicelist = vm_devices_list($vmid);
4651
4652         if (!$devicelist->{xhci}) {
4653             my $pciaddr = print_pci_addr("xhci", undef, $arch, $machine_type);
4654             qemu_deviceadd($vmid, "nec-usb-xhci,id=xhci$pciaddr");
4655         }
4656     }
4657     my $d = parse_usb_device($device->{host});
4658     $d->{usb3} = $device->{usb3};
4659
4660     # add the new one
4661     vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $d, $arch, $machine_type);
4662 }
4663
4664 sub qemu_cpu_hotplug {
4665     my ($vmid, $conf, $vcpus) = @_;
4666
4667     my $machine_type = PVE::QemuServer::get_current_qemu_machine($vmid);
4668
4669     my $sockets = 1;
4670     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
4671     $sockets = $conf->{sockets} if  $conf->{sockets};
4672     my $cores = $conf->{cores} || 1;
4673     my $maxcpus = $sockets * $cores;
4674
4675     $vcpus = $maxcpus if !$vcpus;
4676
4677     die "you can't add more vcpus than maxcpus\n"
4678         if $vcpus > $maxcpus;
4679
4680     my $currentvcpus = $conf->{vcpus} || $maxcpus;
4681
4682     if ($vcpus < $currentvcpus) {
4683
4684         if (qemu_machine_feature_enabled ($machine_type, undef, 2, 7)) {
4685
4686             for (my $i = $currentvcpus; $i > $vcpus; $i--) {
4687                 qemu_devicedel($vmid, "cpu$i");
4688                 my $retry = 0;
4689                 my $currentrunningvcpus = undef;
4690                 while (1) {
4691                     $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
4692                     last if scalar(@{$currentrunningvcpus}) == $i-1;
4693                     raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
4694                     $retry++;
4695                     sleep 1;
4696                 }
4697                 #update conf after each succesfull cpu unplug
4698                 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4699                 PVE::QemuConfig->write_config($vmid, $conf);
4700             }
4701         } else {
4702             die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
4703         }
4704
4705         return;
4706     }
4707
4708     my $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
4709     die "vcpus in running vm does not match its configuration\n"
4710         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
4711
4712     if (qemu_machine_feature_enabled ($machine_type, undef, 2, 7)) {
4713
4714         for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
4715             my $cpustr = print_cpu_device($conf, $i);
4716             qemu_deviceadd($vmid, $cpustr);
4717
4718             my $retry = 0;
4719             my $currentrunningvcpus = undef;
4720             while (1) {
4721                 $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
4722                 last if scalar(@{$currentrunningvcpus}) == $i;
4723                 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
4724                 sleep 1;
4725                 $retry++;
4726             }
4727             #update conf after each succesfull cpu hotplug
4728             $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4729             PVE::QemuConfig->write_config($vmid, $conf);
4730         }
4731     } else {
4732
4733         for (my $i = $currentvcpus; $i < $vcpus; $i++) {
4734             vm_mon_cmd($vmid, "cpu-add", id => int($i));
4735         }
4736     }
4737 }
4738
4739 sub qemu_block_set_io_throttle {
4740     my ($vmid, $deviceid,
4741         $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
4742         $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max,
4743         $bps_max_length, $bps_rd_max_length, $bps_wr_max_length,
4744         $iops_max_length, $iops_rd_max_length, $iops_wr_max_length) = @_;
4745
4746     return if !check_running($vmid) ;
4747
4748     vm_mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
4749         bps => int($bps),
4750         bps_rd => int($bps_rd),
4751         bps_wr => int($bps_wr),
4752         iops => int($iops),
4753         iops_rd => int($iops_rd),
4754         iops_wr => int($iops_wr),
4755         bps_max => int($bps_max),
4756         bps_rd_max => int($bps_rd_max),
4757         bps_wr_max => int($bps_wr_max),
4758         iops_max => int($iops_max),
4759         iops_rd_max => int($iops_rd_max),
4760         iops_wr_max => int($iops_wr_max),
4761         bps_max_length => int($bps_max_length),
4762         bps_rd_max_length => int($bps_rd_max_length),
4763         bps_wr_max_length => int($bps_wr_max_length),
4764         iops_max_length => int($iops_max_length),
4765         iops_rd_max_length => int($iops_rd_max_length),
4766         iops_wr_max_length => int($iops_wr_max_length),
4767     );
4768
4769 }
4770
4771 # old code, only used to shutdown old VM after update
4772 sub __read_avail {
4773     my ($fh, $timeout) = @_;
4774
4775     my $sel = new IO::Select;
4776     $sel->add($fh);
4777
4778     my $res = '';
4779     my $buf;
4780
4781     my @ready;
4782     while (scalar (@ready = $sel->can_read($timeout))) {
4783         my $count;
4784         if ($count = $fh->sysread($buf, 8192)) {
4785             if ($buf =~ /^(.*)\(qemu\) $/s) {
4786                 $res .= $1;
4787                 last;
4788             } else {
4789                 $res .= $buf;
4790             }
4791         } else {
4792             if (!defined($count)) {
4793                 die "$!\n";
4794             }
4795             last;
4796         }
4797     }
4798
4799     die "monitor read timeout\n" if !scalar(@ready);
4800
4801     return $res;
4802 }
4803
4804 sub qemu_block_resize {
4805     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4806
4807     my $running = check_running($vmid);
4808
4809     $size = 0 if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4810
4811     return if !$running;
4812
4813     vm_mon_cmd($vmid, "block_resize", device => $deviceid, size => int($size));
4814
4815 }
4816
4817 sub qemu_volume_snapshot {
4818     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4819
4820     my $running = check_running($vmid);
4821
4822     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4823         vm_mon_cmd($vmid, 'blockdev-snapshot-internal-sync', device => $deviceid, name => $snap);
4824     } else {
4825         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4826     }
4827 }
4828
4829 sub qemu_volume_snapshot_delete {
4830     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4831
4832     my $running = check_running($vmid);
4833
4834     if($running) {
4835
4836         $running = undef;
4837         my $conf = PVE::QemuConfig->load_config($vmid);
4838         foreach_drive($conf, sub {
4839             my ($ds, $drive) = @_;
4840             $running = 1 if $drive->{file} eq $volid;
4841         });
4842     }
4843
4844     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4845         vm_mon_cmd($vmid, 'blockdev-snapshot-delete-internal-sync', device => $deviceid, name => $snap);
4846     } else {
4847         PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4848     }
4849 }
4850
4851 sub set_migration_caps {
4852     my ($vmid) = @_;
4853
4854     my $cap_ref = [];
4855
4856     my $enabled_cap = {
4857         "auto-converge" => 1,
4858         "xbzrle" => 1,
4859         "x-rdma-pin-all" => 0,
4860         "zero-blocks" => 0,
4861         "compress" => 0
4862     };
4863
4864     my $supported_capabilities = vm_mon_cmd_nocheck($vmid, "query-migrate-capabilities");
4865
4866     for my $supported_capability (@$supported_capabilities) {
4867         push @$cap_ref, {
4868             capability => $supported_capability->{capability},
4869             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4870         };
4871     }
4872
4873     vm_mon_cmd_nocheck($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4874 }
4875
4876 my $fast_plug_option = {
4877     'lock' => 1,
4878     'name' => 1,
4879     'onboot' => 1,
4880     'shares' => 1,
4881     'startup' => 1,
4882     'description' => 1,
4883     'protection' => 1,
4884     'vmstatestorage' => 1,
4885     'hookscript' => 1,
4886 };
4887
4888 # hotplug changes in [PENDING]
4889 # $selection hash can be used to only apply specified options, for
4890 # example: { cores => 1 } (only apply changed 'cores')
4891 # $errors ref is used to return error messages
4892 sub vmconfig_hotplug_pending {
4893     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4894
4895     my $defaults = load_defaults();
4896     my ($arch, $machine_type) = get_basic_machine_info($conf, undef);
4897
4898     # commit values which do not have any impact on running VM first
4899     # Note: those option cannot raise errors, we we do not care about
4900     # $selection and always apply them.
4901
4902     my $add_error = sub {
4903         my ($opt, $msg) = @_;
4904         $errors->{$opt} = "hotplug problem - $msg";
4905     };
4906
4907     my $changes = 0;
4908     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4909         if ($fast_plug_option->{$opt}) {
4910             $conf->{$opt} = $conf->{pending}->{$opt};
4911             delete $conf->{pending}->{$opt};
4912             $changes = 1;
4913         }
4914     }
4915
4916     if ($changes) {
4917         PVE::QemuConfig->write_config($vmid, $conf);
4918         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4919     }
4920
4921     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4922
4923     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4924     while (my ($opt, $force) = each %$pending_delete_hash) {
4925         next if $selection && !$selection->{$opt};
4926         eval {
4927             if ($opt eq 'hotplug') {
4928                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
4929             } elsif ($opt eq 'tablet') {
4930                 die "skip\n" if !$hotplug_features->{usb};
4931                 if ($defaults->{tablet}) {
4932                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4933                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4934                         if $arch eq 'aarch64';
4935                 } else {
4936                     vm_deviceunplug($vmid, $conf, 'tablet');
4937                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4938                 }
4939             } elsif ($opt =~ m/^usb\d+/) {
4940                 die "skip\n";
4941                 # since we cannot reliably hot unplug usb devices
4942                 # we are disabling it
4943                 die "skip\n" if !$hotplug_features->{usb} || $conf->{$opt} =~ m/spice/i;
4944                 vm_deviceunplug($vmid, $conf, $opt);
4945             } elsif ($opt eq 'vcpus') {
4946                 die "skip\n" if !$hotplug_features->{cpu};
4947                 qemu_cpu_hotplug($vmid, $conf, undef);
4948             } elsif ($opt eq 'balloon') {
4949                 # enable balloon device is not hotpluggable
4950                 die "skip\n" if defined($conf->{balloon}) && $conf->{balloon} == 0;
4951                 # here we reset the ballooning value to memory
4952                 my $balloon = $conf->{memory} || $defaults->{memory};
4953                 vm_mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4954             } elsif ($fast_plug_option->{$opt}) {
4955                 # do nothing
4956             } elsif ($opt =~ m/^net(\d+)$/) {
4957                 die "skip\n" if !$hotplug_features->{network};
4958                 vm_deviceunplug($vmid, $conf, $opt);
4959             } elsif (is_valid_drivename($opt)) {
4960                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4961                 vm_deviceunplug($vmid, $conf, $opt);
4962                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4963             } elsif ($opt =~ m/^memory$/) {
4964                 die "skip\n" if !$hotplug_features->{memory};
4965                 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
4966             } elsif ($opt eq 'cpuunits') {
4967                 cgroups_write("cpu", $vmid, "cpu.shares", $defaults->{cpuunits});
4968             } elsif ($opt eq 'cpulimit') {
4969                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", -1);
4970             } else {
4971                 die "skip\n";
4972             }
4973         };
4974         if (my $err = $@) {
4975             &$add_error($opt, $err) if $err ne "skip\n";
4976         } else {
4977             # save new config if hotplug was successful
4978             delete $conf->{$opt};
4979             vmconfig_undelete_pending_option($conf, $opt);
4980             PVE::QemuConfig->write_config($vmid, $conf);
4981             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4982         }
4983     }
4984
4985     my $apply_pending_cloudinit;
4986     $apply_pending_cloudinit = sub {
4987         my ($key, $value) = @_;
4988         $apply_pending_cloudinit = sub {}; # once is enough
4989
4990         my @cloudinit_opts = keys %$confdesc_cloudinit;
4991         foreach my $opt (keys %{$conf->{pending}}) {
4992             next if !grep { $_ eq $opt } @cloudinit_opts;
4993             $conf->{$opt} = delete $conf->{pending}->{$opt};
4994         }
4995
4996         my $new_conf = { %$conf };
4997         $new_conf->{$key} = $value;
4998         PVE::QemuServer::Cloudinit::generate_cloudinitconfig($new_conf, $vmid);
4999     };
5000
5001     foreach my $opt (keys %{$conf->{pending}}) {
5002         next if $selection && !$selection->{$opt};
5003         my $value = $conf->{pending}->{$opt};
5004         eval {
5005             if ($opt eq 'hotplug') {
5006                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
5007             } elsif ($opt eq 'tablet') {
5008                 die "skip\n" if !$hotplug_features->{usb};
5009                 if ($value == 1) {
5010                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
5011                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
5012                         if $arch eq 'aarch64';
5013                 } elsif ($value == 0) {
5014                     vm_deviceunplug($vmid, $conf, 'tablet');
5015                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
5016                 }
5017             } elsif ($opt =~ m/^usb\d+$/) {
5018                 die "skip\n";
5019                 # since we cannot reliably hot unplug usb devices
5020                 # we are disabling it
5021                 die "skip\n" if !$hotplug_features->{usb} || $value =~ m/spice/i;
5022                 my $d = eval { PVE::JSONSchema::parse_property_string($usbdesc->{format}, $value) };
5023                 die "skip\n" if !$d;
5024                 qemu_usb_hotplug($storecfg, $conf, $vmid, $opt, $d, $arch, $machine_type);
5025             } elsif ($opt eq 'vcpus') {
5026                 die "skip\n" if !$hotplug_features->{cpu};
5027                 qemu_cpu_hotplug($vmid, $conf, $value);
5028             } elsif ($opt eq 'balloon') {
5029                 # enable/disable balloning device is not hotpluggable
5030                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
5031                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
5032                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
5033
5034                 # allow manual ballooning if shares is set to zero
5035                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
5036                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
5037                     vm_mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
5038                 }
5039             } elsif ($opt =~ m/^net(\d+)$/) {
5040                 # some changes can be done without hotplug
5041                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
5042                                     $vmid, $opt, $value, $arch, $machine_type);
5043             } elsif (is_valid_drivename($opt)) {
5044                 # some changes can be done without hotplug
5045                 my $drive = parse_drive($opt, $value);
5046                 if (drive_is_cloudinit($drive)) {
5047                     &$apply_pending_cloudinit($opt, $value);
5048                 }
5049                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
5050                                      $vmid, $opt, $value, 1, $arch, $machine_type);
5051             } elsif ($opt =~ m/^memory$/) { #dimms
5052                 die "skip\n" if !$hotplug_features->{memory};
5053                 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
5054             } elsif ($opt eq 'cpuunits') {
5055                 cgroups_write("cpu", $vmid, "cpu.shares", $conf->{pending}->{$opt});
5056             } elsif ($opt eq 'cpulimit') {
5057                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
5058                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", $cpulimit);
5059             } else {
5060                 die "skip\n";  # skip non-hot-pluggable options
5061             }
5062         };
5063         if (my $err = $@) {
5064             &$add_error($opt, $err) if $err ne "skip\n";
5065         } else {
5066             # save new config if hotplug was successful
5067             $conf->{$opt} = $value;
5068             delete $conf->{pending}->{$opt};
5069             PVE::QemuConfig->write_config($vmid, $conf);
5070             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5071         }
5072     }
5073 }
5074
5075 sub try_deallocate_drive {
5076     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
5077
5078     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
5079         my $volid = $drive->{file};
5080         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
5081             my $sid = PVE::Storage::parse_volume_id($volid);
5082             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
5083
5084             # check if the disk is really unused
5085             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
5086                 if is_volume_in_use($storecfg, $conf, $key, $volid);
5087             PVE::Storage::vdisk_free($storecfg, $volid);
5088             return 1;
5089         } else {
5090             # If vm is not owner of this disk remove from config
5091             return 1;
5092         }
5093     }
5094
5095     return undef;
5096 }
5097
5098 sub vmconfig_delete_or_detach_drive {
5099     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
5100
5101     my $drive = parse_drive($opt, $conf->{$opt});
5102
5103     my $rpcenv = PVE::RPCEnvironment::get();
5104     my $authuser = $rpcenv->get_user();
5105
5106     if ($force) {
5107         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
5108         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
5109     } else {
5110         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
5111     }
5112 }
5113
5114 sub vmconfig_apply_pending {
5115     my ($vmid, $conf, $storecfg) = @_;
5116
5117     # cold plug
5118
5119     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
5120     while (my ($opt, $force) = each %$pending_delete_hash) {
5121         die "internal error" if $opt =~ m/^unused/;
5122         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5123         if (!defined($conf->{$opt})) {
5124             vmconfig_undelete_pending_option($conf, $opt);
5125             PVE::QemuConfig->write_config($vmid, $conf);
5126         } elsif (is_valid_drivename($opt)) {
5127             vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
5128             vmconfig_undelete_pending_option($conf, $opt);
5129             delete $conf->{$opt};
5130             PVE::QemuConfig->write_config($vmid, $conf);
5131         } else {
5132             vmconfig_undelete_pending_option($conf, $opt);
5133             delete $conf->{$opt};
5134             PVE::QemuConfig->write_config($vmid, $conf);
5135         }
5136     }
5137
5138     $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5139
5140     foreach my $opt (keys %{$conf->{pending}}) { # add/change
5141         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5142
5143         if (defined($conf->{$opt}) && ($conf->{$opt} eq $conf->{pending}->{$opt})) {
5144             # skip if nothing changed
5145         } elsif (is_valid_drivename($opt)) {
5146             vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
5147                 if defined($conf->{$opt});
5148             $conf->{$opt} = $conf->{pending}->{$opt};
5149         } else {
5150             $conf->{$opt} = $conf->{pending}->{$opt};
5151         }
5152
5153         delete $conf->{pending}->{$opt};
5154         PVE::QemuConfig->write_config($vmid, $conf);
5155     }
5156 }
5157
5158 my $safe_num_ne = sub {
5159     my ($a, $b) = @_;
5160
5161     return 0 if !defined($a) && !defined($b);
5162     return 1 if !defined($a);
5163     return 1 if !defined($b);
5164
5165     return $a != $b;
5166 };
5167
5168 my $safe_string_ne = sub {
5169     my ($a, $b) = @_;
5170
5171     return 0 if !defined($a) && !defined($b);
5172     return 1 if !defined($a);
5173     return 1 if !defined($b);
5174
5175     return $a ne $b;
5176 };
5177
5178 sub vmconfig_update_net {
5179     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
5180
5181     my $newnet = parse_net($value);
5182
5183     if ($conf->{$opt}) {
5184         my $oldnet = parse_net($conf->{$opt});
5185
5186         if (&$safe_string_ne($oldnet->{model}, $newnet->{model}) ||
5187             &$safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
5188             &$safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
5189             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
5190
5191             # for non online change, we try to hot-unplug
5192             die "skip\n" if !$hotplug;
5193             vm_deviceunplug($vmid, $conf, $opt);
5194         } else {
5195
5196             die "internal error" if $opt !~ m/net(\d+)/;
5197             my $iface = "tap${vmid}i$1";
5198
5199             if (&$safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
5200                 &$safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
5201                 &$safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
5202                 &$safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
5203                 PVE::Network::tap_unplug($iface);
5204                 PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
5205             } elsif (&$safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
5206                 # Rate can be applied on its own but any change above needs to
5207                 # include the rate in tap_plug since OVS resets everything.
5208                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
5209             }
5210
5211             if (&$safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
5212                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
5213             }
5214
5215             return 1;
5216         }
5217     }
5218
5219     if ($hotplug) {
5220         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet, $arch, $machine_type);
5221     } else {
5222         die "skip\n";
5223     }
5224 }
5225
5226 sub vmconfig_update_disk {
5227     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $force, $arch, $machine_type) = @_;
5228
5229     # fixme: do we need force?
5230
5231     my $drive = parse_drive($opt, $value);
5232
5233     if ($conf->{$opt}) {
5234
5235         if (my $old_drive = parse_drive($opt, $conf->{$opt}))  {
5236
5237             my $media = $drive->{media} || 'disk';
5238             my $oldmedia = $old_drive->{media} || 'disk';
5239             die "unable to change media type\n" if $media ne $oldmedia;
5240
5241             if (!drive_is_cdrom($old_drive)) {
5242
5243                 if ($drive->{file} ne $old_drive->{file}) {
5244
5245                     die "skip\n" if !$hotplug;
5246
5247                     # unplug and register as unused
5248                     vm_deviceunplug($vmid, $conf, $opt);
5249                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
5250
5251                 } else {
5252                     # update existing disk
5253
5254                     # skip non hotpluggable value
5255                     if (&$safe_string_ne($drive->{discard}, $old_drive->{discard}) ||
5256                         &$safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
5257                         &$safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
5258                         &$safe_string_ne($drive->{cache}, $old_drive->{cache})) {
5259                         die "skip\n";
5260                     }
5261
5262                     # apply throttle
5263                     if (&$safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
5264                         &$safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
5265                         &$safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
5266                         &$safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
5267                         &$safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
5268                         &$safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
5269                         &$safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
5270                         &$safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
5271                         &$safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
5272                         &$safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
5273                         &$safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
5274                         &$safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max}) ||
5275                         &$safe_num_ne($drive->{bps_max_length}, $old_drive->{bps_max_length}) ||
5276                         &$safe_num_ne($drive->{bps_rd_max_length}, $old_drive->{bps_rd_max_length}) ||
5277                         &$safe_num_ne($drive->{bps_wr_max_length}, $old_drive->{bps_wr_max_length}) ||
5278                         &$safe_num_ne($drive->{iops_max_length}, $old_drive->{iops_max_length}) ||
5279                         &$safe_num_ne($drive->{iops_rd_max_length}, $old_drive->{iops_rd_max_length}) ||
5280                         &$safe_num_ne($drive->{iops_wr_max_length}, $old_drive->{iops_wr_max_length})) {
5281
5282                         qemu_block_set_io_throttle($vmid,"drive-$opt",
5283                                                    ($drive->{mbps} || 0)*1024*1024,
5284                                                    ($drive->{mbps_rd} || 0)*1024*1024,
5285                                                    ($drive->{mbps_wr} || 0)*1024*1024,
5286                                                    $drive->{iops} || 0,
5287                                                    $drive->{iops_rd} || 0,
5288                                                    $drive->{iops_wr} || 0,
5289                                                    ($drive->{mbps_max} || 0)*1024*1024,
5290                                                    ($drive->{mbps_rd_max} || 0)*1024*1024,
5291                                                    ($drive->{mbps_wr_max} || 0)*1024*1024,
5292                                                    $drive->{iops_max} || 0,
5293                                                    $drive->{iops_rd_max} || 0,
5294                                                    $drive->{iops_wr_max} || 0,
5295                                                    $drive->{bps_max_length} || 1,
5296                                                    $drive->{bps_rd_max_length} || 1,
5297                                                    $drive->{bps_wr_max_length} || 1,
5298                                                    $drive->{iops_max_length} || 1,
5299                                                    $drive->{iops_rd_max_length} || 1,
5300                                                    $drive->{iops_wr_max_length} || 1);
5301
5302                     }
5303
5304                     return 1;
5305                 }
5306
5307             } else { # cdrom
5308
5309                 if ($drive->{file} eq 'none') {
5310                     vm_mon_cmd($vmid, "eject",force => JSON::true,device => "drive-$opt");
5311                     if (drive_is_cloudinit($old_drive)) {
5312                         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive);
5313                     }
5314                 } else {
5315                     my $path = get_iso_path($storecfg, $vmid, $drive->{file});
5316                     vm_mon_cmd($vmid, "eject", force => JSON::true,device => "drive-$opt"); # force eject if locked
5317                     vm_mon_cmd($vmid, "change", device => "drive-$opt",target => "$path") if $path;
5318                 }
5319
5320                 return 1;
5321             }
5322         }
5323     }
5324
5325     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
5326     # hotplug new disks
5327     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
5328     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive, $arch, $machine_type);
5329 }
5330
5331 sub vm_start {
5332     my ($storecfg, $vmid, $statefile, $skiplock, $migratedfrom, $paused,
5333         $forcemachine, $spice_ticket, $migration_network, $migration_type, $targetstorage) = @_;
5334
5335     PVE::QemuConfig->lock_config($vmid, sub {
5336         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
5337
5338         die "you can't start a vm if it's a template\n" if PVE::QemuConfig->is_template($conf);
5339
5340         my $is_suspended = PVE::QemuConfig->has_lock($conf, 'suspended');
5341
5342         PVE::QemuConfig->check_lock($conf)
5343             if !($skiplock || $is_suspended);
5344
5345         die "VM $vmid already running\n" if check_running($vmid, undef, $migratedfrom);
5346
5347         # clean up leftover reboot request files
5348         eval { clear_reboot_request($vmid); };
5349         warn $@ if $@;
5350
5351         if (!$statefile && scalar(keys %{$conf->{pending}})) {
5352             vmconfig_apply_pending($vmid, $conf, $storecfg);
5353             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5354         }
5355
5356         PVE::QemuServer::Cloudinit::generate_cloudinitconfig($conf, $vmid);
5357
5358         my $defaults = load_defaults();
5359
5360         # set environment variable useful inside network script
5361         $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
5362
5363         my $local_volumes = {};
5364
5365         if ($targetstorage) {
5366             foreach_drive($conf, sub {
5367                 my ($ds, $drive) = @_;
5368
5369                 return if drive_is_cdrom($drive);
5370
5371                 my $volid = $drive->{file};
5372
5373                 return if !$volid;
5374
5375                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
5376
5377                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5378                 return if $scfg->{shared};
5379                 $local_volumes->{$ds} = [$volid, $storeid, $volname];
5380             });
5381
5382             my $format = undef;
5383
5384             foreach my $opt (sort keys %$local_volumes) {
5385
5386                 my ($volid, $storeid, $volname) = @{$local_volumes->{$opt}};
5387                 my $drive = parse_drive($opt, $conf->{$opt});
5388
5389                 #if remote storage is specified, use default format
5390                 if ($targetstorage && $targetstorage ne "1") {
5391                     $storeid = $targetstorage;
5392                     my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
5393                     $format = $defFormat;
5394                 } else {
5395                     #else we use same format than original
5396                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5397                     $format = qemu_img_format($scfg, $volid);
5398                 }
5399
5400                 my $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $format, undef, ($drive->{size}/1024));
5401                 my $newdrive = $drive;
5402                 $newdrive->{format} = $format;
5403                 $newdrive->{file} = $newvolid;
5404                 my $drivestr = PVE::QemuServer::print_drive($vmid, $newdrive);
5405                 $local_volumes->{$opt} = $drivestr;
5406                 #pass drive to conf for command line
5407                 $conf->{$opt} = $drivestr;
5408             }
5409         }
5410
5411         PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-start', 1);
5412
5413         if ($is_suspended) {
5414             # enforce machine type on suspended vm to ensure HW compatibility
5415             $forcemachine = $conf->{runningmachine};
5416             print "Resuming suspended VM\n";
5417         }
5418
5419         my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine);
5420
5421         my $migrate_uri;
5422         if ($statefile) {
5423             if ($statefile eq 'tcp') {
5424                 my $localip = "localhost";
5425                 my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
5426                 my $nodename = PVE::INotify::nodename();
5427
5428                 if (!defined($migration_type)) {
5429                     if (defined($datacenterconf->{migration}->{type})) {
5430                         $migration_type = $datacenterconf->{migration}->{type};
5431                     } else {
5432                         $migration_type = 'secure';
5433                     }
5434                 }
5435
5436                 if ($migration_type eq 'insecure') {
5437                     my $migrate_network_addr = PVE::Cluster::get_local_migration_ip($migration_network);
5438                     if ($migrate_network_addr) {
5439                         $localip = $migrate_network_addr;
5440                     } else {
5441                         $localip = PVE::Cluster::remote_node_ip($nodename, 1);
5442                     }
5443
5444                     $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5445                 }
5446
5447                 my $pfamily = PVE::Tools::get_host_address_family($nodename);
5448                 my $migrate_port = PVE::Tools::next_migrate_port($pfamily);
5449                 $migrate_uri = "tcp:${localip}:${migrate_port}";
5450                 push @$cmd, '-incoming', $migrate_uri;
5451                 push @$cmd, '-S';
5452
5453             } elsif ($statefile eq 'unix') {
5454                 # should be default for secure migrations as a ssh TCP forward
5455                 # tunnel is not deterministic reliable ready and fails regurarly
5456                 # to set up in time, so use UNIX socket forwards
5457                 my $socket_addr = "/run/qemu-server/$vmid.migrate";
5458                 unlink $socket_addr;
5459
5460                 $migrate_uri = "unix:$socket_addr";
5461
5462                 push @$cmd, '-incoming', $migrate_uri;
5463                 push @$cmd, '-S';
5464
5465             } elsif (-e $statefile) {
5466                 push @$cmd, '-loadstate', $statefile;
5467             } else {
5468                 my $statepath = PVE::Storage::path($storecfg, $statefile);
5469                 push @$vollist, $statepath;
5470                 push @$cmd, '-loadstate', $statepath;
5471             }
5472         } elsif ($paused) {
5473             push @$cmd, '-S';
5474         }
5475
5476         # host pci devices
5477         for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
5478           my $d = parse_hostpci($conf->{"hostpci$i"});
5479           next if !$d;
5480           my $pcidevices = $d->{pciid};
5481           foreach my $pcidevice (@$pcidevices) {
5482                 my $pciid = $pcidevice->{id};
5483
5484                 my $info = PVE::SysFSTools::pci_device_info("0000:$pciid");
5485                 die "IOMMU not present\n" if !PVE::SysFSTools::check_iommu_support();
5486                 die "no pci device info for device '$pciid'\n" if !$info;
5487
5488                 if ($d->{mdev}) {
5489                     my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $i);
5490                     PVE::SysFSTools::pci_create_mdev_device($pciid, $uuid, $d->{mdev});
5491                 } else {
5492                     die "can't unbind/bind pci group to vfio '$pciid'\n"
5493                         if !PVE::SysFSTools::pci_dev_group_bind_to_vfio($pciid);
5494                     die "can't reset pci device '$pciid'\n"
5495                         if $info->{has_fl_reset} and !PVE::SysFSTools::pci_dev_reset($info);
5496                 }
5497           }
5498         }
5499
5500         PVE::Storage::activate_volumes($storecfg, $vollist);
5501
5502         eval {
5503             run_command(['/bin/systemctl', 'stop', "$vmid.scope"],
5504                 outfunc => sub {}, errfunc => sub {});
5505         };
5506         # Issues with the above 'stop' not being fully completed are extremely rare, a very low
5507         # timeout should be more than enough here...
5508         PVE::Systemd::wait_for_unit_removed("$vmid.scope", 5);
5509
5510         my $cpuunits = defined($conf->{cpuunits}) ? $conf->{cpuunits}
5511                                                   : $defaults->{cpuunits};
5512
5513         my $start_timeout = ($conf->{hugepages} || $is_suspended) ? 300 : 30;
5514         my %run_params = (timeout => $statefile ? undef : $start_timeout, umask => 0077);
5515
5516         my %properties = (
5517             Slice => 'qemu.slice',
5518             KillMode => 'none',
5519             CPUShares => $cpuunits
5520         );
5521
5522         if (my $cpulimit = $conf->{cpulimit}) {
5523             $properties{CPUQuota} = int($cpulimit * 100);
5524         }
5525         $properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
5526
5527         my $run_qemu = sub {
5528             PVE::Tools::run_fork sub {
5529                 PVE::Systemd::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
5530                 run_command($cmd, %run_params);
5531             };
5532         };
5533
5534         if ($conf->{hugepages}) {
5535
5536             my $code = sub {
5537                 my $hugepages_topology = PVE::QemuServer::Memory::hugepages_topology($conf);
5538                 my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
5539
5540                 PVE::QemuServer::Memory::hugepages_mount();
5541                 PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
5542
5543                 eval { $run_qemu->() };
5544                 if (my $err = $@) {
5545                     PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology);
5546                     die $err;
5547                 }
5548
5549                 PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology);
5550             };
5551             eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
5552
5553         } else {
5554             eval { $run_qemu->() };
5555         }
5556
5557         if (my $err = $@) {
5558             # deactivate volumes if start fails
5559             eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
5560             die "start failed: $err";
5561         }
5562
5563         print "migration listens on $migrate_uri\n" if $migrate_uri;
5564
5565         if ($statefile && $statefile ne 'tcp' && $statefile ne 'unix')  {
5566             eval { vm_mon_cmd_nocheck($vmid, "cont"); };
5567             warn $@ if $@;
5568         }
5569
5570         #start nbd server for storage migration
5571         if ($targetstorage) {
5572             my $nodename = PVE::INotify::nodename();
5573             my $migrate_network_addr = PVE::Cluster::get_local_migration_ip($migration_network);
5574             my $localip = $migrate_network_addr ? $migrate_network_addr : PVE::Cluster::remote_node_ip($nodename, 1);
5575             my $pfamily = PVE::Tools::get_host_address_family($nodename);
5576             my $storage_migrate_port = PVE::Tools::next_migrate_port($pfamily);
5577
5578             vm_mon_cmd_nocheck($vmid, "nbd-server-start", addr => { type => 'inet', data => { host => "${localip}", port => "${storage_migrate_port}" } } );
5579
5580             $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5581
5582             foreach my $opt (sort keys %$local_volumes) {
5583                 my $volid = $local_volumes->{$opt};
5584                 vm_mon_cmd_nocheck($vmid, "nbd-server-add", device => "drive-$opt", writable => JSON::true );
5585                 my $migrate_storage_uri = "nbd:${localip}:${storage_migrate_port}:exportname=drive-$opt";
5586                 print "storage migration listens on $migrate_storage_uri volume:$volid\n";
5587             }
5588         }
5589
5590         if ($migratedfrom) {
5591             eval {
5592                 set_migration_caps($vmid);
5593             };
5594             warn $@ if $@;
5595
5596             if ($spice_port) {
5597                 print "spice listens on port $spice_port\n";
5598                 if ($spice_ticket) {
5599                     vm_mon_cmd_nocheck($vmid, "set_password", protocol => 'spice', password => $spice_ticket);
5600                     vm_mon_cmd_nocheck($vmid, "expire_password", protocol => 'spice', time => "+30");
5601                 }
5602             }
5603
5604         } else {
5605             vm_mon_cmd_nocheck($vmid, "balloon", value => $conf->{balloon}*1024*1024)
5606                 if !$statefile && $conf->{balloon};
5607
5608             foreach my $opt (keys %$conf) {
5609                 next if $opt !~  m/^net\d+$/;
5610                 my $nicconf = parse_net($conf->{$opt});
5611                 qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
5612             }
5613         }
5614
5615         vm_mon_cmd_nocheck($vmid, 'qom-set',
5616                     path => "machine/peripheral/balloon0",
5617                     property => "guest-stats-polling-interval",
5618                     value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
5619
5620         if ($is_suspended && (my $vmstate = $conf->{vmstate})) {
5621             print "Resumed VM, removing state\n";
5622             delete $conf->@{qw(lock vmstate runningmachine)};
5623             PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
5624             PVE::Storage::vdisk_free($storecfg, $vmstate);
5625             PVE::QemuConfig->write_config($vmid, $conf);
5626         }
5627
5628         PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'post-start');
5629     });
5630 }
5631
5632 sub vm_mon_cmd {
5633     my ($vmid, $execute, %params) = @_;
5634
5635     my $cmd = { execute => $execute, arguments => \%params };
5636     vm_qmp_command($vmid, $cmd);
5637 }
5638
5639 sub vm_mon_cmd_nocheck {
5640     my ($vmid, $execute, %params) = @_;
5641
5642     my $cmd = { execute => $execute, arguments => \%params };
5643     vm_qmp_command($vmid, $cmd, 1);
5644 }
5645
5646 sub vm_qmp_command {
5647     my ($vmid, $cmd, $nocheck) = @_;
5648
5649     my $res;
5650
5651     my $timeout;
5652     if ($cmd->{arguments}) {
5653         $timeout = delete $cmd->{arguments}->{timeout};
5654     }
5655
5656     eval {
5657         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
5658         my $sname = qmp_socket($vmid);
5659         if (-e $sname) { # test if VM is reasonambe new and supports qmp/qga
5660             my $qmpclient = PVE::QMPClient->new();
5661
5662             $res = $qmpclient->cmd($vmid, $cmd, $timeout);
5663         } else {
5664             die "unable to open monitor socket\n";
5665         }
5666     };
5667     if (my $err = $@) {
5668         syslog("err", "VM $vmid qmp command failed - $err");
5669         die $err;
5670     }
5671
5672     return $res;
5673 }
5674
5675 sub vm_human_monitor_command {
5676     my ($vmid, $cmdline) = @_;
5677
5678     my $cmd = {
5679         execute => 'human-monitor-command',
5680         arguments => { 'command-line' => $cmdline},
5681     };
5682
5683     return vm_qmp_command($vmid, $cmd);
5684 }
5685
5686 sub vm_commandline {
5687     my ($storecfg, $vmid, $snapname) = @_;
5688
5689     my $conf = PVE::QemuConfig->load_config($vmid);
5690
5691     if ($snapname) {
5692         my $snapshot = $conf->{snapshots}->{$snapname};
5693         die "snapshot '$snapname' does not exist\n" if !defined($snapshot);
5694
5695         $snapshot->{digest} = $conf->{digest}; # keep file digest for API
5696
5697         $conf = $snapshot;
5698     }
5699
5700     my $defaults = load_defaults();
5701
5702     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults);
5703
5704     return PVE::Tools::cmd2string($cmd);
5705 }
5706
5707 sub vm_reset {
5708     my ($vmid, $skiplock) = @_;
5709
5710     PVE::QemuConfig->lock_config($vmid, sub {
5711
5712         my $conf = PVE::QemuConfig->load_config($vmid);
5713
5714         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5715
5716         vm_mon_cmd($vmid, "system_reset");
5717     });
5718 }
5719
5720 sub get_vm_volumes {
5721     my ($conf) = @_;
5722
5723     my $vollist = [];
5724     foreach_volid($conf, sub {
5725         my ($volid, $attr) = @_;
5726
5727         return if $volid =~ m|^/|;
5728
5729         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5730         return if !$sid;
5731
5732         push @$vollist, $volid;
5733     });
5734
5735     return $vollist;
5736 }
5737
5738 sub vm_stop_cleanup {
5739     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
5740
5741     eval {
5742
5743         if (!$keepActive) {
5744             my $vollist = get_vm_volumes($conf);
5745             PVE::Storage::deactivate_volumes($storecfg, $vollist);
5746         }
5747
5748         foreach my $ext (qw(mon qmp pid vnc qga)) {
5749             unlink "/var/run/qemu-server/${vmid}.$ext";
5750         }
5751
5752         if ($conf->{ivshmem}) {
5753             my $ivshmem = PVE::JSONSchema::parse_property_string($ivshmem_fmt, $conf->{ivshmem});
5754             # just delete it for now, VMs which have this already open do not
5755             # are affected, but new VMs will get a separated one. If this
5756             # becomes an issue we either add some sort of ref-counting or just
5757             # add a "don't delete on stop" flag to the ivshmem format.
5758             unlink '/dev/shm/pve-shm-' . ($ivshmem->{name} // $vmid);
5759         }
5760
5761         foreach my $key (keys %$conf) {
5762             next if $key !~ m/^hostpci(\d+)$/;
5763             my $hostpciindex = $1;
5764             my $d = parse_hostpci($conf->{$key});
5765             my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $hostpciindex);
5766
5767             foreach my $pci (@{$d->{pciid}}) {
5768                 my $pciid = $pci->{id};
5769                 PVE::SysFSTools::pci_cleanup_mdev_device($pciid, $uuid);
5770             }
5771         }
5772
5773         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
5774     };
5775     warn $@ if $@; # avoid errors - just warn
5776 }
5777
5778 # call only in locked context
5779 sub _do_vm_stop {
5780     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive) = @_;
5781
5782     my $pid = check_running($vmid, $nocheck);
5783     return if !$pid;
5784
5785     my $conf;
5786     if (!$nocheck) {
5787         $conf = PVE::QemuConfig->load_config($vmid);
5788         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5789         if (!defined($timeout) && $shutdown && $conf->{startup}) {
5790             my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
5791             $timeout = $opts->{down} if $opts->{down};
5792         }
5793         PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-stop');
5794     }
5795
5796     eval {
5797         if ($shutdown) {
5798             if (defined($conf) && parse_guest_agent($conf)->{enabled}) {
5799                 vm_qmp_command($vmid, {
5800                         execute => "guest-shutdown",
5801                         arguments => { timeout => $timeout }
5802                     }, $nocheck);
5803             } else {
5804                 vm_qmp_command($vmid, { execute => "system_powerdown" }, $nocheck);
5805             }
5806         } else {
5807             vm_qmp_command($vmid, { execute => "quit" }, $nocheck);
5808         }
5809     };
5810     my $err = $@;
5811
5812     if (!$err) {
5813         $timeout = 60 if !defined($timeout);
5814
5815         my $count = 0;
5816         while (($count < $timeout) && check_running($vmid, $nocheck)) {
5817             $count++;
5818             sleep 1;
5819         }
5820
5821         if ($count >= $timeout) {
5822             if ($force) {
5823                 warn "VM still running - terminating now with SIGTERM\n";
5824                 kill 15, $pid;
5825             } else {
5826                 die "VM quit/powerdown failed - got timeout\n";
5827             }
5828         } else {
5829             vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5830             return;
5831         }
5832     } else {
5833         if ($force) {
5834             warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
5835             kill 15, $pid;
5836         } else {
5837             die "VM quit/powerdown failed\n";
5838         }
5839     }
5840
5841     # wait again
5842     $timeout = 10;
5843
5844     my $count = 0;
5845     while (($count < $timeout) && check_running($vmid, $nocheck)) {
5846         $count++;
5847         sleep 1;
5848     }
5849
5850     if ($count >= $timeout) {
5851         warn "VM still running - terminating now with SIGKILL\n";
5852         kill 9, $pid;
5853         sleep 1;
5854     }
5855
5856     vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5857 }
5858
5859 # Note: use $nocheck to skip tests if VM configuration file exists.
5860 # We need that when migration VMs to other nodes (files already moved)
5861 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
5862 sub vm_stop {
5863     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
5864
5865     $force = 1 if !defined($force) && !$shutdown;
5866
5867     if ($migratedfrom){
5868         my $pid = check_running($vmid, $nocheck, $migratedfrom);
5869         kill 15, $pid if $pid;
5870         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
5871         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
5872         return;
5873     }
5874
5875     PVE::QemuConfig->lock_config($vmid, sub {
5876         _do_vm_stop($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive);
5877    });
5878 }
5879
5880 sub vm_reboot {
5881     my ($vmid, $timeout) = @_;
5882
5883     PVE::QemuConfig->lock_config($vmid, sub {
5884
5885         # only reboot if running, as qmeventd starts it again on a stop event
5886         return if !check_running($vmid);
5887
5888         create_reboot_request($vmid);
5889
5890         my $storecfg = PVE::Storage::config();
5891         _do_vm_stop($storecfg, $vmid, undef, undef, $timeout, 1);
5892
5893    });
5894 }
5895
5896 sub vm_suspend {
5897     my ($vmid, $skiplock, $includestate, $statestorage) = @_;
5898
5899     my $conf;
5900     my $path;
5901     my $storecfg;
5902     my $vmstate;
5903
5904     PVE::QemuConfig->lock_config($vmid, sub {
5905
5906         $conf = PVE::QemuConfig->load_config($vmid);
5907
5908         my $is_backing_up = PVE::QemuConfig->has_lock($conf, 'backup');
5909         PVE::QemuConfig->check_lock($conf)
5910             if !($skiplock || $is_backing_up);
5911
5912         die "cannot suspend to disk during backup\n"
5913             if $is_backing_up && $includestate;
5914
5915         if ($includestate) {
5916             $conf->{lock} = 'suspending';
5917             my $date = strftime("%Y-%m-%d", localtime(time()));
5918             $storecfg = PVE::Storage::config();
5919             $vmstate = PVE::QemuConfig->__snapshot_save_vmstate($vmid, $conf, "suspend-$date", $storecfg, $statestorage, 1);
5920             $path = PVE::Storage::path($storecfg, $vmstate);
5921             PVE::QemuConfig->write_config($vmid, $conf);
5922         } else {
5923             vm_mon_cmd($vmid, "stop");
5924         }
5925     });
5926
5927     if ($includestate) {
5928         # save vm state
5929         PVE::Storage::activate_volumes($storecfg, [$vmstate]);
5930
5931         eval {
5932             vm_mon_cmd($vmid, "savevm-start", statefile => $path);
5933             for(;;) {
5934                 my $state = vm_mon_cmd_nocheck($vmid, "query-savevm");
5935                 if (!$state->{status}) {
5936                     die "savevm not active\n";
5937                 } elsif ($state->{status} eq 'active') {
5938                     sleep(1);
5939                     next;
5940                 } elsif ($state->{status} eq 'completed') {
5941                     print "State saved, quitting\n";
5942                     last;
5943                 } elsif ($state->{status} eq 'failed' && $state->{error}) {
5944                     die "query-savevm failed with error '$state->{error}'\n"
5945                 } else {
5946                     die "query-savevm returned status '$state->{status}'\n";
5947                 }
5948             }
5949         };
5950         my $err = $@;
5951
5952         PVE::QemuConfig->lock_config($vmid, sub {
5953             $conf = PVE::QemuConfig->load_config($vmid);
5954             if ($err) {
5955                 # cleanup, but leave suspending lock, to indicate something went wrong
5956                 eval {
5957                     vm_mon_cmd($vmid, "savevm-end");
5958                     PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
5959                     PVE::Storage::vdisk_free($storecfg, $vmstate);
5960                     delete $conf->@{qw(vmstate runningmachine)};
5961                     PVE::QemuConfig->write_config($vmid, $conf);
5962                 };
5963                 warn $@ if $@;
5964                 die $err;
5965             }
5966
5967             die "lock changed unexpectedly\n"
5968                 if !PVE::QemuConfig->has_lock($conf, 'suspending');
5969
5970             vm_qmp_command($vmid, { execute => "quit" });
5971             $conf->{lock} = 'suspended';
5972             PVE::QemuConfig->write_config($vmid, $conf);
5973         });
5974     }
5975 }
5976
5977 sub vm_resume {
5978     my ($vmid, $skiplock, $nocheck) = @_;
5979
5980     PVE::QemuConfig->lock_config($vmid, sub {
5981         my $vm_mon_cmd = $nocheck ? \&vm_mon_cmd_nocheck : \&vm_mon_cmd;
5982         my $res = $vm_mon_cmd->($vmid, 'query-status');
5983         my $resume_cmd = 'cont';
5984
5985         if ($res->{status} && $res->{status} eq 'suspended') {
5986             $resume_cmd = 'system_wakeup';
5987         }
5988
5989         if (!$nocheck) {
5990
5991             my $conf = PVE::QemuConfig->load_config($vmid);
5992
5993             PVE::QemuConfig->check_lock($conf)
5994                 if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
5995         }
5996
5997         $vm_mon_cmd->($vmid, $resume_cmd);
5998     });
5999 }
6000
6001 sub vm_sendkey {
6002     my ($vmid, $skiplock, $key) = @_;
6003
6004     PVE::QemuConfig->lock_config($vmid, sub {
6005
6006         my $conf = PVE::QemuConfig->load_config($vmid);
6007
6008         # there is no qmp command, so we use the human monitor command
6009         my $res = vm_human_monitor_command($vmid, "sendkey $key");
6010         die $res if $res ne '';
6011     });
6012 }
6013
6014 sub vm_destroy {
6015     my ($storecfg, $vmid, $skiplock) = @_;
6016
6017     PVE::QemuConfig->lock_config($vmid, sub {
6018
6019         my $conf = PVE::QemuConfig->load_config($vmid);
6020
6021         if (!check_running($vmid)) {
6022             destroy_vm($storecfg, $vmid, undef, $skiplock);
6023         } else {
6024             die "VM $vmid is running - destroy failed\n";
6025         }
6026     });
6027 }
6028
6029 # vzdump restore implementaion
6030
6031 sub tar_archive_read_firstfile {
6032     my $archive = shift;
6033
6034     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
6035
6036     # try to detect archive type first
6037     my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
6038         die "unable to open file '$archive'\n";
6039     my $firstfile = <$fh>;
6040     kill 15, $pid;
6041     close $fh;
6042
6043     die "ERROR: archive contaions no data\n" if !$firstfile;
6044     chomp $firstfile;
6045
6046     return $firstfile;
6047 }
6048
6049 sub tar_restore_cleanup {
6050     my ($storecfg, $statfile) = @_;
6051
6052     print STDERR "starting cleanup\n";
6053
6054     if (my $fd = IO::File->new($statfile, "r")) {
6055         while (defined(my $line = <$fd>)) {
6056             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
6057                 my $volid = $2;
6058                 eval {
6059                     if ($volid =~ m|^/|) {
6060                         unlink $volid || die 'unlink failed\n';
6061                     } else {
6062                         PVE::Storage::vdisk_free($storecfg, $volid);
6063                     }
6064                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
6065                 };
6066                 print STDERR "unable to cleanup '$volid' - $@" if $@;
6067             } else {
6068                 print STDERR "unable to parse line in statfile - $line";
6069             }
6070         }
6071         $fd->close();
6072     }
6073 }
6074
6075 sub restore_archive {
6076     my ($archive, $vmid, $user, $opts) = @_;
6077
6078     my $format = $opts->{format};
6079     my $comp;
6080
6081     if ($archive =~ m/\.tgz$/ || $archive =~ m/\.tar\.gz$/) {
6082         $format = 'tar' if !$format;
6083         $comp = 'gzip';
6084     } elsif ($archive =~ m/\.tar$/) {
6085         $format = 'tar' if !$format;
6086     } elsif ($archive =~ m/.tar.lzo$/) {
6087         $format = 'tar' if !$format;
6088         $comp = 'lzop';
6089     } elsif ($archive =~ m/\.vma$/) {
6090         $format = 'vma' if !$format;
6091     } elsif ($archive =~ m/\.vma\.gz$/) {
6092         $format = 'vma' if !$format;
6093         $comp = 'gzip';
6094     } elsif ($archive =~ m/\.vma\.lzo$/) {
6095         $format = 'vma' if !$format;
6096         $comp = 'lzop';
6097     } else {
6098         $format = 'vma' if !$format; # default
6099     }
6100
6101     # try to detect archive format
6102     if ($format eq 'tar') {
6103         return restore_tar_archive($archive, $vmid, $user, $opts);
6104     } else {
6105         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
6106     }
6107 }
6108
6109 sub restore_update_config_line {
6110     my ($outfd, $cookie, $vmid, $map, $line, $unique) = @_;
6111
6112     return if $line =~ m/^\#qmdump\#/;
6113     return if $line =~ m/^\#vzdump\#/;
6114     return if $line =~ m/^lock:/;
6115     return if $line =~ m/^unused\d+:/;
6116     return if $line =~ m/^parent:/;
6117
6118     my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
6119     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
6120         # try to convert old 1.X settings
6121         my ($id, $ind, $ethcfg) = ($1, $2, $3);
6122         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
6123             my ($model, $macaddr) = split(/\=/, $devconfig);
6124             $macaddr = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if !$macaddr || $unique;
6125             my $net = {
6126                 model => $model,
6127                 bridge => "vmbr$ind",
6128                 macaddr => $macaddr,
6129             };
6130             my $netstr = print_net($net);
6131
6132             print $outfd "net$cookie->{netcount}: $netstr\n";
6133             $cookie->{netcount}++;
6134         }
6135     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
6136         my ($id, $netstr) = ($1, $2);
6137         my $net = parse_net($netstr);
6138         $net->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if $net->{macaddr};
6139         $netstr = print_net($net);
6140         print $outfd "$id: $netstr\n";
6141     } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk)\d+):\s*(\S+)\s*$/) {
6142         my $virtdev = $1;
6143         my $value = $3;
6144         my $di = parse_drive($virtdev, $value);
6145         if (defined($di->{backup}) && !$di->{backup}) {
6146             print $outfd "#$line";
6147         } elsif ($map->{$virtdev}) {
6148             delete $di->{format}; # format can change on restore
6149             $di->{file} = $map->{$virtdev};
6150             $value = print_drive($vmid, $di);
6151             print $outfd "$virtdev: $value\n";
6152         } else {
6153             print $outfd $line;
6154         }
6155     } elsif (($line =~ m/^vmgenid: (.*)/)) {
6156         my $vmgenid = $1;
6157         if ($vmgenid ne '0') {
6158             # always generate a new vmgenid if there was a valid one setup
6159             $vmgenid = generate_uuid();
6160         }
6161         print $outfd "vmgenid: $vmgenid\n";
6162     } elsif (($line =~ m/^(smbios1: )(.*)/) && $unique) {
6163         my ($uuid, $uuid_str);
6164         UUID::generate($uuid);
6165         UUID::unparse($uuid, $uuid_str);
6166         my $smbios1 = parse_smbios1($2);
6167         $smbios1->{uuid} = $uuid_str;
6168         print $outfd $1.print_smbios1($smbios1)."\n";
6169     } else {
6170         print $outfd $line;
6171     }
6172 }
6173
6174 sub scan_volids {
6175     my ($cfg, $vmid) = @_;
6176
6177     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid);
6178
6179     my $volid_hash = {};
6180     foreach my $storeid (keys %$info) {
6181         foreach my $item (@{$info->{$storeid}}) {
6182             next if !($item->{volid} && $item->{size});
6183             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
6184             $volid_hash->{$item->{volid}} = $item;
6185         }
6186     }
6187
6188     return $volid_hash;
6189 }
6190
6191 sub is_volume_in_use {
6192     my ($storecfg, $conf, $skip_drive, $volid) = @_;
6193
6194     my $path = PVE::Storage::path($storecfg, $volid);
6195
6196     my $scan_config = sub {
6197         my ($cref, $snapname) = @_;
6198
6199         foreach my $key (keys %$cref) {
6200             my $value = $cref->{$key};
6201             if (is_valid_drivename($key)) {
6202                 next if $skip_drive && $key eq $skip_drive;
6203                 my $drive = parse_drive($key, $value);
6204                 next if !$drive || !$drive->{file} || drive_is_cdrom($drive);
6205                 return 1 if $volid eq $drive->{file};
6206                 if ($drive->{file} =~ m!^/!) {
6207                     return 1 if $drive->{file} eq $path;
6208                 } else {
6209                     my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file}, 1);
6210                     next if !$storeid;
6211                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid, 1);
6212                     next if !$scfg;
6213                     return 1 if $path eq PVE::Storage::path($storecfg, $drive->{file}, $snapname);
6214                 }
6215             }
6216         }
6217
6218         return 0;
6219     };
6220
6221     return 1 if &$scan_config($conf);
6222
6223     undef $skip_drive;
6224
6225     foreach my $snapname (keys %{$conf->{snapshots}}) {
6226         return 1 if &$scan_config($conf->{snapshots}->{$snapname}, $snapname);
6227     }
6228
6229     return 0;
6230 }
6231
6232 sub update_disksize {
6233     my ($vmid, $conf, $volid_hash) = @_;
6234
6235     my $changes;
6236     my $prefix = "VM $vmid:";
6237
6238     # used and unused disks
6239     my $referenced = {};
6240
6241     # Note: it is allowed to define multiple storages with same path (alias), so
6242     # we need to check both 'volid' and real 'path' (two different volid can point
6243     # to the same path).
6244
6245     my $referencedpath = {};
6246
6247     # update size info
6248     foreach my $opt (keys %$conf) {
6249         if (is_valid_drivename($opt)) {
6250             my $drive = parse_drive($opt, $conf->{$opt});
6251             my $volid = $drive->{file};
6252             next if !$volid;
6253
6254             $referenced->{$volid} = 1;
6255             if ($volid_hash->{$volid} &&
6256                 (my $path = $volid_hash->{$volid}->{path})) {
6257                 $referencedpath->{$path} = 1;
6258             }
6259
6260             next if drive_is_cdrom($drive);
6261             next if !$volid_hash->{$volid};
6262
6263             $drive->{size} = $volid_hash->{$volid}->{size};
6264             my $new = print_drive($vmid, $drive);
6265             if ($new ne $conf->{$opt}) {
6266                 $changes = 1;
6267                 $conf->{$opt} = $new;
6268                 print "$prefix update disk '$opt' information.\n";
6269             }
6270         }
6271     }
6272
6273     # remove 'unusedX' entry if volume is used
6274     foreach my $opt (keys %$conf) {
6275         next if $opt !~ m/^unused\d+$/;
6276         my $volid = $conf->{$opt};
6277         my $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
6278         if ($referenced->{$volid} || ($path && $referencedpath->{$path})) {
6279             print "$prefix remove entry '$opt', its volume '$volid' is in use.\n";
6280             $changes = 1;
6281             delete $conf->{$opt};
6282         }
6283
6284         $referenced->{$volid} = 1;
6285         $referencedpath->{$path} = 1 if $path;
6286     }
6287
6288     foreach my $volid (sort keys %$volid_hash) {
6289         next if $volid =~ m/vm-$vmid-state-/;
6290         next if $referenced->{$volid};
6291         my $path = $volid_hash->{$volid}->{path};
6292         next if !$path; # just to be sure
6293         next if $referencedpath->{$path};
6294         $changes = 1;
6295         my $key = PVE::QemuConfig->add_unused_volume($conf, $volid);
6296         print "$prefix add unreferenced volume '$volid' as '$key' to config.\n";
6297         $referencedpath->{$path} = 1; # avoid to add more than once (aliases)
6298     }
6299
6300     return $changes;
6301 }
6302
6303 sub rescan {
6304     my ($vmid, $nolock, $dryrun) = @_;
6305
6306     my $cfg = PVE::Storage::config();
6307
6308     # FIXME: Remove once our RBD plugin can handle CT and VM on a single storage
6309     # see: https://pve.proxmox.com/pipermail/pve-devel/2018-July/032900.html
6310     foreach my $stor (keys %{$cfg->{ids}}) {
6311         delete($cfg->{ids}->{$stor}) if ! $cfg->{ids}->{$stor}->{content}->{images};
6312     }
6313
6314     print "rescan volumes...\n";
6315     my $volid_hash = scan_volids($cfg, $vmid);
6316
6317     my $updatefn =  sub {
6318         my ($vmid) = @_;
6319
6320         my $conf = PVE::QemuConfig->load_config($vmid);
6321
6322         PVE::QemuConfig->check_lock($conf);
6323
6324         my $vm_volids = {};
6325         foreach my $volid (keys %$volid_hash) {
6326             my $info = $volid_hash->{$volid};
6327             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
6328         }
6329
6330         my $changes = update_disksize($vmid, $conf, $vm_volids);
6331
6332         PVE::QemuConfig->write_config($vmid, $conf) if $changes && !$dryrun;
6333     };
6334
6335     if (defined($vmid)) {
6336         if ($nolock) {
6337             &$updatefn($vmid);
6338         } else {
6339             PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6340         }
6341     } else {
6342         my $vmlist = config_list();
6343         foreach my $vmid (keys %$vmlist) {
6344             if ($nolock) {
6345                 &$updatefn($vmid);
6346             } else {
6347                 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6348             }
6349         }
6350     }
6351 }
6352
6353 sub restore_vma_archive {
6354     my ($archive, $vmid, $user, $opts, $comp) = @_;
6355
6356     my $readfrom = $archive;
6357
6358     my $cfg = PVE::Storage::config();
6359     my $commands = [];
6360     my $bwlimit = $opts->{bwlimit};
6361
6362     my $dbg_cmdstring = '';
6363     my $add_pipe = sub {
6364         my ($cmd) = @_;
6365         push @$commands, $cmd;
6366         $dbg_cmdstring .= ' | ' if length($dbg_cmdstring);
6367         $dbg_cmdstring .= PVE::Tools::cmd2string($cmd);
6368         $readfrom = '-';
6369     };
6370
6371     my $input = undef;
6372     if ($archive eq '-') {
6373         $input = '<&STDIN';
6374     } else {
6375         # If we use a backup from a PVE defined storage we also consider that
6376         # storage's rate limit:
6377         my (undef, $volid) = PVE::Storage::path_to_volume_id($cfg, $archive);
6378         if (defined($volid)) {
6379             my ($sid, undef) = PVE::Storage::parse_volume_id($volid);
6380             my $readlimit = PVE::Storage::get_bandwidth_limit('restore', [$sid], $bwlimit);
6381             if ($readlimit) {
6382                 print STDERR "applying read rate limit: $readlimit\n";
6383                 my $cstream = ['cstream', '-t', $readlimit*1024, '--', $readfrom];
6384                 $add_pipe->($cstream);
6385             }
6386         }
6387     }
6388
6389     if ($comp) {
6390         my $cmd;
6391         if ($comp eq 'gzip') {
6392             $cmd = ['zcat', $readfrom];
6393         } elsif ($comp eq 'lzop') {
6394             $cmd = ['lzop', '-d', '-c', $readfrom];
6395         } else {
6396             die "unknown compression method '$comp'\n";
6397         }
6398         $add_pipe->($cmd);
6399     }
6400
6401     my $tmpdir = "/var/tmp/vzdumptmp$$";
6402     rmtree $tmpdir;
6403
6404     # disable interrupts (always do cleanups)
6405     local $SIG{INT} =
6406         local $SIG{TERM} =
6407         local $SIG{QUIT} =
6408         local $SIG{HUP} = sub { warn "got interrupt - ignored\n"; };
6409
6410     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
6411     POSIX::mkfifo($mapfifo, 0600);
6412     my $fifofh;
6413
6414     my $openfifo = sub {
6415         open($fifofh, '>', $mapfifo) || die $!;
6416     };
6417
6418     $add_pipe->(['vma', 'extract', '-v', '-r', $mapfifo, $readfrom, $tmpdir]);
6419
6420     my $oldtimeout;
6421     my $timeout = 5;
6422
6423     my $devinfo = {};
6424
6425     my $rpcenv = PVE::RPCEnvironment::get();
6426
6427     my $conffile = PVE::QemuConfig->config_file($vmid);
6428     my $tmpfn = "$conffile.$$.tmp";
6429
6430     # Note: $oldconf is undef if VM does not exists
6431     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
6432     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
6433
6434     my %storage_limits;
6435
6436     my $print_devmap = sub {
6437         my $virtdev_hash = {};
6438
6439         my $cfgfn = "$tmpdir/qemu-server.conf";
6440
6441         # we can read the config - that is already extracted
6442         my $fh = IO::File->new($cfgfn, "r") ||
6443             "unable to read qemu-server.conf - $!\n";
6444
6445         my $fwcfgfn = "$tmpdir/qemu-server.fw";
6446         if (-f $fwcfgfn) {
6447             my $pve_firewall_dir = '/etc/pve/firewall';
6448             mkdir $pve_firewall_dir; # make sure the dir exists
6449             PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
6450         }
6451
6452         while (defined(my $line = <$fh>)) {
6453             if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
6454                 my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
6455                 die "archive does not contain data for drive '$virtdev'\n"
6456                     if !$devinfo->{$devname};
6457                 if (defined($opts->{storage})) {
6458                     $storeid = $opts->{storage} || 'local';
6459                 } elsif (!$storeid) {
6460                     $storeid = 'local';
6461                 }
6462                 $format = 'raw' if !$format;
6463                 $devinfo->{$devname}->{devname} = $devname;
6464                 $devinfo->{$devname}->{virtdev} = $virtdev;
6465                 $devinfo->{$devname}->{format} = $format;
6466                 $devinfo->{$devname}->{storeid} = $storeid;
6467
6468                 # check permission on storage
6469                 my $pool = $opts->{pool}; # todo: do we need that?
6470                 if ($user ne 'root@pam') {
6471                     $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
6472                 }
6473
6474                 $storage_limits{$storeid} = $bwlimit;
6475
6476                 $virtdev_hash->{$virtdev} = $devinfo->{$devname};
6477             } elsif ($line =~ m/^((?:ide|sata|scsi)\d+):\s*(.*)\s*$/) {
6478                 my $virtdev = $1;
6479                 my $drive = parse_drive($virtdev, $2);
6480                 if (drive_is_cloudinit($drive)) {
6481                     my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6482                     my $scfg = PVE::Storage::storage_config($cfg, $storeid);
6483                     my $format = qemu_img_format($scfg, $volname); # has 'raw' fallback
6484
6485                     my $d = {
6486                         format => $format,
6487                         storeid => $opts->{storage} // $storeid,
6488                         size => PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE,
6489                         file => $drive->{file}, # to make drive_is_cloudinit check possible
6490                         name => "vm-$vmid-cloudinit",
6491                         is_cloudinit => 1,
6492                     };
6493                     $virtdev_hash->{$virtdev} = $d;
6494                 }
6495             }
6496         }
6497
6498         foreach my $key (keys %storage_limits) {
6499             my $limit = PVE::Storage::get_bandwidth_limit('restore', [$key], $bwlimit);
6500             next if !$limit;
6501             print STDERR "rate limit for storage $key: $limit KiB/s\n";
6502             $storage_limits{$key} = $limit * 1024;
6503         }
6504
6505         foreach my $devname (keys %$devinfo) {
6506             die "found no device mapping information for device '$devname'\n"
6507                 if !$devinfo->{$devname}->{virtdev};
6508         }
6509
6510         # create empty/temp config
6511         if ($oldconf) {
6512             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
6513             foreach_drive($oldconf, sub {
6514                 my ($ds, $drive) = @_;
6515
6516                 return if drive_is_cdrom($drive, 1);
6517
6518                 my $volid = $drive->{file};
6519                 return if !$volid || $volid =~ m|^/|;
6520
6521                 my ($path, $owner) = PVE::Storage::path($cfg, $volid);
6522                 return if !$path || !$owner || ($owner != $vmid);
6523
6524                 # Note: only delete disk we want to restore
6525                 # other volumes will become unused
6526                 if ($virtdev_hash->{$ds}) {
6527                     eval { PVE::Storage::vdisk_free($cfg, $volid); };
6528                     if (my $err = $@) {
6529                         warn $err;
6530                     }
6531                 }
6532             });
6533
6534             # delete vmstate files, after the restore we have no snapshots anymore
6535             foreach my $snapname (keys %{$oldconf->{snapshots}}) {
6536                 my $snap = $oldconf->{snapshots}->{$snapname};
6537                 if ($snap->{vmstate}) {
6538                     eval { PVE::Storage::vdisk_free($cfg, $snap->{vmstate}); };
6539                     if (my $err = $@) {
6540                         warn $err;
6541                     }
6542                 }
6543             }
6544         }
6545
6546         my $map = {};
6547         foreach my $virtdev (sort keys %$virtdev_hash) {
6548             my $d = $virtdev_hash->{$virtdev};
6549             my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
6550             my $storeid = $d->{storeid};
6551             my $scfg = PVE::Storage::storage_config($cfg, $storeid);
6552
6553             my $map_opts = '';
6554             if (my $limit = $storage_limits{$storeid}) {
6555                 $map_opts .= "throttling.bps=$limit:throttling.group=$storeid:";
6556             }
6557
6558             # test if requested format is supported
6559             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($cfg, $storeid);
6560             my $supported = grep { $_ eq $d->{format} } @$validFormats;
6561             $d->{format} = $defFormat if !$supported;
6562
6563             my $name;
6564             if ($d->{is_cloudinit}) {
6565                 $name = $d->{name};
6566                 $name .= ".$d->{format}" if $d->{format} ne 'raw';
6567             }
6568
6569             my $volid = PVE::Storage::vdisk_alloc($cfg, $storeid, $vmid, $d->{format}, $name, $alloc_size);
6570             print STDERR "new volume ID is '$volid'\n";
6571             $d->{volid} = $volid;
6572
6573             PVE::Storage::activate_volumes($cfg, [$volid]);
6574
6575             my $write_zeros = 1;
6576             if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
6577                 $write_zeros = 0;
6578             }
6579
6580             if (!$d->{is_cloudinit}) {
6581                 my $path = PVE::Storage::path($cfg, $volid);
6582
6583                 print $fifofh "${map_opts}format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
6584
6585                 print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
6586             }
6587             $map->{$virtdev} = $volid;
6588         }
6589
6590         $fh->seek(0, 0) || die "seek failed - $!\n";
6591
6592         my $outfd = new IO::File ($tmpfn, "w") ||
6593             die "unable to write config for VM $vmid\n";
6594
6595         my $cookie = { netcount => 0 };
6596         while (defined(my $line = <$fh>)) {
6597             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
6598         }
6599
6600         $fh->close();
6601         $outfd->close();
6602     };
6603
6604     eval {
6605         # enable interrupts
6606         local $SIG{INT} =
6607             local $SIG{TERM} =
6608             local $SIG{QUIT} =
6609             local $SIG{HUP} =
6610             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6611         local $SIG{ALRM} = sub { die "got timeout\n"; };
6612
6613         $oldtimeout = alarm($timeout);
6614
6615         my $parser = sub {
6616             my $line = shift;
6617
6618             print "$line\n";
6619
6620             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
6621                 my ($dev_id, $size, $devname) = ($1, $2, $3);
6622                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
6623             } elsif ($line =~ m/^CTIME: /) {
6624                 # we correctly received the vma config, so we can disable
6625                 # the timeout now for disk allocation (set to 10 minutes, so
6626                 # that we always timeout if something goes wrong)
6627                 alarm(600);
6628                 &$print_devmap();
6629                 print $fifofh "done\n";
6630                 my $tmp = $oldtimeout || 0;
6631                 $oldtimeout = undef;
6632                 alarm($tmp);
6633                 close($fifofh);
6634             }
6635         };
6636
6637         print "restore vma archive: $dbg_cmdstring\n";
6638         run_command($commands, input => $input, outfunc => $parser, afterfork => $openfifo);
6639     };
6640     my $err = $@;
6641
6642     alarm($oldtimeout) if $oldtimeout;
6643
6644     my $vollist = [];
6645     foreach my $devname (keys %$devinfo) {
6646         my $volid = $devinfo->{$devname}->{volid};
6647         push @$vollist, $volid if $volid;
6648     }
6649
6650     PVE::Storage::deactivate_volumes($cfg, $vollist);
6651
6652     unlink $mapfifo;
6653
6654     if ($err) {
6655         rmtree $tmpdir;
6656         unlink $tmpfn;
6657
6658         foreach my $devname (keys %$devinfo) {
6659             my $volid = $devinfo->{$devname}->{volid};
6660             next if !$volid;
6661             eval {
6662                 if ($volid =~ m|^/|) {
6663                     unlink $volid || die 'unlink failed\n';
6664                 } else {
6665                     PVE::Storage::vdisk_free($cfg, $volid);
6666                 }
6667                 print STDERR "temporary volume '$volid' sucessfuly removed\n";
6668             };
6669             print STDERR "unable to cleanup '$volid' - $@" if $@;
6670         }
6671         die $err;
6672     }
6673
6674     rmtree $tmpdir;
6675
6676     rename($tmpfn, $conffile) ||
6677         die "unable to commit configuration file '$conffile'\n";
6678
6679     PVE::Cluster::cfs_update(); # make sure we read new file
6680
6681     eval { rescan($vmid, 1); };
6682     warn $@ if $@;
6683 }
6684
6685 sub restore_tar_archive {
6686     my ($archive, $vmid, $user, $opts) = @_;
6687
6688     if ($archive ne '-') {
6689         my $firstfile = tar_archive_read_firstfile($archive);
6690         die "ERROR: file '$archive' dos not lock like a QemuServer vzdump backup\n"
6691             if $firstfile ne 'qemu-server.conf';
6692     }
6693
6694     my $storecfg = PVE::Storage::config();
6695
6696     # destroy existing data - keep empty config
6697     my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
6698     destroy_vm($storecfg, $vmid, 1) if -f $vmcfgfn;
6699
6700     my $tocmd = "/usr/lib/qemu-server/qmextract";
6701
6702     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
6703     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
6704     $tocmd .= ' --prealloc' if $opts->{prealloc};
6705     $tocmd .= ' --info' if $opts->{info};
6706
6707     # tar option "xf" does not autodetect compression when read from STDIN,
6708     # so we pipe to zcat
6709     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
6710         PVE::Tools::shellquote("--to-command=$tocmd");
6711
6712     my $tmpdir = "/var/tmp/vzdumptmp$$";
6713     mkpath $tmpdir;
6714
6715     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
6716     local $ENV{VZDUMP_VMID} = $vmid;
6717     local $ENV{VZDUMP_USER} = $user;
6718
6719     my $conffile = PVE::QemuConfig->config_file($vmid);
6720     my $tmpfn = "$conffile.$$.tmp";
6721
6722     # disable interrupts (always do cleanups)
6723     local $SIG{INT} =
6724         local $SIG{TERM} =
6725         local $SIG{QUIT} =
6726         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
6727
6728     eval {
6729         # enable interrupts
6730         local $SIG{INT} =
6731             local $SIG{TERM} =
6732             local $SIG{QUIT} =
6733             local $SIG{HUP} =
6734             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6735
6736         if ($archive eq '-') {
6737             print "extracting archive from STDIN\n";
6738             run_command($cmd, input => "<&STDIN");
6739         } else {
6740             print "extracting archive '$archive'\n";
6741             run_command($cmd);
6742         }
6743
6744         return if $opts->{info};
6745
6746         # read new mapping
6747         my $map = {};
6748         my $statfile = "$tmpdir/qmrestore.stat";
6749         if (my $fd = IO::File->new($statfile, "r")) {
6750             while (defined (my $line = <$fd>)) {
6751                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
6752                     $map->{$1} = $2 if $1;
6753                 } else {
6754                     print STDERR "unable to parse line in statfile - $line\n";
6755                 }
6756             }
6757             $fd->close();
6758         }
6759
6760         my $confsrc = "$tmpdir/qemu-server.conf";
6761
6762         my $srcfd = new IO::File($confsrc, "r") ||
6763             die "unable to open file '$confsrc'\n";
6764
6765         my $outfd = new IO::File ($tmpfn, "w") ||
6766             die "unable to write config for VM $vmid\n";
6767
6768         my $cookie = { netcount => 0 };
6769         while (defined (my $line = <$srcfd>)) {
6770             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
6771         }
6772
6773         $srcfd->close();
6774         $outfd->close();
6775     };
6776     my $err = $@;
6777
6778     if ($err) {
6779
6780         unlink $tmpfn;
6781
6782         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
6783
6784         die $err;
6785     }
6786
6787     rmtree $tmpdir;
6788
6789     rename $tmpfn, $conffile ||
6790         die "unable to commit configuration file '$conffile'\n";
6791
6792     PVE::Cluster::cfs_update(); # make sure we read new file
6793
6794     eval { rescan($vmid, 1); };
6795     warn $@ if $@;
6796 };
6797
6798 sub foreach_storage_used_by_vm {
6799     my ($conf, $func) = @_;
6800
6801     my $sidhash = {};
6802
6803     foreach_drive($conf, sub {
6804         my ($ds, $drive) = @_;
6805         return if drive_is_cdrom($drive);
6806
6807         my $volid = $drive->{file};
6808
6809         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
6810         $sidhash->{$sid} = $sid if $sid;
6811     });
6812
6813     foreach my $sid (sort keys %$sidhash) {
6814         &$func($sid);
6815     }
6816 }
6817
6818 sub do_snapshots_with_qemu {
6819     my ($storecfg, $volid) = @_;
6820
6821     my $storage_name = PVE::Storage::parse_volume_id($volid);
6822     my $scfg = $storecfg->{ids}->{$storage_name};
6823
6824     if ($qemu_snap_storage->{$scfg->{type}} && !$scfg->{krbd}){
6825         return 1;
6826     }
6827
6828     if ($volid =~ m/\.(qcow2|qed)$/){
6829         return 1;
6830     }
6831
6832     return undef;
6833 }
6834
6835 sub qga_check_running {
6836     my ($vmid, $nowarn) = @_;
6837
6838     eval { vm_mon_cmd($vmid, "guest-ping", timeout => 3); };
6839     if ($@) {
6840         warn "Qemu Guest Agent is not running - $@" if !$nowarn;
6841         return 0;
6842     }
6843     return 1;
6844 }
6845
6846 sub template_create {
6847     my ($vmid, $conf, $disk) = @_;
6848
6849     my $storecfg = PVE::Storage::config();
6850
6851     foreach_drive($conf, sub {
6852         my ($ds, $drive) = @_;
6853
6854         return if drive_is_cdrom($drive);
6855         return if $disk && $ds ne $disk;
6856
6857         my $volid = $drive->{file};
6858         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
6859
6860         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
6861         $drive->{file} = $voliddst;
6862         $conf->{$ds} = print_drive($vmid, $drive);
6863         PVE::QemuConfig->write_config($vmid, $conf);
6864     });
6865 }
6866
6867 sub convert_iscsi_path {
6868     my ($path) = @_;
6869
6870     if ($path =~ m|^iscsi://([^/]+)/([^/]+)/(.+)$|) {
6871         my $portal = $1;
6872         my $target = $2;
6873         my $lun = $3;
6874
6875         my $initiator_name = get_initiator_name();
6876
6877         return "file.driver=iscsi,file.transport=tcp,file.initiator-name=$initiator_name,".
6878                "file.portal=$portal,file.target=$target,file.lun=$lun,driver=raw";
6879     }
6880
6881     die "cannot convert iscsi path '$path', unkown format\n";
6882 }
6883
6884 sub qemu_img_convert {
6885     my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized) = @_;
6886
6887     my $storecfg = PVE::Storage::config();
6888     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
6889     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
6890
6891     die "destination '$dst_volid' is not a valid volid form qemu-img convert\n" if !$dst_storeid;
6892
6893     my $cachemode;
6894     my $src_path;
6895     my $src_is_iscsi = 0;
6896     my $src_format = 'raw';
6897
6898     if ($src_storeid) {
6899         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
6900         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
6901         $src_format = qemu_img_format($src_scfg, $src_volname);
6902         $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
6903         $src_is_iscsi = ($src_path =~ m|^iscsi://|);
6904         $cachemode = 'none' if $src_scfg->{type} eq 'zfspool';
6905     } elsif (-f $src_volid) {
6906         $src_path = $src_volid;
6907         if ($src_path =~ m/\.($QEMU_FORMAT_RE)$/) {
6908             $src_format = $1;
6909         }
6910     }
6911
6912     die "source '$src_volid' is not a valid volid nor path for qemu-img convert\n" if !$src_path;
6913
6914     my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6915     my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
6916     my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6917     my $dst_is_iscsi = ($dst_path =~ m|^iscsi://|);
6918
6919     my $cmd = [];
6920     push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
6921     push @$cmd, '-l', "snapshot.name=$snapname" if($snapname && $src_format eq "qcow2");
6922     push @$cmd, '-t', 'none' if $dst_scfg->{type} eq 'zfspool';
6923     push @$cmd, '-T', $cachemode if defined($cachemode);
6924
6925     if ($src_is_iscsi) {
6926         push @$cmd, '--image-opts';
6927         $src_path = convert_iscsi_path($src_path);
6928     } else {
6929         push @$cmd, '-f', $src_format;
6930     }
6931
6932     if ($dst_is_iscsi) {
6933         push @$cmd, '--target-image-opts';
6934         $dst_path = convert_iscsi_path($dst_path);
6935     } else {
6936         push @$cmd, '-O', $dst_format;
6937     }
6938
6939     push @$cmd, $src_path;
6940
6941     if (!$dst_is_iscsi && $is_zero_initialized) {
6942         push @$cmd, "zeroinit:$dst_path";
6943     } else {
6944         push @$cmd, $dst_path;
6945     }
6946
6947     my $parser = sub {
6948         my $line = shift;
6949         if($line =~ m/\((\S+)\/100\%\)/){
6950             my $percent = $1;
6951             my $transferred = int($size * $percent / 100);
6952             my $remaining = $size - $transferred;
6953
6954             print "transferred: $transferred bytes remaining: $remaining bytes total: $size bytes progression: $percent %\n";
6955         }
6956
6957     };
6958
6959     eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
6960     my $err = $@;
6961     die "copy failed: $err" if $err;
6962 }
6963
6964 sub qemu_img_format {
6965     my ($scfg, $volname) = @_;
6966
6967     if ($scfg->{path} && $volname =~ m/\.($QEMU_FORMAT_RE)$/) {
6968         return $1;
6969     } else {
6970         return "raw";
6971     }
6972 }
6973
6974 sub qemu_drive_mirror {
6975     my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $skipcomplete, $qga, $bwlimit) = @_;
6976
6977     $jobs = {} if !$jobs;
6978
6979     my $qemu_target;
6980     my $format;
6981     $jobs->{"drive-$drive"} = {};
6982
6983     if ($dst_volid =~ /^nbd:/) {
6984         $qemu_target = $dst_volid;
6985         $format = "nbd";
6986     } else {
6987         my $storecfg = PVE::Storage::config();
6988         my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
6989
6990         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6991
6992         $format = qemu_img_format($dst_scfg, $dst_volname);
6993
6994         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6995
6996         $qemu_target = $is_zero_initialized ? "zeroinit:$dst_path" : $dst_path;
6997     }
6998
6999     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $qemu_target };
7000     $opts->{format} = $format if $format;
7001
7002     if (defined($bwlimit)) {
7003         $opts->{speed} = $bwlimit * 1024;
7004         print "drive mirror is starting for drive-$drive with bandwidth limit: ${bwlimit} KB/s\n";
7005     } else {
7006         print "drive mirror is starting for drive-$drive\n";
7007     }
7008
7009     # if a job already runs for this device we get an error, catch it for cleanup
7010     eval { vm_mon_cmd($vmid, "drive-mirror", %$opts); };
7011     if (my $err = $@) {
7012         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
7013         warn "$@\n" if $@;
7014         die "mirroring error: $err\n";
7015     }
7016
7017     qemu_drive_mirror_monitor ($vmid, $vmiddst, $jobs, $skipcomplete, $qga);
7018 }
7019
7020 sub qemu_drive_mirror_monitor {
7021     my ($vmid, $vmiddst, $jobs, $skipcomplete, $qga) = @_;
7022
7023     eval {
7024         my $err_complete = 0;
7025
7026         while (1) {
7027             die "storage migration timed out\n" if $err_complete > 300;
7028
7029             my $stats = vm_mon_cmd($vmid, "query-block-jobs");
7030
7031             my $running_mirror_jobs = {};
7032             foreach my $stat (@$stats) {
7033                 next if $stat->{type} ne 'mirror';
7034                 $running_mirror_jobs->{$stat->{device}} = $stat;
7035             }
7036
7037             my $readycounter = 0;
7038
7039             foreach my $job (keys %$jobs) {
7040
7041                 if(defined($jobs->{$job}->{complete}) && !defined($running_mirror_jobs->{$job})) {
7042                     print "$job : finished\n";
7043                     delete $jobs->{$job};
7044                     next;
7045                 }
7046
7047                 die "$job: mirroring has been cancelled\n" if !defined($running_mirror_jobs->{$job});
7048
7049                 my $busy = $running_mirror_jobs->{$job}->{busy};
7050                 my $ready = $running_mirror_jobs->{$job}->{ready};
7051                 if (my $total = $running_mirror_jobs->{$job}->{len}) {
7052                     my $transferred = $running_mirror_jobs->{$job}->{offset} || 0;
7053                     my $remaining = $total - $transferred;
7054                     my $percent = sprintf "%.2f", ($transferred * 100 / $total);
7055
7056                     print "$job: transferred: $transferred bytes remaining: $remaining bytes total: $total bytes progression: $percent % busy: $busy ready: $ready \n";
7057                 }
7058
7059                 $readycounter++ if $running_mirror_jobs->{$job}->{ready};
7060             }
7061
7062             last if scalar(keys %$jobs) == 0;
7063
7064             if ($readycounter == scalar(keys %$jobs)) {
7065                 print "all mirroring jobs are ready \n";
7066                 last if $skipcomplete; #do the complete later
7067
7068                 if ($vmiddst && $vmiddst != $vmid) {
7069                     my $agent_running = $qga && qga_check_running($vmid);
7070                     if ($agent_running) {
7071                         print "freeze filesystem\n";
7072                         eval { PVE::QemuServer::vm_mon_cmd($vmid, "guest-fsfreeze-freeze"); };
7073                     } else {
7074                         print "suspend vm\n";
7075                         eval { PVE::QemuServer::vm_suspend($vmid, 1); };
7076                     }
7077
7078                     # if we clone a disk for a new target vm, we don't switch the disk
7079                     PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs);
7080
7081                     if ($agent_running) {
7082                         print "unfreeze filesystem\n";
7083                         eval { PVE::QemuServer::vm_mon_cmd($vmid, "guest-fsfreeze-thaw"); };
7084                     } else {
7085                         print "resume vm\n";
7086                         eval {  PVE::QemuServer::vm_resume($vmid, 1, 1); };
7087                     }
7088
7089                     last;
7090                 } else {
7091
7092                     foreach my $job (keys %$jobs) {
7093                         # try to switch the disk if source and destination are on the same guest
7094                         print "$job: Completing block job...\n";
7095
7096                         eval { vm_mon_cmd($vmid, "block-job-complete", device => $job) };
7097                         if ($@ =~ m/cannot be completed/) {
7098                             print "$job: Block job cannot be completed, try again.\n";
7099                             $err_complete++;
7100                         }else {
7101                             print "$job: Completed successfully.\n";
7102                             $jobs->{$job}->{complete} = 1;
7103                         }
7104                     }
7105                 }
7106             }
7107             sleep 1;
7108         }
7109     };
7110     my $err = $@;
7111
7112     if ($err) {
7113         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
7114         die "mirroring error: $err";
7115     }
7116
7117 }
7118
7119 sub qemu_blockjobs_cancel {
7120     my ($vmid, $jobs) = @_;
7121
7122     foreach my $job (keys %$jobs) {
7123         print "$job: Cancelling block job\n";
7124         eval { vm_mon_cmd($vmid, "block-job-cancel", device => $job); };
7125         $jobs->{$job}->{cancel} = 1;
7126     }
7127
7128     while (1) {
7129         my $stats = vm_mon_cmd($vmid, "query-block-jobs");
7130
7131         my $running_jobs = {};
7132         foreach my $stat (@$stats) {
7133             $running_jobs->{$stat->{device}} = $stat;
7134         }
7135
7136         foreach my $job (keys %$jobs) {
7137
7138             if (defined($jobs->{$job}->{cancel}) && !defined($running_jobs->{$job})) {
7139                 print "$job: Done.\n";
7140                 delete $jobs->{$job};
7141             }
7142         }
7143
7144         last if scalar(keys %$jobs) == 0;
7145
7146         sleep 1;
7147     }
7148 }
7149
7150 sub clone_disk {
7151     my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
7152         $newvmid, $storage, $format, $full, $newvollist, $jobs, $skipcomplete, $qga, $bwlimit) = @_;
7153
7154     my $newvolid;
7155
7156     if (!$full) {
7157         print "create linked clone of drive $drivename ($drive->{file})\n";
7158         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
7159         push @$newvollist, $newvolid;
7160     } else {
7161
7162         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
7163         $storeid = $storage if $storage;
7164
7165         my $dst_format = resolve_dst_disk_format($storecfg, $storeid, $volname, $format);
7166         my ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 3);
7167
7168         print "create full clone of drive $drivename ($drive->{file})\n";
7169         my $name = undef;
7170         $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $newvmid, $dst_format, $name, ($size/1024));
7171         push @$newvollist, $newvolid;
7172
7173         PVE::Storage::activate_volumes($storecfg, [$newvolid]);
7174
7175         my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
7176         if (!$running || $snapname) {
7177             # TODO: handle bwlimits
7178             qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit);
7179         } else {
7180
7181             my $kvmver = get_running_qemu_version ($vmid);
7182             if (!qemu_machine_feature_enabled (undef, $kvmver, 2, 7)) {
7183                 die "drive-mirror with iothread requires qemu version 2.7 or higher\n"
7184                     if $drive->{iothread};
7185             }
7186
7187             qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid, $sparseinit, $jobs, $skipcomplete, $qga, $bwlimit);
7188         }
7189     }
7190
7191     my ($size) = PVE::Storage::volume_size_info($storecfg, $newvolid, 3);
7192
7193     my $disk = $drive;
7194     $disk->{format} = undef;
7195     $disk->{file} = $newvolid;
7196     $disk->{size} = $size;
7197
7198     return $disk;
7199 }
7200
7201 # this only works if VM is running
7202 sub get_current_qemu_machine {
7203     my ($vmid) = @_;
7204
7205     my $cmd = { execute => 'query-machines', arguments => {} };
7206     my $res = vm_qmp_command($vmid, $cmd);
7207
7208     my ($current, $default);
7209     foreach my $e (@$res) {
7210         $default = $e->{name} if $e->{'is-default'};
7211         $current = $e->{name} if $e->{'is-current'};
7212     }
7213
7214     # fallback to the default machine if current is not supported by qemu
7215     return $current || $default || 'pc';
7216 }
7217
7218 sub get_running_qemu_version {
7219     my ($vmid) = @_;
7220     my $cmd = { execute => 'query-version', arguments => {} };
7221     my $res = vm_qmp_command($vmid, $cmd);
7222     return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
7223 }
7224
7225 sub qemu_machine_feature_enabled {
7226     my ($machine, $kvmver, $version_major, $version_minor) = @_;
7227
7228     my $current_major;
7229     my $current_minor;
7230
7231     if ($machine && $machine =~ m/^((?:pc(-i440fx|-q35)?|virt)-(\d+)\.(\d+))/) {
7232
7233         $current_major = $3;
7234         $current_minor = $4;
7235
7236     } elsif ($kvmver =~ m/^(\d+)\.(\d+)/) {
7237
7238         $current_major = $1;
7239         $current_minor = $2;
7240     }
7241
7242     return 1 if $current_major > $version_major ||
7243                 ($current_major == $version_major &&
7244                  $current_minor >= $version_minor);
7245 }
7246
7247 sub qemu_machine_pxe {
7248     my ($vmid, $conf) = @_;
7249
7250     my $machine =  PVE::QemuServer::get_current_qemu_machine($vmid);
7251
7252     if ($conf->{machine} && $conf->{machine} =~ m/\.pxe$/) {
7253         $machine .= '.pxe';
7254     }
7255
7256     return $machine;
7257 }
7258
7259 sub qemu_use_old_bios_files {
7260     my ($machine_type) = @_;
7261
7262     return if !$machine_type;
7263
7264     my $use_old_bios_files = undef;
7265
7266     if ($machine_type =~ m/^(\S+)\.pxe$/) {
7267         $machine_type = $1;
7268         $use_old_bios_files = 1;
7269     } else {
7270         my $kvmver = kvm_user_version();
7271         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
7272         # load new efi bios files on migration. So this hack is required to allow
7273         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
7274         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
7275         $use_old_bios_files = !qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 4);
7276     }
7277
7278     return ($use_old_bios_files, $machine_type);
7279 }
7280
7281 sub create_efidisk($$$$$) {
7282     my ($storecfg, $storeid, $vmid, $fmt, $arch) = @_;
7283
7284     my (undef, $ovmf_vars) = get_ovmf_files($arch);
7285     die "EFI vars default image not found\n" if ! -f $ovmf_vars;
7286
7287     my $vars_size_b = -s $ovmf_vars;
7288     my $vars_size = PVE::Tools::convert_size($vars_size_b, 'b' => 'kb');
7289     my $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $fmt, undef, $vars_size);
7290     PVE::Storage::activate_volumes($storecfg, [$volid]);
7291
7292     qemu_img_convert($ovmf_vars, $volid, $vars_size_b, undef, 0);
7293
7294     return ($volid, $vars_size);
7295 }
7296
7297 sub vm_iothreads_list {
7298     my ($vmid) = @_;
7299
7300     my $res = vm_mon_cmd($vmid, 'query-iothreads');
7301
7302     my $iothreads = {};
7303     foreach my $iothread (@$res) {
7304         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
7305     }
7306
7307     return $iothreads;
7308 }
7309
7310 sub scsihw_infos {
7311     my ($conf, $drive) = @_;
7312
7313     my $maxdev = 0;
7314
7315     if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)) {
7316         $maxdev = 7;
7317     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
7318         $maxdev = 1;
7319     } else {
7320         $maxdev = 256;
7321     }
7322
7323     my $controller = int($drive->{index} / $maxdev);
7324     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single') ? "virtioscsi" : "scsihw";
7325
7326     return ($maxdev, $controller, $controller_prefix);
7327 }
7328
7329 sub add_hyperv_enlightenments {
7330     my ($cpuFlags, $winversion, $machine_type, $kvmver, $bios, $gpu_passthrough, $hv_vendor_id) = @_;
7331
7332     return if $winversion < 6;
7333     return if $bios && $bios eq 'ovmf' && $winversion < 8;
7334
7335     if ($gpu_passthrough || defined($hv_vendor_id)) {
7336         $hv_vendor_id //= 'proxmox';
7337         push @$cpuFlags , "hv_vendor_id=$hv_vendor_id";
7338     }
7339
7340     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
7341         push @$cpuFlags , 'hv_spinlocks=0x1fff';
7342         push @$cpuFlags , 'hv_vapic';
7343         push @$cpuFlags , 'hv_time';
7344     } else {
7345         push @$cpuFlags , 'hv_spinlocks=0xffff';
7346     }
7347
7348     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 6)) {
7349         push @$cpuFlags , 'hv_reset';
7350         push @$cpuFlags , 'hv_vpindex';
7351         push @$cpuFlags , 'hv_runtime';
7352     }
7353
7354     if ($winversion >= 7) {
7355         push @$cpuFlags , 'hv_relaxed';
7356
7357         if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 12)) {
7358             push @$cpuFlags , 'hv_synic';
7359             push @$cpuFlags , 'hv_stimer';
7360         }
7361
7362         if (qemu_machine_feature_enabled ($machine_type, $kvmver, 3, 1)) {
7363             push @$cpuFlags , 'hv_ipi';
7364         }
7365     }
7366 }
7367
7368 sub windows_version {
7369     my ($ostype) = @_;
7370
7371     return 0 if !$ostype;
7372
7373     my $winversion = 0;
7374
7375     if($ostype eq 'wxp' || $ostype eq 'w2k3' || $ostype eq 'w2k') {
7376         $winversion = 5;
7377     } elsif($ostype eq 'w2k8' || $ostype eq 'wvista') {
7378         $winversion = 6;
7379     } elsif ($ostype =~ m/^win(\d+)$/) {
7380         $winversion = $1;
7381     }
7382
7383     return $winversion;
7384 }
7385
7386 sub resolve_dst_disk_format {
7387         my ($storecfg, $storeid, $src_volname, $format) = @_;
7388         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
7389
7390         if (!$format) {
7391             # if no target format is specified, use the source disk format as hint
7392             if ($src_volname) {
7393                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
7394                 $format = qemu_img_format($scfg, $src_volname);
7395             } else {
7396                 return $defFormat;
7397             }
7398         }
7399
7400         # test if requested format is supported - else use default
7401         my $supported = grep { $_ eq $format } @$validFormats;
7402         $format = $defFormat if !$supported;
7403         return $format;
7404 }
7405
7406 sub resolve_first_disk {
7407     my $conf = shift;
7408     my @disks = PVE::QemuServer::valid_drive_names();
7409     my $firstdisk;
7410     foreach my $ds (reverse @disks) {
7411         next if !$conf->{$ds};
7412         my $disk = PVE::QemuServer::parse_drive($ds, $conf->{$ds});
7413         next if PVE::QemuServer::drive_is_cdrom($disk);
7414         $firstdisk = $ds;
7415     }
7416     return $firstdisk;
7417 }
7418
7419 sub generate_uuid {
7420     my ($uuid, $uuid_str);
7421     UUID::generate($uuid);
7422     UUID::unparse($uuid, $uuid_str);
7423     return $uuid_str;
7424 }
7425
7426 sub generate_smbios1_uuid {
7427     return "uuid=".generate_uuid();
7428 }
7429
7430 sub nbd_stop {
7431     my ($vmid) = @_;
7432
7433     vm_mon_cmd($vmid, 'nbd-server-stop');
7434 }
7435
7436 sub create_reboot_request {
7437     my ($vmid) = @_;
7438     open(my $fh, '>', "/run/qemu-server/$vmid.reboot")
7439         or die "failed to create reboot trigger file: $!\n";
7440     close($fh);
7441 }
7442
7443 sub clear_reboot_request {
7444     my ($vmid) = @_;
7445     my $path = "/run/qemu-server/$vmid.reboot";
7446     my $res = 0;
7447
7448     $res = unlink($path);
7449     die "could not remove reboot request for $vmid: $!"
7450         if !$res && $! != POSIX::ENOENT;
7451
7452     return $res;
7453 }
7454
7455 # bash completion helper
7456
7457 sub complete_backup_archives {
7458     my ($cmdname, $pname, $cvalue) = @_;
7459
7460     my $cfg = PVE::Storage::config();
7461
7462     my $storeid;
7463
7464     if ($cvalue =~ m/^([^:]+):/) {
7465         $storeid = $1;
7466     }
7467
7468     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
7469
7470     my $res = [];
7471     foreach my $id (keys %$data) {
7472         foreach my $item (@{$data->{$id}}) {
7473             next if $item->{format} !~ m/^vma\.(gz|lzo)$/;
7474             push @$res, $item->{volid} if defined($item->{volid});
7475         }
7476     }
7477
7478     return $res;
7479 }
7480
7481 my $complete_vmid_full = sub {
7482     my ($running) = @_;
7483
7484     my $idlist = vmstatus();
7485
7486     my $res = [];
7487
7488     foreach my $id (keys %$idlist) {
7489         my $d = $idlist->{$id};
7490         if (defined($running)) {
7491             next if $d->{template};
7492             next if $running && $d->{status} ne 'running';
7493             next if !$running && $d->{status} eq 'running';
7494         }
7495         push @$res, $id;
7496
7497     }
7498     return $res;
7499 };
7500
7501 sub complete_vmid {
7502     return &$complete_vmid_full();
7503 }
7504
7505 sub complete_vmid_stopped {
7506     return &$complete_vmid_full(0);
7507 }
7508
7509 sub complete_vmid_running {
7510     return &$complete_vmid_full(1);
7511 }
7512
7513 sub complete_storage {
7514
7515     my $cfg = PVE::Storage::config();
7516     my $ids = $cfg->{ids};
7517
7518     my $res = [];
7519     foreach my $sid (keys %$ids) {
7520         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
7521         next if !$ids->{$sid}->{content}->{images};
7522         push @$res, $sid;
7523     }
7524
7525     return $res;
7526 }
7527
7528 1;