PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5
   6 use POSIX;
   7 use IO::Handle;
   8 use IO::Select;
   9 use IO::File;
  10 use IO::Dir;
  11 use IO::Socket::UNIX;
  12 use File::Basename;
  13 use File::Path;
  14 use File::stat;
  15 use Getopt::Long;
  16 use Digest::SHA;
  17 use Fcntl ':flock';
  18 use Cwd 'abs_path';
  19 use IPC::Open3;
  20 use JSON;
  21 use Fcntl;
  22 use PVE::SafeSyslog;
  23 use Storable qw(dclone);
  24 use MIME::Base64;
  25 use PVE::Exception qw(raise raise_param_exc);
  26 use PVE::Storage;
  27 use PVE::Tools qw(run_command lock_file lock_file_full file_read_firstline dir_glob_foreach $IPV6RE);
  28 use PVE::JSONSchema qw(get_standard_option);
  29 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file cfs_lock_file);
  30 use PVE::INotify;
  31 use PVE::ProcFSTools;
  32 use PVE::QemuConfig;
  33 use PVE::QMPClient;
  34 use PVE::RPCEnvironment;
  35 use PVE::GuestHelpers;
  36 use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr print_pcie_root_port);
  37 use PVE::QemuServer::Memory;
  38 use PVE::QemuServer::USB qw(parse_usb_device);
  39 use PVE::QemuServer::Cloudinit;
  40 use PVE::SysFSTools;
  41 use PVE::Systemd;
  42 use Time::HiRes qw(gettimeofday);
  43 use File::Copy qw(copy);
  44 use URI::Escape;
  45
  46 my $EDK2_FW_BASE = '/usr/share/pve-edk2-firmware/';
  47 my $OVMF = {
  48     x86_64 => [
  49         "$EDK2_FW_BASE/OVMF_CODE.fd",
  50         "$EDK2_FW_BASE/OVMF_VARS.fd"
  51     ],
  52     aarch64 => [
  53         "$EDK2_FW_BASE/AAVMF_CODE.fd",
  54         "$EDK2_FW_BASE/AAVMF_VARS.fd"
  55     ],
  56 };
  57
  58 my $qemu_snap_storage = { rbd => 1 };
  59
  60 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  61
  62 my $QEMU_FORMAT_RE = qr/raw|cow|qcow|qcow2|qed|vmdk|cloop/;
  63
  64 # Note about locking: we use flock on the config file protect
  65 # against concurent actions.
  66 # Aditionaly, we have a 'lock' setting in the config file. This
  67 # can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
  68 # allowed when such lock is set. But you can ignore this kind of
  69 # lock with the --skiplock flag.
  70
  71 cfs_register_file('/qemu-server/',
  72                   \&parse_vm_config,
  73                   \&write_vm_config);
  74
  75 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
  76     description => "Some command save/restore state from this location.",
  77     type => 'string',
  78     maxLength => 128,
  79     optional => 1,
  80 });
  81
  82 PVE::JSONSchema::register_standard_option('pve-qm-image-format', {
  83     type => 'string',
  84     enum => [qw(raw cow qcow qed qcow2 vmdk cloop)],
  85     description => "The drive's backing file's data format.",
  86     optional => 1,
  87 });
  88
  89 PVE::JSONSchema::register_standard_option('pve-qemu-machine', {
  90         description => "Specifies the Qemu machine type.",
  91         type => 'string',
  92         pattern => '(pc|pc(-i440fx)?-\d+\.\d+(\.pxe)?|q35|pc-q35-\d+\.\d+(\.pxe)?|virt(?:-\d+\.\d+)?)',
  93         maxLength => 40,
  94         optional => 1,
  95 });
  96
  97 #no warnings 'redefine';
  98
  99 sub cgroups_write {
 100    my ($controller, $vmid, $option, $value) = @_;
 101
 102    my $path = "/sys/fs/cgroup/$controller/qemu.slice/$vmid.scope/$option";
 103    PVE::ProcFSTools::write_proc_entry($path, $value);
 104
 105 }
 106
 107 my $nodename = PVE::INotify::nodename();
 108
 109 mkdir "/etc/pve/nodes/$nodename";
 110 my $confdir = "/etc/pve/nodes/$nodename/qemu-server";
 111 mkdir $confdir;
 112
 113 my $var_run_tmpdir = "/var/run/qemu-server";
 114 mkdir $var_run_tmpdir;
 115
 116 my $lock_dir = "/var/lock/qemu-server";
 117 mkdir $lock_dir;
 118
 119 my $cpu_vendor_list = {
 120     # Intel CPUs
 121     486 => 'GenuineIntel',
 122     pentium => 'GenuineIntel',
 123     pentium2  => 'GenuineIntel',
 124     pentium3  => 'GenuineIntel',
 125     coreduo => 'GenuineIntel',
 126     core2duo => 'GenuineIntel',
 127     Conroe  => 'GenuineIntel',
 128     Penryn  => 'GenuineIntel',
 129     Nehalem  => 'GenuineIntel',
 130     'Nehalem-IBRS'  => 'GenuineIntel',
 131     Westmere => 'GenuineIntel',
 132     'Westmere-IBRS' => 'GenuineIntel',
 133     SandyBridge => 'GenuineIntel',
 134     'SandyBridge-IBRS' => 'GenuineIntel',
 135     IvyBridge => 'GenuineIntel',
 136     'IvyBridge-IBRS' => 'GenuineIntel',
 137     Haswell => 'GenuineIntel',
 138     'Haswell-IBRS' => 'GenuineIntel',
 139     'Haswell-noTSX' => 'GenuineIntel',
 140     'Haswell-noTSX-IBRS' => 'GenuineIntel',
 141     Broadwell => 'GenuineIntel',
 142     'Broadwell-IBRS' => 'GenuineIntel',
 143     'Broadwell-noTSX' => 'GenuineIntel',
 144     'Broadwell-noTSX-IBRS' => 'GenuineIntel',
 145     'Skylake-Client' => 'GenuineIntel',
 146     'Skylake-Client-IBRS' => 'GenuineIntel',
 147     'Skylake-Server' => 'GenuineIntel',
 148     'Skylake-Server-IBRS' => 'GenuineIntel',
 149     'Cascadelake-Server' => 'GenuineIntel',
 150     KnightsMill => 'GenuineIntel',
 151
 152
 153     # AMD CPUs
 154     athlon => 'AuthenticAMD',
 155     phenom  => 'AuthenticAMD',
 156     Opteron_G1  => 'AuthenticAMD',
 157     Opteron_G2  => 'AuthenticAMD',
 158     Opteron_G3  => 'AuthenticAMD',
 159     Opteron_G4  => 'AuthenticAMD',
 160     Opteron_G5  => 'AuthenticAMD',
 161     EPYC => 'AuthenticAMD',
 162     'EPYC-IBPB' => 'AuthenticAMD',
 163
 164     # generic types, use vendor from host node
 165     host => 'default',
 166     kvm32 => 'default',
 167     kvm64 => 'default',
 168     qemu32 => 'default',
 169     qemu64 => 'default',
 170     max => 'default',
 171 };
 172
 173 my @supported_cpu_flags = (
 174     'pcid',
 175     'spec-ctrl',
 176     'ibpb',
 177     'ssbd',
 178     'virt-ssbd',
 179     'amd-ssbd',
 180     'amd-no-ssb',
 181     'pdpe1gb',
 182     'md-clear',
 183     'hv-tlbflush',
 184     'hv-evmcs',
 185     'aes'
 186 );
 187 my $cpu_flag = qr/[+-](@{[join('|', @supported_cpu_flags)]})/;
 188
 189 my $cpu_fmt = {
 190     cputype => {
 191         description => "Emulated CPU type.",
 192         type => 'string',
 193         enum => [ sort { "\L$a" cmp "\L$b" } keys %$cpu_vendor_list ],
 194         default => 'kvm64',
 195         default_key => 1,
 196     },
 197     hidden => {
 198         description => "Do not identify as a KVM virtual machine.",
 199         type => 'boolean',
 200         optional => 1,
 201         default => 0
 202     },
 203     'hv-vendor-id' => {
 204         type => 'string',
 205         pattern => qr/[a-zA-Z0-9]{1,12}/,
 206         format_description => 'vendor-id',
 207         description => 'The Hyper-V vendor ID. Some drivers or programs inside Windows guests need a specific ID.',
 208         optional => 1,
 209     },
 210     flags => {
 211         description => "List of additional CPU flags separated by ';'."
 212                      . " Use '+FLAG' to enable, '-FLAG' to disable a flag."
 213                      . " Currently supported flags: @{[join(', ', @supported_cpu_flags)]}.",
 214         format_description => '+FLAG[;-FLAG...]',
 215         type => 'string',
 216         pattern => qr/$cpu_flag(;$cpu_flag)*/,
 217         optional => 1,
 218     },
 219 };
 220
 221 my $watchdog_fmt = {
 222     model => {
 223         default_key => 1,
 224         type => 'string',
 225         enum => [qw(i6300esb ib700)],
 226         description => "Watchdog type to emulate.",
 227         default => 'i6300esb',
 228         optional => 1,
 229     },
 230     action => {
 231         type => 'string',
 232         enum => [qw(reset shutdown poweroff pause debug none)],
 233         description => "The action to perform if after activation the guest fails to poll the watchdog in time.",
 234         optional => 1,
 235     },
 236 };
 237 PVE::JSONSchema::register_format('pve-qm-watchdog', $watchdog_fmt);
 238
 239 my $agent_fmt = {
 240     enabled => {
 241         description => "Enable/disable Qemu GuestAgent.",
 242         type => 'boolean',
 243         default => 0,
 244         default_key => 1,
 245     },
 246     fstrim_cloned_disks => {
 247         description => "Run fstrim after cloning/moving a disk.",
 248         type => 'boolean',
 249         optional => 1,
 250         default => 0
 251     },
 252 };
 253
 254 my $vga_fmt = {
 255     type => {
 256         description => "Select the VGA type.",
 257         type => 'string',
 258         default => 'std',
 259         optional => 1,
 260         default_key => 1,
 261         enum => [qw(cirrus qxl qxl2 qxl3 qxl4 none serial0 serial1 serial2 serial3 std virtio vmware)],
 262     },
 263     memory => {
 264         description => "Sets the VGA memory (in MiB). Has no effect with serial display.",
 265         type => 'integer',
 266         optional => 1,
 267         minimum => 4,
 268         maximum => 512,
 269     },
 270 };
 271
 272 my $ivshmem_fmt = {
 273     size => {
 274         type => 'integer',
 275         minimum => 1,
 276         description => "The size of the file in MB.",
 277     },
 278     name => {
 279         type => 'string',
 280         pattern => '[a-zA-Z0-9\-]+',
 281         optional => 1,
 282         format_description => 'string',
 283         description => "The name of the file. Will be prefixed with 'pve-shm-'. Default is the VMID. Will be deleted when the VM is stopped.",
 284     },
 285 };
 286
 287 my $audio_fmt = {
 288     device => {
 289         type => 'string',
 290         enum => [qw(ich9-intel-hda intel-hda AC97)],
 291         description =>  "Configure an audio device."
 292     },
 293     driver =>  {
 294         type => 'string',
 295         enum => ['spice'],
 296         default => 'spice',
 297         optional => 1,
 298         description => "Driver backend for the audio device."
 299     },
 300 };
 301
 302 my $spice_enhancements_fmt = {
 303     foldersharing => {
 304         type => 'boolean',
 305         optional => 1,
 306         default => '0',
 307         description =>  "Enable folder sharing via SPICE. Needs Spice-WebDAV daemon installed in the VM."
 308     },
 309     videostreaming =>  {
 310         type => 'string',
 311         enum => ['off', 'all', 'filter'],
 312         default => 'off',
 313         optional => 1,
 314         description => "Enable video streaming. Uses compression for detected video streams."
 315     },
 316 };
 317
 318 my $confdesc = {
 319     onboot => {
 320         optional => 1,
 321         type => 'boolean',
 322         description => "Specifies whether a VM will be started during system bootup.",
 323         default => 0,
 324     },
 325     autostart => {
 326         optional => 1,
 327         type => 'boolean',
 328         description => "Automatic restart after crash (currently ignored).",
 329         default => 0,
 330     },
 331     hotplug => {
 332         optional => 1,
 333         type => 'string', format => 'pve-hotplug-features',
 334         description => "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
 335         default => 'network,disk,usb',
 336     },
 337     reboot => {
 338         optional => 1,
 339         type => 'boolean',
 340         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 341         default => 1,
 342     },
 343     lock => {
 344         optional => 1,
 345         type => 'string',
 346         description => "Lock/unlock the VM.",
 347         enum => [qw(backup clone create migrate rollback snapshot snapshot-delete suspending suspended)],
 348     },
 349     cpulimit => {
 350         optional => 1,
 351         type => 'number',
 352         description => "Limit of CPU usage.",
 353         verbose_description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
 354         minimum => 0,
 355         maximum => 128,
 356         default => 0,
 357     },
 358     cpuunits => {
 359         optional => 1,
 360         type => 'integer',
 361         description => "CPU weight for a VM.",
 362         verbose_description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.",
 363         minimum => 2,
 364         maximum => 262144,
 365         default => 1024,
 366     },
 367     memory => {
 368         optional => 1,
 369         type => 'integer',
 370         description => "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
 371         minimum => 16,
 372         default => 512,
 373     },
 374     balloon => {
 375         optional => 1,
 376         type => 'integer',
 377         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 378         minimum => 0,
 379     },
 380     shares => {
 381         optional => 1,
 382         type => 'integer',
 383         description => "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning. Auto-ballooning is done by pvestatd.",
 384         minimum => 0,
 385         maximum => 50000,
 386         default => 1000,
 387     },
 388     keyboard => {
 389         optional => 1,
 390         type => 'string',
 391         description => "Keybord layout for vnc server. Default is read from the '/etc/pve/datacenter.cfg' configuration file.".
 392                        "It should not be necessary to set it.",
 393         enum => PVE::Tools::kvmkeymaplist(),
 394         default => undef,
 395     },
 396     name => {
 397         optional => 1,
 398         type => 'string', format => 'dns-name',
 399         description => "Set a name for the VM. Only used on the configuration web interface.",
 400     },
 401     scsihw => {
 402         optional => 1,
 403         type => 'string',
 404         description => "SCSI controller model",
 405         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 406         default => 'lsi',
 407     },
 408     description => {
 409         optional => 1,
 410         type => 'string',
 411         description => "Description for the VM. Only used on the configuration web interface. This is saved as comment inside the configuration file.",
 412     },
 413     ostype => {
 414         optional => 1,
 415         type => 'string',
 416         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 l24 l26 solaris)],
 417         description => "Specify guest operating system.",
 418         verbose_description => <<EODESC,
 419 Specify guest operating system. This is used to enable special
 420 optimization/features for specific operating systems:
 421
 422 [horizontal]
 423 other;; unspecified OS
 424 wxp;; Microsoft Windows XP
 425 w2k;; Microsoft Windows 2000
 426 w2k3;; Microsoft Windows 2003
 427 w2k8;; Microsoft Windows 2008
 428 wvista;; Microsoft Windows Vista
 429 win7;; Microsoft Windows 7
 430 win8;; Microsoft Windows 8/2012/2012r2
 431 win10;; Microsoft Windows 10/2016
 432 l24;; Linux 2.4 Kernel
 433 l26;; Linux 2.6/3.X Kernel
 434 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
 435 EODESC
 436     },
 437     boot => {
 438         optional => 1,
 439         type => 'string',
 440         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n).",
 441         pattern => '[acdn]{1,4}',
 442         default => 'cdn',
 443     },
 444     bootdisk => {
 445         optional => 1,
 446         type => 'string', format => 'pve-qm-bootdisk',
 447         description => "Enable booting from specified disk.",
 448         pattern => '(ide|sata|scsi|virtio)\d+',
 449     },
 450     smp => {
 451         optional => 1,
 452         type => 'integer',
 453         description => "The number of CPUs. Please use option -sockets instead.",
 454         minimum => 1,
 455         default => 1,
 456     },
 457     sockets => {
 458         optional => 1,
 459         type => 'integer',
 460         description => "The number of CPU sockets.",
 461         minimum => 1,
 462         default => 1,
 463     },
 464     cores => {
 465         optional => 1,
 466         type => 'integer',
 467         description => "The number of cores per socket.",
 468         minimum => 1,
 469         default => 1,
 470     },
 471     numa => {
 472         optional => 1,
 473         type => 'boolean',
 474         description => "Enable/disable NUMA.",
 475         default => 0,
 476     },
 477     hugepages => {
 478         optional => 1,
 479         type => 'string',
 480         description => "Enable/disable hugepages memory.",
 481         enum => [qw(any 2 1024)],
 482     },
 483     vcpus => {
 484         optional => 1,
 485         type => 'integer',
 486         description => "Number of hotplugged vcpus.",
 487         minimum => 1,
 488         default => 0,
 489     },
 490     acpi => {
 491         optional => 1,
 492         type => 'boolean',
 493         description => "Enable/disable ACPI.",
 494         default => 1,
 495     },
 496     agent => {
 497         optional => 1,
 498         description => "Enable/disable Qemu GuestAgent and its properties.",
 499         type => 'string',
 500         format => $agent_fmt,
 501     },
 502     kvm => {
 503         optional => 1,
 504         type => 'boolean',
 505         description => "Enable/disable KVM hardware virtualization.",
 506         default => 1,
 507     },
 508     tdf => {
 509         optional => 1,
 510         type => 'boolean',
 511         description => "Enable/disable time drift fix.",
 512         default => 0,
 513     },
 514     localtime => {
 515         optional => 1,
 516         type => 'boolean',
 517         description => "Set the real time clock to local time. This is enabled by default if ostype indicates a Microsoft OS.",
 518     },
 519     freeze => {
 520         optional => 1,
 521         type => 'boolean',
 522         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 523     },
 524     vga => {
 525         optional => 1,
 526         type => 'string', format => $vga_fmt,
 527         description => "Configure the VGA hardware.",
 528         verbose_description => "Configure the VGA Hardware. If you want to use ".
 529             "high resolution modes (>= 1280x1024x16) you may need to increase " .
 530             "the vga memory option. Since QEMU 2.9 the default VGA display type " .
 531             "is 'std' for all OS types besides some Windows versions (XP and " .
 532             "older) which use 'cirrus'. The 'qxl' option enables the SPICE " .
 533             "display server. For win* OS you can select how many independent " .
 534             "displays you want, Linux guests can add displays them self.\n".
 535             "You can also run without any graphic card, using a serial device as terminal.",
 536     },
 537     watchdog => {
 538         optional => 1,
 539         type => 'string', format => 'pve-qm-watchdog',
 540         description => "Create a virtual hardware watchdog device.",
 541         verbose_description => "Create a virtual hardware watchdog device. Once enabled" .
 542             " (by a guest action), the watchdog must be periodically polled " .
 543             "by an agent inside the guest or else the watchdog will reset " .
 544             "the guest (or execute the respective action specified)",
 545     },
 546     startdate => {
 547         optional => 1,
 548         type => 'string',
 549         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 550         description => "Set the initial date of the real time clock. Valid format for date are: 'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 551         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 552         default => 'now',
 553     },
 554     startup =>  get_standard_option('pve-startup-order'),
 555     template => {
 556         optional => 1,
 557         type => 'boolean',
 558         description => "Enable/disable Template.",
 559         default => 0,
 560     },
 561     args => {
 562         optional => 1,
 563         type => 'string',
 564         description => "Arbitrary arguments passed to kvm.",
 565         verbose_description => <<EODESCR,
 566 Arbitrary arguments passed to kvm, for example:
 567
 568 args: -no-reboot -no-hpet
 569
 570 NOTE: this option is for experts only.
 571 EODESCR
 572     },
 573     tablet => {
 574         optional => 1,
 575         type => 'boolean',
 576         default => 1,
 577         description => "Enable/disable the USB tablet device.",
 578         verbose_description => "Enable/disable the USB tablet device. This device is " .
 579             "usually needed to allow absolute mouse positioning with VNC. " .
 580             "Else the mouse runs out of sync with normal VNC clients. " .
 581             "If you're running lots of console-only guests on one host, " .
 582             "you may consider disabling this to save some context switches. " .
 583             "This is turned off by default if you use spice (-vga=qxl).",
 584     },
 585     migrate_speed => {
 586         optional => 1,
 587         type => 'integer',
 588         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 589         minimum => 0,
 590         default => 0,
 591     },
 592     migrate_downtime => {
 593         optional => 1,
 594         type => 'number',
 595         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 596         minimum => 0,
 597         default => 0.1,
 598     },
 599     cdrom => {
 600         optional => 1,
 601         type => 'string', format => 'pve-qm-ide',
 602         typetext => '<volume>',
 603         description => "This is an alias for option -ide2",
 604     },
 605     cpu => {
 606         optional => 1,
 607         description => "Emulated CPU type.",
 608         type => 'string',
 609         format => $cpu_fmt,
 610     },
 611     parent => get_standard_option('pve-snapshot-name', {
 612         optional => 1,
 613         description => "Parent snapshot name. This is used internally, and should not be modified.",
 614     }),
 615     snaptime => {
 616         optional => 1,
 617         description => "Timestamp for snapshots.",
 618         type => 'integer',
 619         minimum => 0,
 620     },
 621     vmstate => {
 622         optional => 1,
 623         type => 'string', format => 'pve-volume-id',
 624         description => "Reference to a volume which stores the VM state. This is used internally for snapshots.",
 625     },
 626     vmstatestorage => get_standard_option('pve-storage-id', {
 627         description => "Default storage for VM state volumes/files.",
 628         optional => 1,
 629     }),
 630     runningmachine => get_standard_option('pve-qemu-machine', {
 631         description => "Specifies the Qemu machine type of the running vm. This is used internally for snapshots.",
 632     }),
 633     machine => get_standard_option('pve-qemu-machine'),
 634     arch => {
 635         description => "Virtual processor architecture. Defaults to the host.",
 636         optional => 1,
 637         type => 'string',
 638         enum => [qw(x86_64 aarch64)],
 639     },
 640     smbios1 => {
 641         description => "Specify SMBIOS type 1 fields.",
 642         type => 'string', format => 'pve-qm-smbios1',
 643         maxLength => 512,
 644         optional => 1,
 645     },
 646     protection => {
 647         optional => 1,
 648         type => 'boolean',
 649         description => "Sets the protection flag of the VM. This will disable the remove VM and remove disk operations.",
 650         default => 0,
 651     },
 652     bios => {
 653         optional => 1,
 654         type => 'string',
 655         enum => [ qw(seabios ovmf) ],
 656         description => "Select BIOS implementation.",
 657         default => 'seabios',
 658     },
 659     vmgenid => {
 660         type => 'string',
 661         pattern => '(?:[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}|[01])',
 662         format_description => 'UUID',
 663         description => "Set VM Generation ID. Use '1' to autogenerate on create or update, pass '0' to disable explicitly.",
 664         verbose_description => "The VM generation ID (vmgenid) device exposes a".
 665             " 128-bit integer value identifier to the guest OS. This allows to".
 666             " notify the guest operating system when the virtual machine is".
 667             " executed with a different configuration (e.g. snapshot execution".
 668             " or creation from a template). The guest operating system notices".
 669             " the change, and is then able to react as appropriate by marking".
 670             " its copies of distributed databases as dirty, re-initializing its".
 671             " random number generator, etc.\n".
 672             "Note that auto-creation only works when done throug API/CLI create".
 673             " or update methods, but not when manually editing the config file.",
 674         default => "1 (autogenerated)",
 675         optional => 1,
 676     },
 677     hookscript => {
 678         type => 'string',
 679         format => 'pve-volume-id',
 680         optional => 1,
 681         description => "Script that will be executed during various steps in the vms lifetime.",
 682     },
 683     ivshmem => {
 684         type => 'string',
 685         format => $ivshmem_fmt,
 686         description => "Inter-VM shared memory. Useful for direct communication between VMs, or to the host.",
 687         optional => 1,
 688     },
 689     audio0 => {
 690         type => 'string',
 691         format => $audio_fmt,
 692         description => "Configure a audio device, useful in combination with QXL/Spice.",
 693         optional => 1
 694     },
 695     spice_enhancements => {
 696         type => 'string',
 697         format => $spice_enhancements_fmt,
 698         description => "Configure additional enhancements for SPICE.",
 699         optional => 1
 700     },
 701 };
 702
 703 my $cicustom_fmt = {
 704     meta => {
 705         type => 'string',
 706         optional => 1,
 707         description => 'Specify a custom file containing all meta data passed to the VM via cloud-init. This is provider specific meaning configdrive2 and nocloud differ.',
 708         format => 'pve-volume-id',
 709         format_description => 'volume',
 710     },
 711     network => {
 712         type => 'string',
 713         optional => 1,
 714         description => 'Specify a custom file containing all network data passed to the VM via cloud-init.',
 715         format => 'pve-volume-id',
 716         format_description => 'volume',
 717     },
 718     user => {
 719         type => 'string',
 720         optional => 1,
 721         description => 'Specify a custom file containing all user data passed to the VM via cloud-init.',
 722         format => 'pve-volume-id',
 723         format_description => 'volume',
 724     },
 725 };
 726 PVE::JSONSchema::register_format('pve-qm-cicustom', $cicustom_fmt);
 727
 728 my $confdesc_cloudinit = {
 729     citype => {
 730         optional => 1,
 731         type => 'string',
 732         description => 'Specifies the cloud-init configuration format. The default depends on the configured operating system type (`ostype`. We use the `nocloud` format for Linux, and `configdrive2` for windows.',
 733         enum => ['configdrive2', 'nocloud'],
 734     },
 735     ciuser => {
 736         optional => 1,
 737         type => 'string',
 738         description => "cloud-init: User name to change ssh keys and password for instead of the image's configured default user.",
 739     },
 740     cipassword => {
 741         optional => 1,
 742         type => 'string',
 743         description => 'cloud-init: Password to assign the user. Using this is generally not recommended. Use ssh keys instead. Also note that older cloud-init versions do not support hashed passwords.',
 744     },
 745     cicustom => {
 746         optional => 1,
 747         type => 'string',
 748         description => 'cloud-init: Specify custom files to replace the automatically generated ones at start.',
 749         format => 'pve-qm-cicustom',
 750     },
 751     searchdomain => {
 752         optional => 1,
 753         type => 'string',
 754         description => "cloud-init: Sets DNS search domains for a container. Create will automatically use the setting from the host if neither searchdomain nor nameserver are set.",
 755     },
 756     nameserver => {
 757         optional => 1,
 758         type => 'string', format => 'address-list',
 759         description => "cloud-init: Sets DNS server IP address for a container. Create will automatically use the setting from the host if neither searchdomain nor nameserver are set.",
 760     },
 761     sshkeys => {
 762         optional => 1,
 763         type => 'string',
 764         format => 'urlencoded',
 765         description => "cloud-init: Setup public SSH keys (one key per line, OpenSSH format).",
 766     },
 767 };
 768
 769 # what about other qemu settings ?
 770 #cpu => 'string',
 771 #machine => 'string',
 772 #fda => 'file',
 773 #fdb => 'file',
 774 #mtdblock => 'file',
 775 #sd => 'file',
 776 #pflash => 'file',
 777 #snapshot => 'bool',
 778 #bootp => 'file',
 779 ##tftp => 'dir',
 780 ##smb => 'dir',
 781 #kernel => 'file',
 782 #append => 'string',
 783 #initrd => 'file',
 784 ##soundhw => 'string',
 785
 786 while (my ($k, $v) = each %$confdesc) {
 787     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 788 }
 789
 790 my $MAX_IDE_DISKS = 4;
 791 my $MAX_SCSI_DISKS = 14;
 792 my $MAX_VIRTIO_DISKS = 16;
 793 my $MAX_SATA_DISKS = 6;
 794 my $MAX_USB_DEVICES = 5;
 795 my $MAX_NETS = 32;
 796 my $MAX_UNUSED_DISKS = 256;
 797 my $MAX_HOSTPCI_DEVICES = 16;
 798 my $MAX_SERIAL_PORTS = 4;
 799 my $MAX_PARALLEL_PORTS = 3;
 800 my $MAX_NUMA = 8;
 801
 802 my $numa_fmt = {
 803     cpus => {
 804         type => "string",
 805         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 806         description => "CPUs accessing this NUMA node.",
 807         format_description => "id[-id];...",
 808     },
 809     memory => {
 810         type => "number",
 811         description => "Amount of memory this NUMA node provides.",
 812         optional => 1,
 813     },
 814     hostnodes => {
 815         type => "string",
 816         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 817         description => "Host NUMA nodes to use.",
 818         format_description => "id[-id];...",
 819         optional => 1,
 820     },
 821     policy => {
 822         type => 'string',
 823         enum => [qw(preferred bind interleave)],
 824         description => "NUMA allocation policy.",
 825         optional => 1,
 826     },
 827 };
 828 PVE::JSONSchema::register_format('pve-qm-numanode', $numa_fmt);
 829 my $numadesc = {
 830     optional => 1,
 831     type => 'string', format => $numa_fmt,
 832     description => "NUMA topology.",
 833 };
 834 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 835
 836 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 837     $confdesc->{"numa$i"} = $numadesc;
 838 }
 839
 840 my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000',  'pcnet',  'virtio',
 841                       'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3',
 842                       'e1000-82540em', 'e1000-82544gc', 'e1000-82545em'];
 843 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 844
 845 my $net_fmt_bridge_descr = <<__EOD__;
 846 Bridge to attach the network device to. The Proxmox VE standard bridge
 847 is called 'vmbr0'.
 848
 849 If you do not specify a bridge, we create a kvm user (NATed) network
 850 device, which provides DHCP and DNS services. The following addresses
 851 are used:
 852
 853  10.0.2.2   Gateway
 854  10.0.2.3   DNS Server
 855  10.0.2.4   SMB Server
 856
 857 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 858 __EOD__
 859
 860 my $net_fmt = {
 861     macaddr  => get_standard_option('mac-addr', {
 862         description => "MAC address. That address must be unique withing your network. This is automatically generated if not specified.",
 863     }),
 864     model => {
 865         type => 'string',
 866         description => "Network Card Model. The 'virtio' model provides the best performance with very low CPU overhead. If your guest does not support this driver, it is usually best to use 'e1000'.",
 867         enum => $nic_model_list,
 868         default_key => 1,
 869     },
 870     (map { $_ => { keyAlias => 'model', alias => 'macaddr' }} @$nic_model_list),
 871     bridge => {
 872         type => 'string',
 873         description => $net_fmt_bridge_descr,
 874         format_description => 'bridge',
 875         optional => 1,
 876     },
 877     queues => {
 878         type => 'integer',
 879         minimum => 0, maximum => 16,
 880         description => 'Number of packet queues to be used on the device.',
 881         optional => 1,
 882     },
 883     rate => {
 884         type => 'number',
 885         minimum => 0,
 886         description => "Rate limit in mbps (megabytes per second) as floating point number.",
 887         optional => 1,
 888     },
 889     tag => {
 890         type => 'integer',
 891         minimum => 1, maximum => 4094,
 892         description => 'VLAN tag to apply to packets on this interface.',
 893         optional => 1,
 894     },
 895     trunks => {
 896         type => 'string',
 897         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 898         description => 'VLAN trunks to pass through this interface.',
 899         format_description => 'vlanid[;vlanid...]',
 900         optional => 1,
 901     },
 902     firewall => {
 903         type => 'boolean',
 904         description => 'Whether this interface should be protected by the firewall.',
 905         optional => 1,
 906     },
 907     link_down => {
 908         type => 'boolean',
 909         description => 'Whether this interface should be disconnected (like pulling the plug).',
 910         optional => 1,
 911     },
 912 };
 913
 914 my $netdesc = {
 915     optional => 1,
 916     type => 'string', format => $net_fmt,
 917     description => "Specify network devices.",
 918 };
 919
 920 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 921
 922 my $ipconfig_fmt = {
 923     ip => {
 924         type => 'string',
 925         format => 'pve-ipv4-config',
 926         format_description => 'IPv4Format/CIDR',
 927         description => 'IPv4 address in CIDR format.',
 928         optional => 1,
 929         default => 'dhcp',
 930     },
 931     gw => {
 932         type => 'string',
 933         format => 'ipv4',
 934         format_description => 'GatewayIPv4',
 935         description => 'Default gateway for IPv4 traffic.',
 936         optional => 1,
 937         requires => 'ip',
 938     },
 939     ip6 => {
 940         type => 'string',
 941         format => 'pve-ipv6-config',
 942         format_description => 'IPv6Format/CIDR',
 943         description => 'IPv6 address in CIDR format.',
 944         optional => 1,
 945         default => 'dhcp',
 946     },
 947     gw6 => {
 948         type => 'string',
 949         format => 'ipv6',
 950         format_description => 'GatewayIPv6',
 951         description => 'Default gateway for IPv6 traffic.',
 952         optional => 1,
 953         requires => 'ip6',
 954     },
 955 };
 956 PVE::JSONSchema::register_format('pve-qm-ipconfig', $ipconfig_fmt);
 957 my $ipconfigdesc = {
 958     optional => 1,
 959     type => 'string', format => 'pve-qm-ipconfig',
 960     description => <<'EODESCR',
 961 cloud-init: Specify IP addresses and gateways for the corresponding interface.
 962
 963 IP addresses use CIDR notation, gateways are optional but need an IP of the same type specified.
 964
 965 The special string 'dhcp' can be used for IP addresses to use DHCP, in which case no explicit gateway should be provided.
 966 For IPv6 the special string 'auto' can be used to use stateless autoconfiguration.
 967
 968 If cloud-init is enabled and neither an IPv4 nor an IPv6 address is specified, it defaults to using dhcp on IPv4.
 969 EODESCR
 970 };
 971 PVE::JSONSchema::register_standard_option("pve-qm-ipconfig", $netdesc);
 972
 973 for (my $i = 0; $i < $MAX_NETS; $i++)  {
 974     $confdesc->{"net$i"} = $netdesc;
 975     $confdesc_cloudinit->{"ipconfig$i"} = $ipconfigdesc;
 976 }
 977
 978 foreach my $key (keys %$confdesc_cloudinit) {
 979     $confdesc->{$key} = $confdesc_cloudinit->{$key};
 980 }
 981
 982 PVE::JSONSchema::register_format('pve-volume-id-or-qm-path', \&verify_volume_id_or_qm_path);
 983 sub verify_volume_id_or_qm_path {
 984     my ($volid, $noerr) = @_;
 985
 986     if ($volid eq 'none' || $volid eq 'cdrom' || $volid =~ m|^/|) {
 987         return $volid;
 988     }
 989
 990     # if its neither 'none' nor 'cdrom' nor a path, check if its a volume-id
 991     $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') };
 992     if ($@) {
 993         return undef if $noerr;
 994         die $@;
 995     }
 996     return $volid;
 997 }
 998
 999 my $drivename_hash;
1000
1001 my %drivedesc_base = (
1002     volume => { alias => 'file' },
1003     file => {
1004         type => 'string',
1005         format => 'pve-volume-id-or-qm-path',
1006         default_key => 1,
1007         format_description => 'volume',
1008         description => "The drive's backing volume.",
1009     },
1010     media => {
1011         type => 'string',
1012         enum => [qw(cdrom disk)],
1013         description => "The drive's media type.",
1014         default => 'disk',
1015         optional => 1
1016     },
1017     cyls => {
1018         type => 'integer',
1019         description => "Force the drive's physical geometry to have a specific cylinder count.",
1020         optional => 1
1021     },
1022     heads => {
1023         type => 'integer',
1024         description => "Force the drive's physical geometry to have a specific head count.",
1025         optional => 1
1026     },
1027     secs => {
1028         type => 'integer',
1029         description => "Force the drive's physical geometry to have a specific sector count.",
1030         optional => 1
1031     },
1032     trans => {
1033         type => 'string',
1034         enum => [qw(none lba auto)],
1035         description => "Force disk geometry bios translation mode.",
1036         optional => 1,
1037     },
1038     snapshot => {
1039         type => 'boolean',
1040         description => "Controls qemu's snapshot mode feature."
1041             . " If activated, changes made to the disk are temporary and will"
1042             . " be discarded when the VM is shutdown.",
1043         optional => 1,
1044     },
1045     cache => {
1046         type => 'string',
1047         enum => [qw(none writethrough writeback unsafe directsync)],
1048         description => "The drive's cache mode",
1049         optional => 1,
1050     },
1051     format => get_standard_option('pve-qm-image-format'),
1052     size => {
1053         type => 'string',
1054         format => 'disk-size',
1055         format_description => 'DiskSize',
1056         description => "Disk size. This is purely informational and has no effect.",
1057         optional => 1,
1058     },
1059     backup => {
1060         type => 'boolean',
1061         description => "Whether the drive should be included when making backups.",
1062         optional => 1,
1063     },
1064     replicate => {
1065         type => 'boolean',
1066         description => 'Whether the drive should considered for replication jobs.',
1067         optional => 1,
1068         default => 1,
1069     },
1070     rerror => {
1071         type => 'string',
1072         enum => [qw(ignore report stop)],
1073         description => 'Read error action.',
1074         optional => 1,
1075     },
1076     werror => {
1077         type => 'string',
1078         enum => [qw(enospc ignore report stop)],
1079         description => 'Write error action.',
1080         optional => 1,
1081     },
1082     aio => {
1083         type => 'string',
1084         enum => [qw(native threads)],
1085         description => 'AIO type to use.',
1086         optional => 1,
1087     },
1088     discard => {
1089         type => 'string',
1090         enum => [qw(ignore on)],
1091         description => 'Controls whether to pass discard/trim requests to the underlying storage.',
1092         optional => 1,
1093     },
1094     detect_zeroes => {
1095         type => 'boolean',
1096         description => 'Controls whether to detect and try to optimize writes of zeroes.',
1097         optional => 1,
1098     },
1099     serial => {
1100         type => 'string',
1101         format => 'urlencoded',
1102         format_description => 'serial',
1103         maxLength => 20*3, # *3 since it's %xx url enoded
1104         description => "The drive's reported serial number, url-encoded, up to 20 bytes long.",
1105         optional => 1,
1106     },
1107     shared => {
1108         type => 'boolean',
1109         description => 'Mark this locally-managed volume as available on all nodes',
1110         verbose_description => "Mark this locally-managed volume as available on all nodes.\n\nWARNING: This option does not share the volume automatically, it assumes it is shared already!",
1111         optional => 1,
1112         default => 0,
1113     }
1114 );
1115
1116 my %iothread_fmt = ( iothread => {
1117         type => 'boolean',
1118         description => "Whether to use iothreads for this drive",
1119         optional => 1,
1120 });
1121
1122 my %model_fmt = (
1123     model => {
1124         type => 'string',
1125         format => 'urlencoded',
1126         format_description => 'model',
1127         maxLength => 40*3, # *3 since it's %xx url enoded
1128         description => "The drive's reported model name, url-encoded, up to 40 bytes long.",
1129         optional => 1,
1130     },
1131 );
1132
1133 my %queues_fmt = (
1134     queues => {
1135         type => 'integer',
1136         description => "Number of queues.",
1137         minimum => 2,
1138         optional => 1
1139     }
1140 );
1141
1142 my %scsiblock_fmt = (
1143     scsiblock => {
1144         type => 'boolean',
1145         description => "whether to use scsi-block for full passthrough of host block device\n\nWARNING: can lead to I/O errors in combination with low memory or high memory fragmentation on host",
1146         optional => 1,
1147         default => 0,
1148     },
1149 );
1150
1151 my %ssd_fmt = (
1152     ssd => {
1153         type => 'boolean',
1154         description => "Whether to expose this drive as an SSD, rather than a rotational hard disk.",
1155         optional => 1,
1156     },
1157 );
1158
1159 my %wwn_fmt = (
1160     wwn => {
1161         type => 'string',
1162         pattern => qr/^(0x)[0-9a-fA-F]{16}/,
1163         format_description => 'wwn',
1164         description => "The drive's worldwide name, encoded as 16 bytes hex string, prefixed by '0x'.",
1165         optional => 1,
1166     },
1167 );
1168
1169 my $add_throttle_desc = sub {
1170     my ($key, $type, $what, $unit, $longunit, $minimum) = @_;
1171     my $d = {
1172         type => $type,
1173         format_description => $unit,
1174         description => "Maximum $what in $longunit.",
1175         optional => 1,
1176     };
1177     $d->{minimum} = $minimum if defined($minimum);
1178     $drivedesc_base{$key} = $d;
1179 };
1180 # throughput: (leaky bucket)
1181 $add_throttle_desc->('bps',     'integer', 'r/w speed',   'bps',  'bytes per second');
1182 $add_throttle_desc->('bps_rd',  'integer', 'read speed',  'bps',  'bytes per second');
1183 $add_throttle_desc->('bps_wr',  'integer', 'write speed', 'bps',  'bytes per second');
1184 $add_throttle_desc->('mbps',    'number',  'r/w speed',   'mbps', 'megabytes per second');
1185 $add_throttle_desc->('mbps_rd', 'number',  'read speed',  'mbps', 'megabytes per second');
1186 $add_throttle_desc->('mbps_wr', 'number',  'write speed', 'mbps', 'megabytes per second');
1187 $add_throttle_desc->('iops',    'integer', 'r/w I/O',     'iops', 'operations per second');
1188 $add_throttle_desc->('iops_rd', 'integer', 'read I/O',    'iops', 'operations per second');
1189 $add_throttle_desc->('iops_wr', 'integer', 'write I/O',   'iops', 'operations per second');
1190
1191 # pools: (pool of IO before throttling starts taking effect)
1192 $add_throttle_desc->('mbps_max',    'number',  'unthrottled r/w pool',       'mbps', 'megabytes per second');
1193 $add_throttle_desc->('mbps_rd_max', 'number',  'unthrottled read pool',      'mbps', 'megabytes per second');
1194 $add_throttle_desc->('mbps_wr_max', 'number',  'unthrottled write pool',     'mbps', 'megabytes per second');
1195 $add_throttle_desc->('iops_max',    'integer', 'unthrottled r/w I/O pool',   'iops', 'operations per second');
1196 $add_throttle_desc->('iops_rd_max', 'integer', 'unthrottled read I/O pool',  'iops', 'operations per second');
1197 $add_throttle_desc->('iops_wr_max', 'integer', 'unthrottled write I/O pool', 'iops', 'operations per second');
1198
1199 # burst lengths
1200 $add_throttle_desc->('bps_max_length',     'integer', 'length of I/O bursts',       'seconds', 'seconds', 1);
1201 $add_throttle_desc->('bps_rd_max_length',  'integer', 'length of read I/O bursts',  'seconds', 'seconds', 1);
1202 $add_throttle_desc->('bps_wr_max_length',  'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
1203 $add_throttle_desc->('iops_max_length',    'integer', 'length of I/O bursts',       'seconds', 'seconds', 1);
1204 $add_throttle_desc->('iops_rd_max_length', 'integer', 'length of read I/O bursts',  'seconds', 'seconds', 1);
1205 $add_throttle_desc->('iops_wr_max_length', 'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
1206
1207 # legacy support
1208 $drivedesc_base{'bps_rd_length'} = { alias => 'bps_rd_max_length' };
1209 $drivedesc_base{'bps_wr_length'} = { alias => 'bps_wr_max_length' };
1210 $drivedesc_base{'iops_rd_length'} = { alias => 'iops_rd_max_length' };
1211 $drivedesc_base{'iops_wr_length'} = { alias => 'iops_wr_max_length' };
1212
1213 my $ide_fmt = {
1214     %drivedesc_base,
1215     %model_fmt,
1216     %ssd_fmt,
1217     %wwn_fmt,
1218 };
1219 PVE::JSONSchema::register_format("pve-qm-ide", $ide_fmt);
1220
1221 my $idedesc = {
1222     optional => 1,
1223     type => 'string', format => $ide_fmt,
1224     description => "Use volume as IDE hard disk or CD-ROM (n is 0 to " .($MAX_IDE_DISKS -1) . ").",
1225 };
1226 PVE::JSONSchema::register_standard_option("pve-qm-ide", $idedesc);
1227
1228 my $scsi_fmt = {
1229     %drivedesc_base,
1230     %iothread_fmt,
1231     %queues_fmt,
1232     %scsiblock_fmt,
1233     %ssd_fmt,
1234     %wwn_fmt,
1235 };
1236 my $scsidesc = {
1237     optional => 1,
1238     type => 'string', format => $scsi_fmt,
1239     description => "Use volume as SCSI hard disk or CD-ROM (n is 0 to " . ($MAX_SCSI_DISKS - 1) . ").",
1240 };
1241 PVE::JSONSchema::register_standard_option("pve-qm-scsi", $scsidesc);
1242
1243 my $sata_fmt = {
1244     %drivedesc_base,
1245     %ssd_fmt,
1246     %wwn_fmt,
1247 };
1248 my $satadesc = {
1249     optional => 1,
1250     type => 'string', format => $sata_fmt,
1251     description => "Use volume as SATA hard disk or CD-ROM (n is 0 to " . ($MAX_SATA_DISKS - 1). ").",
1252 };
1253 PVE::JSONSchema::register_standard_option("pve-qm-sata", $satadesc);
1254
1255 my $virtio_fmt = {
1256     %drivedesc_base,
1257     %iothread_fmt,
1258 };
1259 my $virtiodesc = {
1260     optional => 1,
1261     type => 'string', format => $virtio_fmt,
1262     description => "Use volume as VIRTIO hard disk (n is 0 to " . ($MAX_VIRTIO_DISKS - 1) . ").",
1263 };
1264 PVE::JSONSchema::register_standard_option("pve-qm-virtio", $virtiodesc);
1265
1266 my $alldrive_fmt = {
1267     %drivedesc_base,
1268     %iothread_fmt,
1269     %model_fmt,
1270     %queues_fmt,
1271     %scsiblock_fmt,
1272     %ssd_fmt,
1273     %wwn_fmt,
1274 };
1275
1276 my $efidisk_fmt = {
1277     volume => { alias => 'file' },
1278     file => {
1279         type => 'string',
1280         format => 'pve-volume-id-or-qm-path',
1281         default_key => 1,
1282         format_description => 'volume',
1283         description => "The drive's backing volume.",
1284     },
1285     format => get_standard_option('pve-qm-image-format'),
1286     size => {
1287         type => 'string',
1288         format => 'disk-size',
1289         format_description => 'DiskSize',
1290         description => "Disk size. This is purely informational and has no effect.",
1291         optional => 1,
1292     },
1293 };
1294
1295 my $efidisk_desc = {
1296     optional => 1,
1297     type => 'string', format => $efidisk_fmt,
1298     description => "Configure a Disk for storing EFI vars",
1299 };
1300
1301 PVE::JSONSchema::register_standard_option("pve-qm-efidisk", $efidisk_desc);
1302
1303 my $usb_fmt = {
1304     host => {
1305         default_key => 1,
1306         type => 'string', format => 'pve-qm-usb-device',
1307         format_description => 'HOSTUSBDEVICE|spice',
1308         description => <<EODESCR,
1309 The Host USB device or port or the value 'spice'. HOSTUSBDEVICE syntax is:
1310
1311  'bus-port(.port)*' (decimal numbers) or
1312  'vendor_id:product_id' (hexadeciaml numbers) or
1313  'spice'
1314
1315 You can use the 'lsusb -t' command to list existing usb devices.
1316
1317 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1318
1319 The value 'spice' can be used to add a usb redirection devices for spice.
1320 EODESCR
1321     },
1322     usb3 => {
1323         optional => 1,
1324         type => 'boolean',
1325         description => "Specifies whether if given host option is a USB3 device or port.",
1326         default => 0,
1327     },
1328 };
1329
1330 my $usbdesc = {
1331     optional => 1,
1332     type => 'string', format => $usb_fmt,
1333     description => "Configure an USB device (n is 0 to 4).",
1334 };
1335 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
1336
1337 my $PCIRE = qr/[a-f0-9]{2}:[a-f0-9]{2}(?:\.[a-f0-9])?/;
1338 my $hostpci_fmt = {
1339     host => {
1340         default_key => 1,
1341         type => 'string',
1342         pattern => qr/$PCIRE(;$PCIRE)*/,
1343         format_description => 'HOSTPCIID[;HOSTPCIID2...]',
1344         description => <<EODESCR,
1345 Host PCI device pass through. The PCI ID of a host's PCI device or a list
1346 of PCI virtual functions of the host. HOSTPCIID syntax is:
1347
1348 'bus:dev.func' (hexadecimal numbers)
1349
1350 You can us the 'lspci' command to list existing PCI devices.
1351 EODESCR
1352     },
1353     rombar => {
1354         type => 'boolean',
1355         description =>  "Specify whether or not the device's ROM will be visible in the guest's memory map.",
1356         optional => 1,
1357         default => 1,
1358     },
1359     romfile => {
1360         type => 'string',
1361         pattern => '[^,;]+',
1362         format_description => 'string',
1363         description => "Custom pci device rom filename (must be located in /usr/share/kvm/).",
1364         optional => 1,
1365     },
1366     pcie => {
1367         type => 'boolean',
1368         description =>  "Choose the PCI-express bus (needs the 'q35' machine model).",
1369         optional => 1,
1370         default => 0,
1371     },
1372     'x-vga' => {
1373         type => 'boolean',
1374         description =>  "Enable vfio-vga device support.",
1375         optional => 1,
1376         default => 0,
1377     },
1378     'mdev' => {
1379         type => 'string',
1380         format_description => 'string',
1381         pattern => '[^/\.:]+',
1382         optional => 1,
1383         description => <<EODESCR
1384 The type of mediated device to use.
1385 An instance of this type will be created on startup of the VM and
1386 will be cleaned up when the VM stops.
1387 EODESCR
1388     }
1389 };
1390 PVE::JSONSchema::register_format('pve-qm-hostpci', $hostpci_fmt);
1391
1392 my $hostpcidesc = {
1393         optional => 1,
1394         type => 'string', format => 'pve-qm-hostpci',
1395         description => "Map host PCI devices into guest.",
1396         verbose_description =>  <<EODESCR,
1397 Map host PCI devices into guest.
1398
1399 NOTE: This option allows direct access to host hardware. So it is no longer
1400 possible to migrate such machines - use with special care.
1401
1402 CAUTION: Experimental! User reported problems with this option.
1403 EODESCR
1404 };
1405 PVE::JSONSchema::register_standard_option("pve-qm-hostpci", $hostpcidesc);
1406
1407 my $serialdesc = {
1408         optional => 1,
1409         type => 'string',
1410         pattern => '(/dev/.+|socket)',
1411         description =>  "Create a serial device inside the VM (n is 0 to 3)",
1412         verbose_description =>  <<EODESCR,
1413 Create a serial device inside the VM (n is 0 to 3), and pass through a
1414 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1415 host side (use 'qm terminal' to open a terminal connection).
1416
1417 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines - use with special care.
1418
1419 CAUTION: Experimental! User reported problems with this option.
1420 EODESCR
1421 };
1422
1423 my $paralleldesc= {
1424         optional => 1,
1425         type => 'string',
1426         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
1427         description =>  "Map host parallel devices (n is 0 to 2).",
1428         verbose_description =>  <<EODESCR,
1429 Map host parallel devices (n is 0 to 2).
1430
1431 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1432
1433 CAUTION: Experimental! User reported problems with this option.
1434 EODESCR
1435 };
1436
1437 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
1438     $confdesc->{"parallel$i"} = $paralleldesc;
1439 }
1440
1441 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
1442     $confdesc->{"serial$i"} = $serialdesc;
1443 }
1444
1445 for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
1446     $confdesc->{"hostpci$i"} = $hostpcidesc;
1447 }
1448
1449 for (my $i = 0; $i < $MAX_IDE_DISKS; $i++)  {
1450     $drivename_hash->{"ide$i"} = 1;
1451     $confdesc->{"ide$i"} = $idedesc;
1452 }
1453
1454 for (my $i = 0; $i < $MAX_SATA_DISKS; $i++)  {
1455     $drivename_hash->{"sata$i"} = 1;
1456     $confdesc->{"sata$i"} = $satadesc;
1457 }
1458
1459 for (my $i = 0; $i < $MAX_SCSI_DISKS; $i++)  {
1460     $drivename_hash->{"scsi$i"} = 1;
1461     $confdesc->{"scsi$i"} = $scsidesc ;
1462 }
1463
1464 for (my $i = 0; $i < $MAX_VIRTIO_DISKS; $i++)  {
1465     $drivename_hash->{"virtio$i"} = 1;
1466     $confdesc->{"virtio$i"} = $virtiodesc;
1467 }
1468
1469 $drivename_hash->{efidisk0} = 1;
1470 $confdesc->{efidisk0} = $efidisk_desc;
1471
1472 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
1473     $confdesc->{"usb$i"} = $usbdesc;
1474 }
1475
1476 my $unuseddesc = {
1477     optional => 1,
1478     type => 'string', format => 'pve-volume-id',
1479     description => "Reference to unused volumes. This is used internally, and should not be modified manually.",
1480 };
1481
1482 for (my $i = 0; $i < $MAX_UNUSED_DISKS; $i++)  {
1483     $confdesc->{"unused$i"} = $unuseddesc;
1484 }
1485
1486 my $kvm_api_version = 0;
1487
1488 sub kvm_version {
1489     return $kvm_api_version if $kvm_api_version;
1490
1491     open my $fh, '<', '/dev/kvm'
1492         or return undef;
1493
1494     # 0xae00 => KVM_GET_API_VERSION
1495     $kvm_api_version = ioctl($fh, 0xae00, 0);
1496
1497     return $kvm_api_version;
1498 }
1499
1500 my $kvm_user_version = {};
1501 my $kvm_mtime = {};
1502
1503 sub kvm_user_version {
1504     my ($binary) = @_;
1505
1506     $binary //= get_command_for_arch(get_host_arch()); # get the native arch by default
1507     my $st = stat($binary);
1508
1509     my $cachedmtime = $kvm_mtime->{$binary} // -1;
1510     return $kvm_user_version->{$binary} if $kvm_user_version->{$binary} &&
1511         $cachedmtime == $st->mtime;
1512
1513     $kvm_user_version->{$binary} = 'unknown';
1514     $kvm_mtime->{$binary} = $st->mtime;
1515
1516     my $code = sub {
1517         my $line = shift;
1518         if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1519             $kvm_user_version->{$binary} = $2;
1520         }
1521     };
1522
1523     eval { run_command([$binary, '--version'], outfunc => $code); };
1524     warn $@ if $@;
1525
1526     return $kvm_user_version->{$binary};
1527
1528 }
1529
1530 sub kernel_has_vhost_net {
1531     return -c '/dev/vhost-net';
1532 }
1533
1534 sub valid_drive_names {
1535     # order is important - used to autoselect boot disk
1536     return ((map { "ide$_" } (0 .. ($MAX_IDE_DISKS - 1))),
1537             (map { "scsi$_" } (0 .. ($MAX_SCSI_DISKS - 1))),
1538             (map { "virtio$_" } (0 .. ($MAX_VIRTIO_DISKS - 1))),
1539             (map { "sata$_" } (0 .. ($MAX_SATA_DISKS - 1))),
1540             'efidisk0');
1541 }
1542
1543 sub is_valid_drivename {
1544     my $dev = shift;
1545
1546     return defined($drivename_hash->{$dev});
1547 }
1548
1549 sub option_exists {
1550     my $key = shift;
1551     return defined($confdesc->{$key});
1552 }
1553
1554 sub nic_models {
1555     return $nic_model_list;
1556 }
1557
1558 sub os_list_description {
1559
1560     return {
1561         other => 'Other',
1562         wxp => 'Windows XP',
1563         w2k => 'Windows 2000',
1564         w2k3 =>, 'Windows 2003',
1565         w2k8 => 'Windows 2008',
1566         wvista => 'Windows Vista',
1567         win7 => 'Windows 7',
1568         win8 => 'Windows 8/2012',
1569         win10 => 'Windows 10/2016',
1570         l24 => 'Linux 2.4',
1571         l26 => 'Linux 2.6',
1572     };
1573 }
1574
1575 my $cdrom_path;
1576
1577 sub get_cdrom_path {
1578
1579     return  $cdrom_path if $cdrom_path;
1580
1581     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
1582     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
1583     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
1584 }
1585
1586 sub get_iso_path {
1587     my ($storecfg, $vmid, $cdrom) = @_;
1588
1589     if ($cdrom eq 'cdrom') {
1590         return get_cdrom_path();
1591     } elsif ($cdrom eq 'none') {
1592         return '';
1593     } elsif ($cdrom =~ m|^/|) {
1594         return $cdrom;
1595     } else {
1596         return PVE::Storage::path($storecfg, $cdrom);
1597     }
1598 }
1599
1600 # try to convert old style file names to volume IDs
1601 sub filename_to_volume_id {
1602     my ($vmid, $file, $media) = @_;
1603
1604      if (!($file eq 'none' || $file eq 'cdrom' ||
1605           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1606
1607         return undef if $file =~ m|/|;
1608
1609         if ($media && $media eq 'cdrom') {
1610             $file = "local:iso/$file";
1611         } else {
1612             $file = "local:$vmid/$file";
1613         }
1614     }
1615
1616     return $file;
1617 }
1618
1619 sub verify_media_type {
1620     my ($opt, $vtype, $media) = @_;
1621
1622     return if !$media;
1623
1624     my $etype;
1625     if ($media eq 'disk') {
1626         $etype = 'images';
1627     } elsif ($media eq 'cdrom') {
1628         $etype = 'iso';
1629     } else {
1630         die "internal error";
1631     }
1632
1633     return if ($vtype eq $etype);
1634
1635     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1636 }
1637
1638 sub cleanup_drive_path {
1639     my ($opt, $storecfg, $drive) = @_;
1640
1641     # try to convert filesystem paths to volume IDs
1642
1643     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1644         ($drive->{file} !~ m|^/dev/.+|) &&
1645         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1646         ($drive->{file} !~ m/^\d+$/)) {
1647         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1648         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"}) if !$vtype;
1649         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1650         verify_media_type($opt, $vtype, $drive->{media});
1651         $drive->{file} = $volid;
1652     }
1653
1654     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1655 }
1656
1657 sub parse_hotplug_features {
1658     my ($data) = @_;
1659
1660     my $res = {};
1661
1662     return $res if $data eq '0';
1663
1664     $data = $confdesc->{hotplug}->{default} if $data eq '1';
1665
1666     foreach my $feature (PVE::Tools::split_list($data)) {
1667         if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
1668             $res->{$1} = 1;
1669         } else {
1670             die "invalid hotplug feature '$feature'\n";
1671         }
1672     }
1673     return $res;
1674 }
1675
1676 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1677 sub pve_verify_hotplug_features {
1678     my ($value, $noerr) = @_;
1679
1680     return $value if parse_hotplug_features($value);
1681
1682     return undef if $noerr;
1683
1684     die "unable to parse hotplug option\n";
1685 }
1686
1687 # ideX = [volume=]volume-id[,media=d][,cyls=c,heads=h,secs=s[,trans=t]]
1688 #        [,snapshot=on|off][,cache=on|off][,format=f][,backup=yes|no]
1689 #        [,rerror=ignore|report|stop][,werror=enospc|ignore|report|stop]
1690 #        [,aio=native|threads][,discard=ignore|on][,detect_zeroes=on|off]
1691 #        [,iothread=on][,serial=serial][,model=model]
1692
1693 sub parse_drive {
1694     my ($key, $data) = @_;
1695
1696     my ($interface, $index);
1697
1698     if ($key =~ m/^([^\d]+)(\d+)$/) {
1699         $interface = $1;
1700         $index = $2;
1701     } else {
1702         return undef;
1703     }
1704
1705     my $desc = $key =~ /^unused\d+$/ ? $alldrive_fmt
1706                                      : $confdesc->{$key}->{format};
1707     if (!$desc) {
1708         warn "invalid drive key: $key\n";
1709         return undef;
1710     }
1711     my $res = eval { PVE::JSONSchema::parse_property_string($desc, $data) };
1712     return undef if !$res;
1713     $res->{interface} = $interface;
1714     $res->{index} = $index;
1715
1716     my $error = 0;
1717     foreach my $opt (qw(bps bps_rd bps_wr)) {
1718         if (my $bps = defined(delete $res->{$opt})) {
1719             if (defined($res->{"m$opt"})) {
1720                 warn "both $opt and m$opt specified\n";
1721                 ++$error;
1722                 next;
1723             }
1724             $res->{"m$opt"} = sprintf("%.3f", $bps / (1024*1024.0));
1725         }
1726     }
1727
1728     # can't use the schema's 'requires' because of the mbps* => bps* "transforming aliases"
1729     for my $requirement (
1730         [mbps_max => 'mbps'],
1731         [mbps_rd_max => 'mbps_rd'],
1732         [mbps_wr_max => 'mbps_wr'],
1733         [miops_max => 'miops'],
1734         [miops_rd_max => 'miops_rd'],
1735         [miops_wr_max => 'miops_wr'],
1736         [bps_max_length => 'mbps_max'],
1737         [bps_rd_max_length => 'mbps_rd_max'],
1738         [bps_wr_max_length => 'mbps_wr_max'],
1739         [iops_max_length => 'iops_max'],
1740         [iops_rd_max_length => 'iops_rd_max'],
1741         [iops_wr_max_length => 'iops_wr_max']) {
1742         my ($option, $requires) = @$requirement;
1743         if ($res->{$option} && !$res->{$requires}) {
1744             warn "$option requires $requires\n";
1745             ++$error;
1746         }
1747     }
1748
1749     return undef if $error;
1750
1751     return undef if $res->{mbps_rd} && $res->{mbps};
1752     return undef if $res->{mbps_wr} && $res->{mbps};
1753     return undef if $res->{iops_rd} && $res->{iops};
1754     return undef if $res->{iops_wr} && $res->{iops};
1755
1756     if ($res->{media} && ($res->{media} eq 'cdrom')) {
1757         return undef if $res->{snapshot} || $res->{trans} || $res->{format};
1758         return undef if $res->{heads} || $res->{secs} || $res->{cyls};
1759         return undef if $res->{interface} eq 'virtio';
1760     }
1761
1762     if (my $size = $res->{size}) {
1763         return undef if !defined($res->{size} = PVE::JSONSchema::parse_size($size));
1764     }
1765
1766     return $res;
1767 }
1768
1769 sub print_drive {
1770     my ($vmid, $drive) = @_;
1771     my $data = { %$drive };
1772     delete $data->{$_} for qw(index interface);
1773     return PVE::JSONSchema::print_property_string($data, $alldrive_fmt);
1774 }
1775
1776 sub scsi_inquiry {
1777     my($fh, $noerr) = @_;
1778
1779     my $SG_IO = 0x2285;
1780     my $SG_GET_VERSION_NUM = 0x2282;
1781
1782     my $versionbuf = "\x00" x 8;
1783     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1784     if (!$ret) {
1785         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1786         return undef;
1787     }
1788     my $version = unpack("I", $versionbuf);
1789     if ($version < 30000) {
1790         die "scsi generic interface too old\n"  if !$noerr;
1791         return undef;
1792     }
1793
1794     my $buf = "\x00" x 36;
1795     my $sensebuf = "\x00" x 8;
1796     my $cmd = pack("C x3 C x1", 0x12, 36);
1797
1798     # see /usr/include/scsi/sg.h
1799     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1800
1801     my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1802                       length($sensebuf), 0, length($buf), $buf,
1803                       $cmd, $sensebuf, 6000);
1804
1805     $ret = ioctl($fh, $SG_IO, $packet);
1806     if (!$ret) {
1807         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1808         return undef;
1809     }
1810
1811     my @res = unpack($sg_io_hdr_t, $packet);
1812     if ($res[17] || $res[18]) {
1813         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1814         return undef;
1815     }
1816
1817     my $res = {};
1818     (my $byte0, my $byte1, $res->{vendor},
1819      $res->{product}, $res->{revision}) = unpack("C C x6 A8 A16 A4", $buf);
1820
1821     $res->{removable} = $byte1 & 128 ? 1 : 0;
1822     $res->{type} = $byte0 & 31;
1823
1824     return $res;
1825 }
1826
1827 sub path_is_scsi {
1828     my ($path) = @_;
1829
1830     my $fh = IO::File->new("+<$path") || return undef;
1831     my $res = scsi_inquiry($fh, 1);
1832     close($fh);
1833
1834     return $res;
1835 }
1836
1837 sub machine_type_is_q35 {
1838     my ($conf) = @_;
1839
1840     return $conf->{machine} && ($conf->{machine} =~ m/q35/) ? 1 : 0;
1841 }
1842
1843 sub print_tabletdevice_full {
1844     my ($conf, $arch) = @_;
1845
1846     my $q35 = machine_type_is_q35($conf);
1847
1848     # we use uhci for old VMs because tablet driver was buggy in older qemu
1849     my $usbbus;
1850     if (machine_type_is_q35($conf) || $arch eq 'aarch64') {
1851         $usbbus = 'ehci';
1852     } else {
1853         $usbbus = 'uhci';
1854     }
1855
1856     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1857 }
1858
1859 sub print_keyboarddevice_full {
1860     my ($conf, $arch, $machine) = @_;
1861
1862     return undef if $arch ne 'aarch64';
1863
1864     return "usb-kbd,id=keyboard,bus=ehci.0,port=2";
1865 }
1866
1867 sub print_drivedevice_full {
1868     my ($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type) = @_;
1869
1870     my $device = '';
1871     my $maxdev = 0;
1872
1873     if ($drive->{interface} eq 'virtio') {
1874         my $pciaddr = print_pci_addr("$drive->{interface}$drive->{index}", $bridges, $arch, $machine_type);
1875         $device = "virtio-blk-pci,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}$pciaddr";
1876         $device .= ",iothread=iothread-$drive->{interface}$drive->{index}" if $drive->{iothread};
1877     } elsif ($drive->{interface} eq 'scsi') {
1878
1879         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1880         my $unit = $drive->{index} % $maxdev;
1881         my $devicetype = 'hd';
1882         my $path = '';
1883         if (drive_is_cdrom($drive)) {
1884             $devicetype = 'cd';
1885         } else {
1886             if ($drive->{file} =~ m|^/|) {
1887                 $path = $drive->{file};
1888                 if (my $info = path_is_scsi($path)) {
1889                     if ($info->{type} == 0 && $drive->{scsiblock}) {
1890                         $devicetype = 'block';
1891                     } elsif ($info->{type} == 1) { # tape
1892                         $devicetype = 'generic';
1893                     }
1894                 }
1895             } else {
1896                  $path = PVE::Storage::path($storecfg, $drive->{file});
1897             }
1898
1899             if($path =~ m/^iscsi\:\/\//){
1900                 $devicetype = 'generic';
1901             }
1902         }
1903
1904         if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)){
1905             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1906         } else {
1907             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0,lun=$drive->{index},drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1908         }
1909
1910         if ($drive->{ssd} && ($devicetype eq 'block' || $devicetype eq 'hd')) {
1911             $device .= ",rotation_rate=1";
1912         }
1913         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1914
1915     } elsif ($drive->{interface} eq 'ide' || $drive->{interface} eq 'sata') {
1916         my $maxdev = ($drive->{interface} eq 'sata') ? $MAX_SATA_DISKS : 2;
1917         my $controller = int($drive->{index} / $maxdev);
1918         my $unit = $drive->{index} % $maxdev;
1919         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1920
1921         $device = "ide-$devicetype";
1922         if ($drive->{interface} eq 'ide') {
1923             $device .= ",bus=ide.$controller,unit=$unit";
1924         } else {
1925             $device .= ",bus=ahci$controller.$unit";
1926         }
1927         $device .= ",drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1928
1929         if ($devicetype eq 'hd') {
1930             if (my $model = $drive->{model}) {
1931                 $model = URI::Escape::uri_unescape($model);
1932                 $device .= ",model=$model";
1933             }
1934             if ($drive->{ssd}) {
1935                 $device .= ",rotation_rate=1";
1936             }
1937         }
1938         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1939     } elsif ($drive->{interface} eq 'usb') {
1940         die "implement me";
1941         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1942     } else {
1943         die "unsupported interface type";
1944     }
1945
1946     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1947
1948     if (my $serial = $drive->{serial}) {
1949         $serial = URI::Escape::uri_unescape($serial);
1950         $device .= ",serial=$serial";
1951     }
1952
1953
1954     return $device;
1955 }
1956
1957 sub get_initiator_name {
1958     my $initiator;
1959
1960     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return undef;
1961     while (defined(my $line = <$fh>)) {
1962         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1963         $initiator = $1;
1964         last;
1965     }
1966     $fh->close();
1967
1968     return $initiator;
1969 }
1970
1971 sub print_drive_full {
1972     my ($storecfg, $vmid, $drive) = @_;
1973
1974     my $path;
1975     my $volid = $drive->{file};
1976     my $format;
1977
1978     if (drive_is_cdrom($drive)) {
1979         $path = get_iso_path($storecfg, $vmid, $volid);
1980     } else {
1981         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1982         if ($storeid) {
1983             $path = PVE::Storage::path($storecfg, $volid);
1984             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
1985             $format = qemu_img_format($scfg, $volname);
1986         } else {
1987             $path = $volid;
1988             $format = "raw";
1989         }
1990    }
1991
1992     my $opts = '';
1993     my @qemu_drive_options = qw(heads secs cyls trans media format cache rerror werror aio discard);
1994     foreach my $o (@qemu_drive_options) {
1995         $opts .= ",$o=$drive->{$o}" if defined($drive->{$o});
1996     }
1997
1998     # snapshot only accepts on|off
1999     if (defined($drive->{snapshot})) {
2000         my $v = $drive->{snapshot} ? 'on' : 'off';
2001         $opts .= ",snapshot=$v";
2002     }
2003
2004     foreach my $type (['', '-total'], [_rd => '-read'], [_wr => '-write']) {
2005         my ($dir, $qmpname) = @$type;
2006         if (my $v = $drive->{"mbps$dir"}) {
2007             $opts .= ",throttling.bps$qmpname=".int($v*1024*1024);
2008         }
2009         if (my $v = $drive->{"mbps${dir}_max"}) {
2010             $opts .= ",throttling.bps$qmpname-max=".int($v*1024*1024);
2011         }
2012         if (my $v = $drive->{"bps${dir}_max_length"}) {
2013             $opts .= ",throttling.bps$qmpname-max-length=$v";
2014         }
2015         if (my $v = $drive->{"iops${dir}"}) {
2016             $opts .= ",throttling.iops$qmpname=$v";
2017         }
2018         if (my $v = $drive->{"iops${dir}_max"}) {
2019             $opts .= ",throttling.iops$qmpname-max=$v";
2020         }
2021         if (my $v = $drive->{"iops${dir}_max_length"}) {
2022             $opts .= ",throttling.iops$qmpname-max-length=$v";
2023         }
2024     }
2025
2026     $opts .= ",format=$format" if $format && !$drive->{format};
2027
2028     my $cache_direct = 0;
2029
2030     if (my $cache = $drive->{cache}) {
2031         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
2032     } elsif (!drive_is_cdrom($drive)) {
2033         $opts .= ",cache=none";
2034         $cache_direct = 1;
2035     }
2036
2037     # aio native works only with O_DIRECT
2038     if (!$drive->{aio}) {
2039         if($cache_direct) {
2040             $opts .= ",aio=native";
2041         } else {
2042             $opts .= ",aio=threads";
2043         }
2044     }
2045
2046     if (!drive_is_cdrom($drive)) {
2047         my $detectzeroes;
2048         if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
2049             $detectzeroes = 'off';
2050         } elsif ($drive->{discard}) {
2051             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
2052         } else {
2053             # This used to be our default with discard not being specified:
2054             $detectzeroes = 'on';
2055         }
2056         $opts .= ",detect-zeroes=$detectzeroes" if $detectzeroes;
2057     }
2058
2059     my $pathinfo = $path ? "file=$path," : '';
2060
2061     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
2062 }
2063
2064 sub print_netdevice_full {
2065     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files, $arch, $machine_type) = @_;
2066
2067     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
2068
2069     my $device = $net->{model};
2070     if ($net->{model} eq 'virtio') {
2071          $device = 'virtio-net-pci';
2072      };
2073
2074     my $pciaddr = print_pci_addr("$netid", $bridges, $arch, $machine_type);
2075     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
2076     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
2077         #Consider we have N queues, the number of vectors needed is 2*N + 2 (plus one config interrupt and control vq)
2078         my $vectors = $net->{queues} * 2 + 2;
2079         $tmpstr .= ",vectors=$vectors,mq=on";
2080     }
2081     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
2082
2083     if ($use_old_bios_files) {
2084         my $romfile;
2085         if ($device eq 'virtio-net-pci') {
2086             $romfile = 'pxe-virtio.rom';
2087         } elsif ($device eq 'e1000') {
2088             $romfile = 'pxe-e1000.rom';
2089         } elsif ($device eq 'ne2k') {
2090             $romfile = 'pxe-ne2k_pci.rom';
2091         } elsif ($device eq 'pcnet') {
2092             $romfile = 'pxe-pcnet.rom';
2093         } elsif ($device eq 'rtl8139') {
2094             $romfile = 'pxe-rtl8139.rom';
2095         }
2096         $tmpstr .= ",romfile=$romfile" if $romfile;
2097     }
2098
2099     return $tmpstr;
2100 }
2101
2102 sub print_netdev_full {
2103     my ($vmid, $conf, $arch, $net, $netid, $hotplug) = @_;
2104
2105     my $i = '';
2106     if ($netid =~ m/^net(\d+)$/) {
2107         $i = int($1);
2108     }
2109
2110     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
2111
2112     my $ifname = "tap${vmid}i$i";
2113
2114     # kvm uses TUNSETIFF ioctl, and that limits ifname length
2115     die "interface name '$ifname' is too long (max 15 character)\n"
2116         if length($ifname) >= 16;
2117
2118     my $vhostparam = '';
2119     if (is_native($arch)) {
2120         $vhostparam = ',vhost=on' if kernel_has_vhost_net() && $net->{model} eq 'virtio';
2121     }
2122
2123     my $vmname = $conf->{name} || "vm$vmid";
2124
2125     my $netdev = "";
2126     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
2127
2128     if ($net->{bridge}) {
2129         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script,downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
2130     } else {
2131         $netdev = "type=user,id=$netid,hostname=$vmname";
2132     }
2133
2134     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
2135
2136     return $netdev;
2137 }
2138
2139
2140 sub print_cpu_device {
2141     my ($conf, $id) = @_;
2142
2143     my $kvm = $conf->{kvm} // 1;
2144     my $cpu = $kvm ? "kvm64" : "qemu64";
2145     if (my $cputype = $conf->{cpu}) {
2146         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
2147             or die "Cannot parse cpu description: $cputype\n";
2148         $cpu = $cpuconf->{cputype};
2149     }
2150
2151     my $cores = $conf->{cores} || 1;
2152
2153     my $current_core = ($id - 1) % $cores;
2154     my $current_socket = int(($id - 1 - $current_core)/$cores);
2155
2156     return "$cpu-x86_64-cpu,id=cpu$id,socket-id=$current_socket,core-id=$current_core,thread-id=0";
2157 }
2158
2159 my $vga_map = {
2160     'cirrus' => 'cirrus-vga',
2161     'std' => 'VGA',
2162     'vmware' => 'vmware-svga',
2163     'virtio' => 'virtio-vga',
2164 };
2165
2166 sub print_vga_device {
2167     my ($conf, $vga, $arch, $machine, $id, $qxlnum, $bridges) = @_;
2168
2169     my $type = $vga_map->{$vga->{type}};
2170     if ($arch eq 'aarch64' && defined($type) && $type eq 'virtio-vga') {
2171         $type = 'virtio-gpu';
2172     }
2173     my $vgamem_mb = $vga->{memory};
2174     if ($qxlnum) {
2175         $type = $id ? 'qxl' : 'qxl-vga';
2176     }
2177     die "no devicetype for $vga->{type}\n" if !$type;
2178
2179     my $memory = "";
2180     if ($vgamem_mb) {
2181         if ($vga->{type} eq 'virtio') {
2182             my $bytes = PVE::Tools::convert_size($vgamem_mb, "mb" => "b");
2183             $memory = ",max_hostmem=$bytes";
2184         } elsif ($qxlnum) {
2185             # from https://www.spice-space.org/multiple-monitors.html
2186             $memory = ",vgamem_mb=$vga->{memory}";
2187             my $ram = $vgamem_mb * 4;
2188             my $vram = $vgamem_mb * 2;
2189             $memory .= ",ram_size_mb=$ram,vram_size_mb=$vram";
2190         } else {
2191             $memory = ",vgamem_mb=$vga->{memory}";
2192         }
2193     } elsif ($qxlnum && $id) {
2194         $memory = ",ram_size=67108864,vram_size=33554432";
2195     }
2196
2197     my $q35 = machine_type_is_q35($conf);
2198     my $vgaid = "vga" . ($id // '');
2199     my $pciaddr;
2200
2201     if ($q35 && $vgaid eq 'vga') {
2202         # the first display uses pcie.0 bus on q35 machines
2203         $pciaddr = print_pcie_addr($vgaid, $bridges, $arch, $machine);
2204     } else {
2205         $pciaddr = print_pci_addr($vgaid, $bridges, $arch, $machine);
2206     }
2207
2208     return "$type,id=${vgaid}${memory}${pciaddr}";
2209 }
2210
2211 sub drive_is_cloudinit {
2212     my ($drive) = @_;
2213     return $drive->{file} =~ m@[:/]vm-\d+-cloudinit(?:\.$QEMU_FORMAT_RE)?$@;
2214 }
2215
2216 sub drive_is_cdrom {
2217     my ($drive, $exclude_cloudinit) = @_;
2218
2219     return 0 if $exclude_cloudinit && drive_is_cloudinit($drive);
2220
2221     return $drive && $drive->{media} && ($drive->{media} eq 'cdrom');
2222
2223 }
2224
2225 sub parse_number_sets {
2226     my ($set) = @_;
2227     my $res = [];
2228     foreach my $part (split(/;/, $set)) {
2229         if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
2230             die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
2231             push @$res, [ $1, $2 ];
2232         } else {
2233             die "invalid range: $part\n";
2234         }
2235     }
2236     return $res;
2237 }
2238
2239 sub parse_numa {
2240     my ($data) = @_;
2241
2242     my $res = PVE::JSONSchema::parse_property_string($numa_fmt, $data);
2243     $res->{cpus} = parse_number_sets($res->{cpus}) if defined($res->{cpus});
2244     $res->{hostnodes} = parse_number_sets($res->{hostnodes}) if defined($res->{hostnodes});
2245     return $res;
2246 }
2247
2248 sub parse_hostpci {
2249     my ($value) = @_;
2250
2251     return undef if !$value;
2252
2253     my $res = PVE::JSONSchema::parse_property_string($hostpci_fmt, $value);
2254
2255     my @idlist = split(/;/, $res->{host});
2256     delete $res->{host};
2257     foreach my $id (@idlist) {
2258         if ($id =~ m/\./) { # full id 00:00.1
2259             push @{$res->{pciid}}, {
2260                 id => $id,
2261             };
2262         } else { # partial id 00:00
2263             $res->{pciid} = PVE::SysFSTools::lspci($id);
2264         }
2265     }
2266     return $res;
2267 }
2268
2269 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
2270 sub parse_net {
2271     my ($data) = @_;
2272
2273     my $res = eval { PVE::JSONSchema::parse_property_string($net_fmt, $data) };
2274     if ($@) {
2275         warn $@;
2276         return undef;
2277     }
2278     if (!defined($res->{macaddr})) {
2279         my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
2280         $res->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix});
2281     }
2282     return $res;
2283 }
2284
2285 # ipconfigX ip=cidr,gw=ip,ip6=cidr,gw6=ip
2286 sub parse_ipconfig {
2287     my ($data) = @_;
2288
2289     my $res = eval { PVE::JSONSchema::parse_property_string($ipconfig_fmt, $data) };
2290     if ($@) {
2291         warn $@;
2292         return undef;
2293     }
2294
2295     if ($res->{gw} && !$res->{ip}) {
2296         warn 'gateway specified without specifying an IP address';
2297         return undef;
2298     }
2299     if ($res->{gw6} && !$res->{ip6}) {
2300         warn 'IPv6 gateway specified without specifying an IPv6 address';
2301         return undef;
2302     }
2303     if ($res->{gw} && $res->{ip} eq 'dhcp') {
2304         warn 'gateway specified together with DHCP';
2305         return undef;
2306     }
2307     if ($res->{gw6} && $res->{ip6} !~ /^$IPV6RE/) {
2308         # gw6 + auto/dhcp
2309         warn "IPv6 gateway specified together with $res->{ip6} address";
2310         return undef;
2311     }
2312
2313     if (!$res->{ip} && !$res->{ip6}) {
2314         return { ip => 'dhcp', ip6 => 'dhcp' };
2315     }
2316
2317     return $res;
2318 }
2319
2320 sub print_net {
2321     my $net = shift;
2322
2323     return PVE::JSONSchema::print_property_string($net, $net_fmt);
2324 }
2325
2326 sub add_random_macs {
2327     my ($settings) = @_;
2328
2329     foreach my $opt (keys %$settings) {
2330         next if $opt !~ m/^net(\d+)$/;
2331         my $net = parse_net($settings->{$opt});
2332         next if !$net;
2333         $settings->{$opt} = print_net($net);
2334     }
2335 }
2336
2337 sub vm_is_volid_owner {
2338     my ($storecfg, $vmid, $volid) = @_;
2339
2340     if ($volid !~  m|^/|) {
2341         my ($path, $owner);
2342         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
2343         if ($owner && ($owner == $vmid)) {
2344             return 1;
2345         }
2346     }
2347
2348     return undef;
2349 }
2350
2351 sub split_flagged_list {
2352     my $text = shift || '';
2353     $text =~ s/[,;]/ /g;
2354     $text =~ s/^\s+//;
2355     return { map { /^(!?)(.*)$/ && ($2, $1) } ($text =~ /\S+/g) };
2356 }
2357
2358 sub join_flagged_list {
2359     my ($how, $lst) = @_;
2360     join $how, map { $lst->{$_} . $_ } keys %$lst;
2361 }
2362
2363 sub vmconfig_delete_pending_option {
2364     my ($conf, $key, $force) = @_;
2365
2366     delete $conf->{pending}->{$key};
2367     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
2368     $pending_delete_hash->{$key} = $force ? '!' : '';
2369     $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
2370 }
2371
2372 sub vmconfig_undelete_pending_option {
2373     my ($conf, $key) = @_;
2374
2375     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
2376     delete $pending_delete_hash->{$key};
2377
2378     if (%$pending_delete_hash) {
2379         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
2380     } else {
2381         delete $conf->{pending}->{delete};
2382     }
2383 }
2384
2385 sub vmconfig_register_unused_drive {
2386     my ($storecfg, $vmid, $conf, $drive) = @_;
2387
2388     if (drive_is_cloudinit($drive)) {
2389         eval { PVE::Storage::vdisk_free($storecfg, $drive->{file}) };
2390         warn $@ if $@;
2391     } elsif (!drive_is_cdrom($drive)) {
2392         my $volid = $drive->{file};
2393         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
2394             PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
2395         }
2396     }
2397 }
2398
2399 sub vmconfig_cleanup_pending {
2400     my ($conf) = @_;
2401
2402     # remove pending changes when nothing changed
2403     my $changes;
2404     foreach my $opt (keys %{$conf->{pending}}) {
2405         if (defined($conf->{$opt}) && ($conf->{pending}->{$opt} eq  $conf->{$opt})) {
2406             $changes = 1;
2407             delete $conf->{pending}->{$opt};
2408         }
2409     }
2410
2411     my $current_delete_hash = split_flagged_list($conf->{pending}->{delete});
2412     my $pending_delete_hash = {};
2413     while (my ($opt, $force) = each %$current_delete_hash) {
2414         if (defined($conf->{$opt})) {
2415             $pending_delete_hash->{$opt} = $force;
2416         } else {
2417             $changes = 1;
2418         }
2419     }
2420
2421     if (%$pending_delete_hash) {
2422         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
2423     } else {
2424         delete $conf->{pending}->{delete};
2425     }
2426
2427     return $changes;
2428 }
2429
2430 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str][,base64=bool]
2431 my $smbios1_fmt = {
2432     uuid => {
2433         type => 'string',
2434         pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
2435         format_description => 'UUID',
2436         description => "Set SMBIOS1 UUID.",
2437         optional => 1,
2438     },
2439     version => {
2440         type => 'string',
2441         pattern => '[A-Za-z0-9+\/]+={0,2}',
2442         format_description => 'Base64 encoded string',
2443         description => "Set SMBIOS1 version.",
2444         optional => 1,
2445     },
2446     serial => {
2447         type => 'string',
2448         pattern => '[A-Za-z0-9+\/]+={0,2}',
2449         format_description => 'Base64 encoded string',
2450         description => "Set SMBIOS1 serial number.",
2451         optional => 1,
2452     },
2453     manufacturer => {
2454         type => 'string',
2455         pattern => '[A-Za-z0-9+\/]+={0,2}',
2456         format_description => 'Base64 encoded string',
2457         description => "Set SMBIOS1 manufacturer.",
2458         optional => 1,
2459     },
2460     product => {
2461         type => 'string',
2462         pattern => '[A-Za-z0-9+\/]+={0,2}',
2463         format_description => 'Base64 encoded string',
2464         description => "Set SMBIOS1 product ID.",
2465         optional => 1,
2466     },
2467     sku => {
2468         type => 'string',
2469         pattern => '[A-Za-z0-9+\/]+={0,2}',
2470         format_description => 'Base64 encoded string',
2471         description => "Set SMBIOS1 SKU string.",
2472         optional => 1,
2473     },
2474     family => {
2475         type => 'string',
2476         pattern => '[A-Za-z0-9+\/]+={0,2}',
2477         format_description => 'Base64 encoded string',
2478         description => "Set SMBIOS1 family string.",
2479         optional => 1,
2480     },
2481     base64 => {
2482         type => 'boolean',
2483         description => 'Flag to indicate that the SMBIOS values are base64 encoded',
2484         optional => 1,
2485     },
2486 };
2487
2488 sub parse_smbios1 {
2489     my ($data) = @_;
2490
2491     my $res = eval { PVE::JSONSchema::parse_property_string($smbios1_fmt, $data) };
2492     warn $@ if $@;
2493     return $res;
2494 }
2495
2496 sub print_smbios1 {
2497     my ($smbios1) = @_;
2498     return PVE::JSONSchema::print_property_string($smbios1, $smbios1_fmt);
2499 }
2500
2501 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_fmt);
2502
2503 PVE::JSONSchema::register_format('pve-qm-bootdisk', \&verify_bootdisk);
2504 sub verify_bootdisk {
2505     my ($value, $noerr) = @_;
2506
2507     return $value if is_valid_drivename($value);
2508
2509     return undef if $noerr;
2510
2511     die "invalid boot disk '$value'\n";
2512 }
2513
2514 sub parse_watchdog {
2515     my ($value) = @_;
2516
2517     return undef if !$value;
2518
2519     my $res = eval { PVE::JSONSchema::parse_property_string($watchdog_fmt, $value) };
2520     warn $@ if $@;
2521     return $res;
2522 }
2523
2524 sub parse_guest_agent {
2525     my ($value) = @_;
2526
2527     return {} if !defined($value->{agent});
2528
2529     my $res = eval { PVE::JSONSchema::parse_property_string($agent_fmt, $value->{agent}) };
2530     warn $@ if $@;
2531
2532     # if the agent is disabled ignore the other potentially set properties
2533     return {} if !$res->{enabled};
2534     return $res;
2535 }
2536
2537 sub parse_vga {
2538     my ($value) = @_;
2539
2540     return {} if !$value;
2541     my $res = eval { PVE::JSONSchema::parse_property_string($vga_fmt, $value) };
2542     warn $@ if $@;
2543     return $res;
2544 }
2545
2546 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
2547 sub verify_usb_device {
2548     my ($value, $noerr) = @_;
2549
2550     return $value if parse_usb_device($value);
2551
2552     return undef if $noerr;
2553
2554     die "unable to parse usb device\n";
2555 }
2556
2557 # add JSON properties for create and set function
2558 sub json_config_properties {
2559     my $prop = shift;
2560
2561     foreach my $opt (keys %$confdesc) {
2562         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate' || $opt eq 'runningmachine';
2563         $prop->{$opt} = $confdesc->{$opt};
2564     }
2565
2566     return $prop;
2567 }
2568
2569 # return copy of $confdesc_cloudinit to generate documentation
2570 sub cloudinit_config_properties {
2571
2572     return dclone($confdesc_cloudinit);
2573 }
2574
2575 sub check_type {
2576     my ($key, $value) = @_;
2577
2578     die "unknown setting '$key'\n" if !$confdesc->{$key};
2579
2580     my $type = $confdesc->{$key}->{type};
2581
2582     if (!defined($value)) {
2583         die "got undefined value\n";
2584     }
2585
2586     if ($value =~ m/[\n\r]/) {
2587         die "property contains a line feed\n";
2588     }
2589
2590     if ($type eq 'boolean') {
2591         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
2592         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
2593         die "type check ('boolean') failed - got '$value'\n";
2594     } elsif ($type eq 'integer') {
2595         return int($1) if $value =~ m/^(\d+)$/;
2596         die "type check ('integer') failed - got '$value'\n";
2597     } elsif ($type eq 'number') {
2598         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
2599         die "type check ('number') failed - got '$value'\n";
2600     } elsif ($type eq 'string') {
2601         if (my $fmt = $confdesc->{$key}->{format}) {
2602             PVE::JSONSchema::check_format($fmt, $value);
2603             return $value;
2604         }
2605         $value =~ s/^\"(.*)\"$/$1/;
2606         return $value;
2607     } else {
2608         die "internal error"
2609     }
2610 }
2611
2612 sub touch_config {
2613     my ($vmid) = @_;
2614
2615     my $conf = PVE::QemuConfig->config_file($vmid);
2616     utime undef, undef, $conf;
2617 }
2618
2619 sub destroy_vm {
2620     my ($storecfg, $vmid, $keep_empty_config, $skiplock) = @_;
2621
2622     my $conffile = PVE::QemuConfig->config_file($vmid);
2623
2624     my $conf = PVE::QemuConfig->load_config($vmid);
2625
2626     PVE::QemuConfig->check_lock($conf) if !$skiplock;
2627
2628     if ($conf->{template}) {
2629         # check if any base image is still used by a linked clone
2630         foreach_drive($conf, sub {
2631                 my ($ds, $drive) = @_;
2632
2633                 return if drive_is_cdrom($drive);
2634
2635                 my $volid = $drive->{file};
2636
2637                 return if !$volid || $volid =~ m|^/|;
2638
2639                 die "base volume '$volid' is still in use by linked cloned\n"
2640                     if PVE::Storage::volume_is_base_and_used($storecfg, $volid);
2641
2642         });
2643     }
2644
2645     # only remove disks owned by this VM
2646     foreach_drive($conf, sub {
2647         my ($ds, $drive) = @_;
2648
2649         return if drive_is_cdrom($drive, 1);
2650
2651         my $volid = $drive->{file};
2652
2653         return if !$volid || $volid =~ m|^/|;
2654
2655         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
2656         return if !$path || !$owner || ($owner != $vmid);
2657
2658         eval {
2659             PVE::Storage::vdisk_free($storecfg, $volid);
2660         };
2661         warn "Could not remove disk '$volid', check manually: $@" if $@;
2662
2663     });
2664
2665     if ($keep_empty_config) {
2666         PVE::Tools::file_set_contents($conffile, "memory: 128\n");
2667     } else {
2668         unlink $conffile;
2669     }
2670
2671     # also remove unused disk
2672     eval {
2673         my $dl = PVE::Storage::vdisk_list($storecfg, undef, $vmid);
2674
2675         eval {
2676             PVE::Storage::foreach_volid($dl, sub {
2677                 my ($volid, $sid, $volname, $d) = @_;
2678                 PVE::Storage::vdisk_free($storecfg, $volid);
2679             });
2680         };
2681         warn $@ if $@;
2682
2683     };
2684     warn $@ if $@;
2685 }
2686
2687 sub parse_vm_config {
2688     my ($filename, $raw) = @_;
2689
2690     return undef if !defined($raw);
2691
2692     my $res = {
2693         digest => Digest::SHA::sha1_hex($raw),
2694         snapshots => {},
2695         pending => {},
2696     };
2697
2698     $filename =~ m|/qemu-server/(\d+)\.conf$|
2699         || die "got strange filename '$filename'";
2700
2701     my $vmid = $1;
2702
2703     my $conf = $res;
2704     my $descr;
2705     my $section = '';
2706
2707     my @lines = split(/\n/, $raw);
2708     foreach my $line (@lines) {
2709         next if $line =~ m/^\s*$/;
2710
2711         if ($line =~ m/^\[PENDING\]\s*$/i) {
2712             $section = 'pending';
2713             if (defined($descr)) {
2714                 $descr =~ s/\s+$//;
2715                 $conf->{description} = $descr;
2716             }
2717             $descr = undef;
2718             $conf = $res->{$section} = {};
2719             next;
2720
2721         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2722             $section = $1;
2723             if (defined($descr)) {
2724                 $descr =~ s/\s+$//;
2725                 $conf->{description} = $descr;
2726             }
2727             $descr = undef;
2728             $conf = $res->{snapshots}->{$section} = {};
2729             next;
2730         }
2731
2732         if ($line =~ m/^\#(.*)\s*$/) {
2733             $descr = '' if !defined($descr);
2734             $descr .= PVE::Tools::decode_text($1) . "\n";
2735             next;
2736         }
2737
2738         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2739             $descr = '' if !defined($descr);
2740             $descr .= PVE::Tools::decode_text($2);
2741         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2742             $conf->{snapstate} = $1;
2743         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2744             my $key = $1;
2745             my $value = $2;
2746             $conf->{$key} = $value;
2747         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2748             my $value = $1;
2749             if ($section eq 'pending') {
2750                 $conf->{delete} = $value; # we parse this later
2751             } else {
2752                 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
2753             }
2754         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(.+?)\s*$/) {
2755             my $key = $1;
2756             my $value = $2;
2757             eval { $value = check_type($key, $value); };
2758             if ($@) {
2759                 warn "vm $vmid - unable to parse value of '$key' - $@";
2760             } else {
2761                 $key = 'ide2' if $key eq 'cdrom';
2762                 my $fmt = $confdesc->{$key}->{format};
2763                 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2764                     my $v = parse_drive($key, $value);
2765                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2766                         $v->{file} = $volid;
2767                         $value = print_drive($vmid, $v);
2768                     } else {
2769                         warn "vm $vmid - unable to parse value of '$key'\n";
2770                         next;
2771                     }
2772                 }
2773
2774                 $conf->{$key} = $value;
2775             }
2776         }
2777     }
2778
2779     if (defined($descr)) {
2780         $descr =~ s/\s+$//;
2781         $conf->{description} = $descr;
2782     }
2783     delete $res->{snapstate}; # just to be sure
2784
2785     return $res;
2786 }
2787
2788 sub write_vm_config {
2789     my ($filename, $conf) = @_;
2790
2791     delete $conf->{snapstate}; # just to be sure
2792
2793     if ($conf->{cdrom}) {
2794         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2795         $conf->{ide2} = $conf->{cdrom};
2796         delete $conf->{cdrom};
2797     }
2798
2799     # we do not use 'smp' any longer
2800     if ($conf->{sockets}) {
2801         delete $conf->{smp};
2802     } elsif ($conf->{smp}) {
2803         $conf->{sockets} = $conf->{smp};
2804         delete $conf->{cores};
2805         delete $conf->{smp};
2806     }
2807
2808     my $used_volids = {};
2809
2810     my $cleanup_config = sub {
2811         my ($cref, $pending, $snapname) = @_;
2812
2813         foreach my $key (keys %$cref) {
2814             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2815                 $key eq 'snapstate' || $key eq 'pending';
2816             my $value = $cref->{$key};
2817             if ($key eq 'delete') {
2818                 die "propertry 'delete' is only allowed in [PENDING]\n"
2819                     if !$pending;
2820                 # fixme: check syntax?
2821                 next;
2822             }
2823             eval { $value = check_type($key, $value); };
2824             die "unable to parse value of '$key' - $@" if $@;
2825
2826             $cref->{$key} = $value;
2827
2828             if (!$snapname && is_valid_drivename($key)) {
2829                 my $drive = parse_drive($key, $value);
2830                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2831             }
2832         }
2833     };
2834
2835     &$cleanup_config($conf);
2836
2837     &$cleanup_config($conf->{pending}, 1);
2838
2839     foreach my $snapname (keys %{$conf->{snapshots}}) {
2840         die "internal error" if $snapname eq 'pending';
2841         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2842     }
2843
2844     # remove 'unusedX' settings if we re-add a volume
2845     foreach my $key (keys %$conf) {
2846         my $value = $conf->{$key};
2847         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2848             delete $conf->{$key};
2849         }
2850     }
2851
2852     my $generate_raw_config = sub {
2853         my ($conf, $pending) = @_;
2854
2855         my $raw = '';
2856
2857         # add description as comment to top of file
2858         if (defined(my $descr = $conf->{description})) {
2859             if ($descr) {
2860                 foreach my $cl (split(/\n/, $descr)) {
2861                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2862                 }
2863             } else {
2864                 $raw .= "#\n" if $pending;
2865             }
2866         }
2867
2868         foreach my $key (sort keys %$conf) {
2869             next if $key eq 'digest' || $key eq 'description' || $key eq 'pending' || $key eq 'snapshots';
2870             $raw .= "$key: $conf->{$key}\n";
2871         }
2872         return $raw;
2873     };
2874
2875     my $raw = &$generate_raw_config($conf);
2876
2877     if (scalar(keys %{$conf->{pending}})){
2878         $raw .= "\n[PENDING]\n";
2879         $raw .= &$generate_raw_config($conf->{pending}, 1);
2880     }
2881
2882     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2883         $raw .= "\n[$snapname]\n";
2884         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2885     }
2886
2887     return $raw;
2888 }
2889
2890 sub load_defaults {
2891
2892     my $res = {};
2893
2894     # we use static defaults from our JSON schema configuration
2895     foreach my $key (keys %$confdesc) {
2896         if (defined(my $default = $confdesc->{$key}->{default})) {
2897             $res->{$key} = $default;
2898         }
2899     }
2900
2901     return $res;
2902 }
2903
2904 sub config_list {
2905     my $vmlist = PVE::Cluster::get_vmlist();
2906     my $res = {};
2907     return $res if !$vmlist || !$vmlist->{ids};
2908     my $ids = $vmlist->{ids};
2909
2910     foreach my $vmid (keys %$ids) {
2911         my $d = $ids->{$vmid};
2912         next if !$d->{node} || $d->{node} ne $nodename;
2913         next if !$d->{type} || $d->{type} ne 'qemu';
2914         $res->{$vmid}->{exists} = 1;
2915     }
2916     return $res;
2917 }
2918
2919 # test if VM uses local resources (to prevent migration)
2920 sub check_local_resources {
2921     my ($conf, $noerr) = @_;
2922
2923     my @loc_res = ();
2924
2925     push @loc_res, "hostusb" if $conf->{hostusb}; # old syntax
2926     push @loc_res, "hostpci" if $conf->{hostpci}; # old syntax
2927
2928     push @loc_res, "ivshmem" if $conf->{ivshmem};
2929
2930     foreach my $k (keys %$conf) {
2931         next if $k =~ m/^usb/ && ($conf->{$k} =~ m/^spice(?![^,])/);
2932         # sockets are safe: they will recreated be on the target side post-migrate
2933         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2934         push @loc_res, $k if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2935     }
2936
2937     die "VM uses local resources\n" if scalar @loc_res && !$noerr;
2938
2939     return \@loc_res;
2940 }
2941
2942 # check if used storages are available on all nodes (use by migrate)
2943 sub check_storage_availability {
2944     my ($storecfg, $conf, $node) = @_;
2945
2946     foreach_drive($conf, sub {
2947         my ($ds, $drive) = @_;
2948
2949         my $volid = $drive->{file};
2950         return if !$volid;
2951
2952         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2953         return if !$sid;
2954
2955         # check if storage is available on both nodes
2956         my $scfg = PVE::Storage::storage_check_node($storecfg, $sid);
2957         PVE::Storage::storage_check_node($storecfg, $sid, $node);
2958    });
2959 }
2960
2961 # list nodes where all VM images are available (used by has_feature API)
2962 sub shared_nodes {
2963     my ($conf, $storecfg) = @_;
2964
2965     my $nodelist = PVE::Cluster::get_nodelist();
2966     my $nodehash = { map { $_ => 1 } @$nodelist };
2967     my $nodename = PVE::INotify::nodename();
2968
2969     foreach_drive($conf, sub {
2970         my ($ds, $drive) = @_;
2971
2972         my $volid = $drive->{file};
2973         return if !$volid;
2974
2975         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2976         if ($storeid) {
2977             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2978             if ($scfg->{disable}) {
2979                 $nodehash = {};
2980             } elsif (my $avail = $scfg->{nodes}) {
2981                 foreach my $node (keys %$nodehash) {
2982                     delete $nodehash->{$node} if !$avail->{$node};
2983                 }
2984             } elsif (!$scfg->{shared}) {
2985                 foreach my $node (keys %$nodehash) {
2986                     delete $nodehash->{$node} if $node ne $nodename
2987                 }
2988             }
2989         }
2990     });
2991
2992     return $nodehash
2993 }
2994
2995 sub check_local_storage_availability {
2996     my ($conf, $storecfg) = @_;
2997
2998     my $nodelist = PVE::Cluster::get_nodelist();
2999     my $nodehash = { map { $_ => {} } @$nodelist };
3000
3001     foreach_drive($conf, sub {
3002         my ($ds, $drive) = @_;
3003
3004         my $volid = $drive->{file};
3005         return if !$volid;
3006
3007         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
3008         if ($storeid) {
3009             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
3010
3011             if ($scfg->{disable}) {
3012                 foreach my $node (keys %$nodehash) {
3013                     $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
3014                 }
3015             } elsif (my $avail = $scfg->{nodes}) {
3016                 foreach my $node (keys %$nodehash) {
3017                     if (!$avail->{$node}) {
3018                         $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
3019                     }
3020                 }
3021             }
3022         }
3023     });
3024
3025     foreach my $node (values %$nodehash) {
3026         if (my $unavail = $node->{unavailable_storages}) {
3027             $node->{unavailable_storages} = [ sort keys %$unavail ];
3028         }
3029     }
3030
3031     return $nodehash
3032 }
3033
3034 sub check_cmdline {
3035     my ($pidfile, $pid) = @_;
3036
3037     my $fh = IO::File->new("/proc/$pid/cmdline", "r");
3038     if (defined($fh)) {
3039         my $line = <$fh>;
3040         $fh->close;
3041         return undef if !$line;
3042         my @param = split(/\0/, $line);
3043
3044         my $cmd = $param[0];
3045         return if !$cmd || ($cmd !~ m|kvm$| && $cmd !~ m@(?:^|/)qemu-system-[^/]+$@);
3046
3047         for (my $i = 0; $i < scalar (@param); $i++) {
3048             my $p = $param[$i];
3049             next if !$p;
3050             if (($p eq '-pidfile') || ($p eq '--pidfile')) {
3051                 my $p = $param[$i+1];
3052                 return 1 if $p && ($p eq $pidfile);
3053                 return undef;
3054             }
3055         }
3056     }
3057     return undef;
3058 }
3059
3060 sub check_running {
3061     my ($vmid, $nocheck, $node) = @_;
3062
3063     my $filename = PVE::QemuConfig->config_file($vmid, $node);
3064
3065     die "unable to find configuration file for VM $vmid - no such machine\n"
3066         if !$nocheck && ! -f $filename;
3067
3068     my $pidfile = pidfile_name($vmid);
3069
3070     if (my $fd = IO::File->new("<$pidfile")) {
3071         my $st = stat($fd);
3072         my $line = <$fd>;
3073         close($fd);
3074
3075         my $mtime = $st->mtime;
3076         if ($mtime > time()) {
3077             warn "file '$filename' modified in future\n";
3078         }
3079
3080         if ($line =~ m/^(\d+)$/) {
3081             my $pid = $1;
3082             if (check_cmdline($pidfile, $pid)) {
3083                 if (my $pinfo = PVE::ProcFSTools::check_process_running($pid)) {
3084                     return $pid;
3085                 }
3086             }
3087         }
3088     }
3089
3090     return undef;
3091 }
3092
3093 sub vzlist {
3094
3095     my $vzlist = config_list();
3096
3097     my $fd = IO::Dir->new($var_run_tmpdir) || return $vzlist;
3098
3099     while (defined(my $de = $fd->read)) {
3100         next if $de !~ m/^(\d+)\.pid$/;
3101         my $vmid = $1;
3102         next if !defined($vzlist->{$vmid});
3103         if (my $pid = check_running($vmid)) {
3104             $vzlist->{$vmid}->{pid} = $pid;
3105         }
3106     }
3107
3108     return $vzlist;
3109 }
3110
3111 sub disksize {
3112     my ($storecfg, $conf) = @_;
3113
3114     my $bootdisk = $conf->{bootdisk};
3115     return undef if !$bootdisk;
3116     return undef if !is_valid_drivename($bootdisk);
3117
3118     return undef if !$conf->{$bootdisk};
3119
3120     my $drive = parse_drive($bootdisk, $conf->{$bootdisk});
3121     return undef if !defined($drive);
3122
3123     return undef if drive_is_cdrom($drive);
3124
3125     my $volid = $drive->{file};
3126     return undef if !$volid;
3127
3128     return $drive->{size};
3129 }
3130
3131 our $vmstatus_return_properties = {
3132     vmid => get_standard_option('pve-vmid'),
3133     status => {
3134         description => "Qemu process status.",
3135         type => 'string',
3136         enum => ['stopped', 'running'],
3137     },
3138     maxmem => {
3139         description => "Maximum memory in bytes.",
3140         type => 'integer',
3141         optional => 1,
3142         renderer => 'bytes',
3143     },
3144     maxdisk => {
3145         description => "Root disk size in bytes.",
3146         type => 'integer',
3147         optional => 1,
3148         renderer => 'bytes',
3149     },
3150     name => {
3151         description => "VM name.",
3152         type => 'string',
3153         optional => 1,
3154     },
3155     qmpstatus => {
3156         description => "Qemu QMP agent status.",
3157         type => 'string',
3158         optional => 1,
3159     },
3160     pid => {
3161         description => "PID of running qemu process.",
3162         type => 'integer',
3163         optional => 1,
3164     },
3165     uptime => {
3166         description => "Uptime.",
3167         type => 'integer',
3168         optional => 1,
3169         renderer => 'duration',
3170     },
3171     cpus => {
3172         description => "Maximum usable CPUs.",
3173         type => 'number',
3174         optional => 1,
3175     },
3176     lock => {
3177         description => "The current config lock, if any.",
3178         type => 'string',
3179         optional => 1,
3180     }
3181 };
3182
3183 my $last_proc_pid_stat;
3184
3185 # get VM status information
3186 # This must be fast and should not block ($full == false)
3187 # We only query KVM using QMP if $full == true (this can be slow)
3188 sub vmstatus {
3189     my ($opt_vmid, $full) = @_;
3190
3191     my $res = {};
3192
3193     my $storecfg = PVE::Storage::config();
3194
3195     my $list = vzlist();
3196     my $defaults = load_defaults();
3197
3198     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
3199
3200     my $cpucount = $cpuinfo->{cpus} || 1;
3201
3202     foreach my $vmid (keys %$list) {
3203         next if $opt_vmid && ($vmid ne $opt_vmid);
3204
3205         my $cfspath = PVE::QemuConfig->cfs_config_path($vmid);
3206         my $conf = PVE::Cluster::cfs_read_file($cfspath) || {};
3207
3208         my $d = { vmid => $vmid };
3209         $d->{pid} = $list->{$vmid}->{pid};
3210
3211         # fixme: better status?
3212         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
3213
3214         my $size = disksize($storecfg, $conf);
3215         if (defined($size)) {
3216             $d->{disk} = 0; # no info available
3217             $d->{maxdisk} = $size;
3218         } else {
3219             $d->{disk} = 0;
3220             $d->{maxdisk} = 0;
3221         }
3222
3223         $d->{cpus} = ($conf->{sockets} || $defaults->{sockets})
3224             * ($conf->{cores} || $defaults->{cores});
3225         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
3226         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
3227
3228         $d->{name} = $conf->{name} || "VM $vmid";
3229         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024)
3230             : $defaults->{memory}*(1024*1024);
3231
3232         if ($conf->{balloon}) {
3233             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
3234             $d->{shares} = defined($conf->{shares}) ? $conf->{shares}
3235                 : $defaults->{shares};
3236         }
3237
3238         $d->{uptime} = 0;
3239         $d->{cpu} = 0;
3240         $d->{mem} = 0;
3241
3242         $d->{netout} = 0;
3243         $d->{netin} = 0;
3244
3245         $d->{diskread} = 0;
3246         $d->{diskwrite} = 0;
3247
3248         $d->{template} = PVE::QemuConfig->is_template($conf);
3249
3250         $d->{serial} = 1 if conf_has_serial($conf);
3251         $d->{lock} = $conf->{lock} if $conf->{lock};
3252
3253         $res->{$vmid} = $d;
3254     }
3255
3256     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
3257     foreach my $dev (keys %$netdev) {
3258         next if $dev !~ m/^tap([1-9]\d*)i/;
3259         my $vmid = $1;
3260         my $d = $res->{$vmid};
3261         next if !$d;
3262
3263         $d->{netout} += $netdev->{$dev}->{receive};
3264         $d->{netin} += $netdev->{$dev}->{transmit};
3265
3266         if ($full) {
3267             $d->{nics}->{$dev}->{netout} = $netdev->{$dev}->{receive};
3268             $d->{nics}->{$dev}->{netin} = $netdev->{$dev}->{transmit};
3269         }
3270
3271     }
3272
3273     my $ctime = gettimeofday;
3274
3275     foreach my $vmid (keys %$list) {
3276
3277         my $d = $res->{$vmid};
3278         my $pid = $d->{pid};
3279         next if !$pid;
3280
3281         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
3282         next if !$pstat; # not running
3283
3284         my $used = $pstat->{utime} + $pstat->{stime};
3285
3286         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
3287
3288         if ($pstat->{vsize}) {
3289             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
3290         }
3291
3292         my $old = $last_proc_pid_stat->{$pid};
3293         if (!$old) {
3294             $last_proc_pid_stat->{$pid} = {
3295                 time => $ctime,
3296                 used => $used,
3297                 cpu => 0,
3298             };
3299             next;
3300         }
3301
3302         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
3303
3304         if ($dtime > 1000) {
3305             my $dutime = $used -  $old->{used};
3306
3307             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
3308             $last_proc_pid_stat->{$pid} = {
3309                 time => $ctime,
3310                 used => $used,
3311                 cpu => $d->{cpu},
3312             };
3313         } else {
3314             $d->{cpu} = $old->{cpu};
3315         }
3316     }
3317
3318     return $res if !$full;
3319
3320     my $qmpclient = PVE::QMPClient->new();
3321
3322     my $ballooncb = sub {
3323         my ($vmid, $resp) = @_;
3324
3325         my $info = $resp->{'return'};
3326         return if !$info->{max_mem};
3327
3328         my $d = $res->{$vmid};
3329
3330         # use memory assigned to VM
3331         $d->{maxmem} = $info->{max_mem};
3332         $d->{balloon} = $info->{actual};
3333
3334         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
3335             $d->{mem} = $info->{total_mem} - $info->{free_mem};
3336             $d->{freemem} = $info->{free_mem};
3337         }
3338
3339         $d->{ballooninfo} = $info;
3340     };
3341
3342     my $blockstatscb = sub {
3343         my ($vmid, $resp) = @_;
3344         my $data = $resp->{'return'} || [];
3345         my $totalrdbytes = 0;
3346         my $totalwrbytes = 0;
3347
3348         for my $blockstat (@$data) {
3349             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
3350             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
3351
3352             $blockstat->{device} =~ s/drive-//;
3353             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
3354         }
3355         $res->{$vmid}->{diskread} = $totalrdbytes;
3356         $res->{$vmid}->{diskwrite} = $totalwrbytes;
3357     };
3358
3359     my $statuscb = sub {
3360         my ($vmid, $resp) = @_;
3361
3362         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
3363         # this fails if ballon driver is not loaded, so this must be
3364         # the last commnand (following command are aborted if this fails).
3365         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
3366
3367         my $status = 'unknown';
3368         if (!defined($status = $resp->{'return'}->{status})) {
3369             warn "unable to get VM status\n";
3370             return;
3371         }
3372
3373         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
3374     };
3375
3376     foreach my $vmid (keys %$list) {
3377         next if $opt_vmid && ($vmid ne $opt_vmid);
3378         next if !$res->{$vmid}->{pid}; # not running
3379         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
3380     }
3381
3382     $qmpclient->queue_execute(undef, 2);
3383
3384     foreach my $vmid (keys %$list) {
3385         next if $opt_vmid && ($vmid ne $opt_vmid);
3386         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
3387     }
3388
3389     return $res;
3390 }
3391
3392 sub foreach_drive {
3393     my ($conf, $func, @param) = @_;
3394
3395     foreach my $ds (valid_drive_names()) {
3396         next if !defined($conf->{$ds});
3397
3398         my $drive = parse_drive($ds, $conf->{$ds});
3399         next if !$drive;
3400
3401         &$func($ds, $drive, @param);
3402     }
3403 }
3404
3405 sub foreach_volid {
3406     my ($conf, $func, @param) = @_;
3407
3408     my $volhash = {};
3409
3410     my $test_volid = sub {
3411         my ($volid, $is_cdrom, $replicate, $shared, $snapname, $size) = @_;
3412
3413         return if !$volid;
3414
3415         $volhash->{$volid}->{cdrom} //= 1;
3416         $volhash->{$volid}->{cdrom} = 0 if !$is_cdrom;
3417
3418         $volhash->{$volid}->{replicate} //= 0;
3419         $volhash->{$volid}->{replicate} = 1 if $replicate;
3420
3421         $volhash->{$volid}->{shared} //= 0;
3422         $volhash->{$volid}->{shared} = 1 if $shared;
3423
3424         $volhash->{$volid}->{referenced_in_config} //= 0;
3425         $volhash->{$volid}->{referenced_in_config} = 1 if !defined($snapname);
3426
3427         $volhash->{$volid}->{referenced_in_snapshot}->{$snapname} = 1
3428             if defined($snapname);
3429         $volhash->{$volid}->{size} = $size if $size;
3430     };
3431
3432     foreach_drive($conf, sub {
3433         my ($ds, $drive) = @_;
3434         $test_volid->($drive->{file}, drive_is_cdrom($drive), $drive->{replicate} // 1, $drive->{shared}, undef, $drive->{size});
3435     });
3436
3437     foreach my $snapname (keys %{$conf->{snapshots}}) {
3438         my $snap = $conf->{snapshots}->{$snapname};
3439         $test_volid->($snap->{vmstate}, 0, 1, $snapname);
3440         foreach_drive($snap, sub {
3441             my ($ds, $drive) = @_;
3442             $test_volid->($drive->{file}, drive_is_cdrom($drive), $drive->{replicate} // 1, $drive->{shared}, $snapname);
3443         });
3444     }
3445
3446     foreach my $volid (keys %$volhash) {
3447         &$func($volid, $volhash->{$volid}, @param);
3448     }
3449 }
3450
3451 sub conf_has_serial {
3452     my ($conf) = @_;
3453
3454     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3455         if ($conf->{"serial$i"}) {
3456             return 1;
3457         }
3458     }
3459
3460     return 0;
3461 }
3462
3463 sub conf_has_audio {
3464     my ($conf, $id) = @_;
3465
3466     $id //= 0;
3467     my $audio = $conf->{"audio$id"};
3468     return undef if !defined($audio);
3469
3470     my $audioproperties = PVE::JSONSchema::parse_property_string($audio_fmt, $audio);
3471     my $audiodriver = $audioproperties->{driver} // 'spice';
3472
3473     return {
3474         dev => $audioproperties->{device},
3475         dev_id => "audiodev$id",
3476         backend => $audiodriver,
3477         backend_id => "$audiodriver-backend${id}",
3478     };
3479 }
3480
3481 sub vga_conf_has_spice {
3482     my ($vga) = @_;
3483
3484     my $vgaconf = parse_vga($vga);
3485     my $vgatype = $vgaconf->{type};
3486     return 0 if !$vgatype || $vgatype !~ m/^qxl([234])?$/;
3487
3488     return $1 || 1;
3489 }
3490
3491 my $host_arch; # FIXME: fix PVE::Tools::get_host_arch
3492 sub get_host_arch() {
3493     $host_arch = (POSIX::uname())[4] if !$host_arch;
3494     return $host_arch;
3495 }
3496
3497 sub is_native($) {
3498     my ($arch) = @_;
3499     return get_host_arch() eq $arch;
3500 }
3501
3502 my $default_machines = {
3503     x86_64 => 'pc',
3504     aarch64 => 'virt',
3505 };
3506
3507 sub get_basic_machine_info {
3508     my ($conf, $forcemachine) = @_;
3509
3510     my $arch = $conf->{arch} // get_host_arch();
3511     my $machine = $forcemachine || $conf->{machine} || $default_machines->{$arch};
3512     return ($arch, $machine);
3513 }
3514
3515 sub get_ovmf_files($) {
3516     my ($arch) = @_;
3517
3518     my $ovmf = $OVMF->{$arch}
3519         or die "no OVMF images known for architecture '$arch'\n";
3520
3521     return @$ovmf;
3522 }
3523
3524 my $Arch2Qemu = {
3525     aarch64 => '/usr/bin/qemu-system-aarch64',
3526     x86_64 => '/usr/bin/qemu-system-x86_64',
3527 };
3528 sub get_command_for_arch($) {
3529     my ($arch) = @_;
3530     return '/usr/bin/kvm' if is_native($arch);
3531
3532     my $cmd = $Arch2Qemu->{$arch}
3533         or die "don't know how to emulate architecture '$arch'\n";
3534     return $cmd;
3535 }
3536
3537 sub get_cpu_options {
3538     my ($conf, $arch, $kvm, $machine_type, $kvm_off, $kvmver, $winversion, $gpu_passthrough) = @_;
3539
3540     my $cpuFlags = [];
3541     my $ostype = $conf->{ostype};
3542
3543     my $cpu = $kvm ? "kvm64" : "qemu64";
3544     if ($arch eq 'aarch64') {
3545         $cpu = 'cortex-a57';
3546     }
3547     my $hv_vendor_id;
3548     if (my $cputype = $conf->{cpu}) {
3549         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
3550             or die "Cannot parse cpu description: $cputype\n";
3551         $cpu = $cpuconf->{cputype};
3552         $kvm_off = 1 if $cpuconf->{hidden};
3553         $hv_vendor_id = $cpuconf->{'hv-vendor-id'};
3554
3555         if (defined(my $flags = $cpuconf->{flags})) {
3556             push @$cpuFlags, split(";", $flags);
3557         }
3558     }
3559
3560     push @$cpuFlags , '+lahf_lm' if $cpu eq 'kvm64' && $arch eq 'x86_64';
3561
3562     push @$cpuFlags , '-x2apic'
3563         if $conf->{ostype} && $conf->{ostype} eq 'solaris';
3564
3565     push @$cpuFlags, '+sep' if $cpu eq 'kvm64' || $cpu eq 'kvm32';
3566
3567     push @$cpuFlags, '-rdtscp' if $cpu =~ m/^Opteron/;
3568
3569     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3) && $arch eq 'x86_64') {
3570
3571         push @$cpuFlags , '+kvm_pv_unhalt' if $kvm;
3572         push @$cpuFlags , '+kvm_pv_eoi' if $kvm;
3573     }
3574
3575     add_hyperv_enlightenments($cpuFlags, $winversion, $machine_type, $kvmver, $conf->{bios}, $gpu_passthrough, $hv_vendor_id) if $kvm;
3576
3577     push @$cpuFlags, 'enforce' if $cpu ne 'host' && $kvm && $arch eq 'x86_64';
3578
3579     push @$cpuFlags, 'kvm=off' if $kvm_off;
3580
3581     if (my $cpu_vendor = $cpu_vendor_list->{$cpu}) {
3582         push @$cpuFlags, "vendor=${cpu_vendor}"
3583             if $cpu_vendor ne 'default';
3584     } elsif ($arch ne 'aarch64') {
3585         die "internal error"; # should not happen
3586     }
3587
3588     $cpu .= "," . join(',', @$cpuFlags) if scalar(@$cpuFlags);
3589
3590     return ('-cpu', $cpu);
3591 }
3592
3593 sub config_to_command {
3594     my ($storecfg, $vmid, $conf, $defaults, $forcemachine) = @_;
3595
3596     my $cmd = [];
3597     my $globalFlags = [];
3598     my $machineFlags = [];
3599     my $rtcFlags = [];
3600     my $devices = [];
3601     my $pciaddr = '';
3602     my $bridges = {};
3603     my $vernum = 0; # unknown
3604     my $ostype = $conf->{ostype};
3605     my $winversion = windows_version($ostype);
3606     my $kvm = $conf->{kvm};
3607
3608     my ($arch, $machine_type) = get_basic_machine_info($conf, $forcemachine);
3609     my $kvm_binary = get_command_for_arch($arch);
3610     my $kvmver = kvm_user_version($kvm_binary);
3611     $kvm //= 1 if is_native($arch);
3612
3613     if ($kvm) {
3614         die "KVM virtualisation configured, but not available. Either disable in VM configuration or enable in BIOS.\n"
3615             if !defined kvm_version();
3616     }
3617
3618     if ($kvmver =~ m/^(\d+)\.(\d+)$/) {
3619         $vernum = $1*1000000+$2*1000;
3620     } elsif ($kvmver =~ m/^(\d+)\.(\d+)\.(\d+)$/) {
3621         $vernum = $1*1000000+$2*1000+$3;
3622     }
3623
3624     die "detected old qemu-kvm binary ($kvmver)\n" if $vernum < 15000;
3625
3626     my $q35 = machine_type_is_q35($conf);
3627     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
3628     my $use_old_bios_files = undef;
3629     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3630
3631     my $cpuunits = defined($conf->{cpuunits}) ?
3632             $conf->{cpuunits} : $defaults->{cpuunits};
3633
3634     push @$cmd, $kvm_binary;
3635
3636     push @$cmd, '-id', $vmid;
3637
3638     my $vmname = $conf->{name} || "vm$vmid";
3639
3640     push @$cmd, '-name', $vmname;
3641
3642     my $use_virtio = 0;
3643
3644     my $qmpsocket = qmp_socket($vmid);
3645     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server,nowait";
3646     push @$cmd, '-mon', "chardev=qmp,mode=control";
3647
3648     if (qemu_machine_feature_enabled($machine_type, $kvmver, 2, 12)) {
3649         push @$cmd, '-chardev', "socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5";
3650         push @$cmd, '-mon', "chardev=qmp-event,mode=control";
3651     }
3652
3653     push @$cmd, '-pidfile' , pidfile_name($vmid);
3654
3655     push @$cmd, '-daemonize';
3656
3657     if ($conf->{smbios1}) {
3658         my $smbios_conf = parse_smbios1($conf->{smbios1});
3659         if ($smbios_conf->{base64}) {
3660             # Do not pass base64 flag to qemu
3661             delete $smbios_conf->{base64};
3662             my $smbios_string = "";
3663             foreach my $key (keys %$smbios_conf) {
3664                 my $value;
3665                 if ($key eq "uuid") {
3666                     $value = $smbios_conf->{uuid}
3667                 } else {
3668                     $value = decode_base64($smbios_conf->{$key});
3669                 }
3670                 # qemu accepts any binary data, only commas need escaping by double comma
3671                 $value =~ s/,/,,/g;
3672                 $smbios_string .= "," . $key . "=" . $value if $value;
3673             }
3674             push @$cmd, '-smbios', "type=1" . $smbios_string;
3675         } else {
3676             push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
3677         }
3678     }
3679
3680     if ($conf->{vmgenid}) {
3681         push @$devices, '-device', 'vmgenid,guid='.$conf->{vmgenid};
3682     }
3683
3684     my ($ovmf_code, $ovmf_vars) = get_ovmf_files($arch);
3685     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3686         die "uefi base image not found\n" if ! -f $ovmf_code;
3687
3688         my $path;
3689         my $format;
3690         if (my $efidisk = $conf->{efidisk0}) {
3691             my $d = PVE::JSONSchema::parse_property_string($efidisk_fmt, $efidisk);
3692             my ($storeid, $volname) = PVE::Storage::parse_volume_id($d->{file}, 1);
3693             $format = $d->{format};
3694             if ($storeid) {
3695                 $path = PVE::Storage::path($storecfg, $d->{file});
3696                 if (!defined($format)) {
3697                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
3698                     $format = qemu_img_format($scfg, $volname);
3699                 }
3700             } else {
3701                 $path = $d->{file};
3702                 die "efidisk format must be specified\n"
3703                     if !defined($format);
3704             }
3705         } else {
3706             warn "no efidisk configured! Using temporary efivars disk.\n";
3707             $path = "/tmp/$vmid-ovmf.fd";
3708             PVE::Tools::file_copy($ovmf_vars, $path, -s $ovmf_vars);
3709             $format = 'raw';
3710         }
3711
3712         push @$cmd, '-drive', "if=pflash,unit=0,format=raw,readonly,file=$ovmf_code";
3713         push @$cmd, '-drive', "if=pflash,unit=1,format=$format,id=drive-efidisk0,file=$path";
3714     }
3715
3716     # load q35 config
3717     if ($q35) {
3718         # we use different pcie-port hardware for qemu >= 4.0 for passthrough
3719         if (qemu_machine_feature_enabled($machine_type, $kvmver, 4, 0)) {
3720             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35-4.0.cfg';
3721         } else {
3722             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35.cfg';
3723         }
3724     }
3725
3726     # add usb controllers
3727     my @usbcontrollers = PVE::QemuServer::USB::get_usb_controllers($conf, $bridges, $arch, $machine_type, $usbdesc->{format}, $MAX_USB_DEVICES);
3728     push @$devices, @usbcontrollers if @usbcontrollers;
3729     my $vga = parse_vga($conf->{vga});
3730
3731     my $qxlnum = vga_conf_has_spice($conf->{vga});
3732     $vga->{type} = 'qxl' if $qxlnum;
3733
3734     if (!$vga->{type}) {
3735         if ($arch eq 'aarch64') {
3736             $vga->{type} = 'virtio';
3737         } elsif (qemu_machine_feature_enabled($machine_type, $kvmver, 2, 9)) {
3738             $vga->{type} = (!$winversion || $winversion >= 6) ? 'std' : 'cirrus';
3739         } else {
3740             $vga->{type} = ($winversion >= 6) ? 'std' : 'cirrus';
3741         }
3742     }
3743
3744     # enable absolute mouse coordinates (needed by vnc)
3745     my $tablet;
3746     if (defined($conf->{tablet})) {
3747         $tablet = $conf->{tablet};
3748     } else {
3749         $tablet = $defaults->{tablet};
3750         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
3751         $tablet = 0 if $vga->{type} =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
3752     }
3753
3754     if ($tablet) {
3755         push @$devices, '-device', print_tabletdevice_full($conf, $arch) if $tablet;
3756         my $kbd = print_keyboarddevice_full($conf, $arch);
3757         push @$devices, '-device', $kbd if defined($kbd);
3758     }
3759
3760     my $kvm_off = 0;
3761     my $gpu_passthrough;
3762
3763     # host pci devices
3764     for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
3765         my $id = "hostpci$i";
3766         my $d = parse_hostpci($conf->{$id});
3767         next if !$d;
3768
3769         if (my $pcie = $d->{pcie}) {
3770             die "q35 machine model is not enabled" if !$q35;
3771             # win7 wants to have the pcie devices directly on the pcie bus
3772             # instead of in the root port
3773             if ($winversion == 7) {
3774                 $pciaddr = print_pcie_addr("${id}bus0");
3775             } else {
3776                 # add more root ports if needed, 4 are present by default
3777                 # by pve-q35 cfgs, rest added here on demand.
3778                 if ($i > 3) {
3779                     push @$devices, '-device', print_pcie_root_port($i);
3780                 }
3781                 $pciaddr = print_pcie_addr($id);
3782             }
3783         } else {
3784             $pciaddr = print_pci_addr($id, $bridges, $arch, $machine_type);
3785         }
3786
3787         my $xvga = '';
3788         if ($d->{'x-vga'}) {
3789             $xvga = ',x-vga=on' if !($conf->{bios} && $conf->{bios} eq 'ovmf');
3790             $kvm_off = 1;
3791             $vga->{type} = 'none' if !defined($conf->{vga});
3792             $gpu_passthrough = 1;
3793         }
3794
3795         my $pcidevices = $d->{pciid};
3796         my $multifunction = 1 if @$pcidevices > 1;
3797
3798         my $sysfspath;
3799         if ($d->{mdev} && scalar(@$pcidevices) == 1) {
3800             my $pci_id = $pcidevices->[0]->{id};
3801             my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $i);
3802             $sysfspath = "/sys/bus/pci/devices/0000:$pci_id/$uuid";
3803         } elsif ($d->{mdev}) {
3804             warn "ignoring mediated device '$id' with multifunction device\n";
3805         }
3806
3807         my $j=0;
3808         foreach my $pcidevice (@$pcidevices) {
3809             my $devicestr = "vfio-pci";
3810
3811             if ($sysfspath) {
3812                 $devicestr .= ",sysfsdev=$sysfspath";
3813             } else {
3814                 $devicestr .= ",host=$pcidevice->{id}";
3815             }
3816
3817             my $mf_addr = $multifunction ? ".$j" : '';
3818             $devicestr .= ",id=${id}${mf_addr}${pciaddr}${mf_addr}";
3819
3820             if ($j == 0) {
3821                 $devicestr .= ',rombar=0' if defined($d->{rombar}) && !$d->{rombar};
3822                 $devicestr .= "$xvga";
3823                 $devicestr .= ",multifunction=on" if $multifunction;
3824                 $devicestr .= ",romfile=/usr/share/kvm/$d->{romfile}" if $d->{romfile};
3825             }
3826
3827             push @$devices, '-device', $devicestr;
3828             $j++;
3829         }
3830     }
3831
3832     # usb devices
3833     my $usb_dev_features = {};
3834     $usb_dev_features->{spice_usb3} = 1 if qemu_machine_feature_enabled($machine_type, $kvmver, 4, 0);
3835
3836     my @usbdevices = PVE::QemuServer::USB::get_usb_devices($conf, $usbdesc->{format}, $MAX_USB_DEVICES, $usb_dev_features);
3837     push @$devices, @usbdevices if @usbdevices;
3838     # serial devices
3839     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3840         if (my $path = $conf->{"serial$i"}) {
3841             if ($path eq 'socket') {
3842                 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
3843                 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server,nowait";
3844                 # On aarch64, serial0 is the UART device. Qemu only allows
3845                 # connecting UART devices via the '-serial' command line, as
3846                 # the device has a fixed slot on the hardware...
3847                 if ($arch eq 'aarch64' && $i == 0) {
3848                     push @$devices, '-serial', "chardev:serial$i";
3849                 } else {
3850                     push @$devices, '-device', "isa-serial,chardev=serial$i";
3851                 }
3852             } else {
3853                 die "no such serial device\n" if ! -c $path;
3854                 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
3855                 push @$devices, '-device', "isa-serial,chardev=serial$i";
3856             }
3857         }
3858     }
3859
3860     # parallel devices
3861     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
3862         if (my $path = $conf->{"parallel$i"}) {
3863             die "no such parallel device\n" if ! -c $path;
3864             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
3865             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
3866             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
3867         }
3868     }
3869
3870     if (my $audio = conf_has_audio($conf)) {
3871
3872         my $audiopciaddr = print_pci_addr("audio0", $bridges, $arch, $machine_type);
3873
3874         my $id = $audio->{dev_id};
3875         if ($audio->{dev} eq 'AC97') {
3876             push @$devices, '-device', "AC97,id=${id}${audiopciaddr}";
3877         } elsif ($audio->{dev} =~ /intel\-hda$/) {
3878             push @$devices, '-device', "$audio->{dev},id=${id}${audiopciaddr}";
3879             push @$devices, '-device', "hda-micro,id=${id}-codec0,bus=${id}.0,cad=0";
3880             push @$devices, '-device', "hda-duplex,id=${id}-codec1,bus=${id}.0,cad=1";
3881         } else {
3882             die "unkown audio device '$audio->{dev}', implement me!";
3883         }
3884
3885         push @$devices, '-audiodev', "$audio->{backend},id=$audio->{backend_id}";
3886     }
3887
3888     my $sockets = 1;
3889     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3890     $sockets = $conf->{sockets} if  $conf->{sockets};
3891
3892     my $cores = $conf->{cores} || 1;
3893
3894     my $maxcpus = $sockets * $cores;
3895
3896     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
3897
3898     my $allowed_vcpus = $cpuinfo->{cpus};
3899
3900     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n"
3901         if ($allowed_vcpus < $maxcpus);
3902
3903     if($hotplug_features->{cpu} && qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 7)) {
3904
3905         push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3906         for (my $i = 2; $i <= $vcpus; $i++)  {
3907             my $cpustr = print_cpu_device($conf,$i);
3908             push @$cmd, '-device', $cpustr;
3909         }
3910
3911     } else {
3912
3913         push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3914     }
3915     push @$cmd, '-nodefaults';
3916
3917     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
3918
3919     my $bootindex_hash = {};
3920     my $i = 1;
3921     foreach my $o (split(//, $bootorder)) {
3922         $bootindex_hash->{$o} = $i*100;
3923         $i++;
3924     }
3925
3926     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
3927
3928     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
3929
3930     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
3931
3932     if ($vga->{type} && $vga->{type} !~ m/^serial\d+$/ && $vga->{type} ne 'none'){
3933         push @$devices, '-device', print_vga_device($conf, $vga, $arch, $machine_type, undef, $qxlnum, $bridges);
3934         my $socket = vnc_socket($vmid);
3935         push @$cmd,  '-vnc', "unix:$socket,password";
3936     } else {
3937         push @$cmd, '-vga', 'none' if $vga->{type} eq 'none';
3938         push @$cmd, '-nographic';
3939     }
3940
3941     # time drift fix
3942     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
3943
3944     my $useLocaltime = $conf->{localtime};
3945
3946     if ($winversion >= 5) { # windows
3947         $useLocaltime = 1 if !defined($conf->{localtime});
3948
3949         # use time drift fix when acpi is enabled
3950         if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
3951             $tdf = 1 if !defined($conf->{tdf});
3952         }
3953     }
3954
3955     if ($winversion >= 6) {
3956         push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3957         push @$cmd, '-no-hpet';
3958     }
3959
3960     push @$rtcFlags, 'driftfix=slew' if $tdf;
3961
3962     if (!$kvm) {
3963         push @$machineFlags, 'accel=tcg';
3964     }
3965
3966     if ($machine_type) {
3967         push @$machineFlags, "type=${machine_type}";
3968     }
3969
3970     if (($conf->{startdate}) && ($conf->{startdate} ne 'now')) {
3971         push @$rtcFlags, "base=$conf->{startdate}";
3972     } elsif ($useLocaltime) {
3973         push @$rtcFlags, 'base=localtime';
3974     }
3975
3976     push @$cmd, get_cpu_options($conf, $arch, $kvm, $machine_type, $kvm_off, $kvmver, $winversion, $gpu_passthrough);
3977
3978     PVE::QemuServer::Memory::config($conf, $vmid, $sockets, $cores, $defaults, $hotplug_features, $cmd);
3979
3980     push @$cmd, '-S' if $conf->{freeze};
3981
3982     push @$cmd, '-k', $conf->{keyboard} if defined($conf->{keyboard});
3983
3984     if (parse_guest_agent($conf)->{enabled}) {
3985         my $qgasocket = qmp_socket($vmid, 1);
3986         my $pciaddr = print_pci_addr("qga0", $bridges, $arch, $machine_type);
3987         push @$devices, '-chardev', "socket,path=$qgasocket,server,nowait,id=qga0";
3988         push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3989         push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3990     }
3991
3992     my $spice_port;
3993
3994     if ($qxlnum) {
3995         if ($qxlnum > 1) {
3996             if ($winversion){
3997                 for(my $i = 1; $i < $qxlnum; $i++){
3998                     push @$devices, '-device', print_vga_device($conf, $vga, $arch, $machine_type, $i, $qxlnum, $bridges);
3999                 }
4000             } else {
4001                 # assume other OS works like Linux
4002                 my ($ram, $vram) = ("134217728", "67108864");
4003                 if ($vga->{memory}) {
4004                     $ram = PVE::Tools::convert_size($qxlnum*4*$vga->{memory}, 'mb' => 'b');
4005                     $vram = PVE::Tools::convert_size($qxlnum*2*$vga->{memory}, 'mb' => 'b');
4006                 }
4007                 push @$cmd, '-global', "qxl-vga.ram_size=$ram";
4008                 push @$cmd, '-global', "qxl-vga.vram_size=$vram";
4009             }
4010         }
4011
4012         my $pciaddr = print_pci_addr("spice", $bridges, $arch, $machine_type);
4013
4014         my $nodename = PVE::INotify::nodename();
4015         my $pfamily = PVE::Tools::get_host_address_family($nodename);
4016         my @nodeaddrs = PVE::Tools::getaddrinfo_all('localhost', family => $pfamily);
4017         die "failed to get an ip address of type $pfamily for 'localhost'\n" if !@nodeaddrs;
4018
4019         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
4020         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
4021         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
4022
4023         my $localhost = PVE::Network::addr_to_ip($nodeaddrs[0]->{addr});
4024         $spice_port = PVE::Tools::next_spice_port($pfamily, $localhost);
4025
4026         my $spice_enhancement = PVE::JSONSchema::parse_property_string($spice_enhancements_fmt, $conf->{spice_enhancements} // '');
4027         if ($spice_enhancement->{foldersharing}) {
4028             push @$devices, '-chardev', "spiceport,id=foldershare,name=org.spice-space.webdav.0";
4029             push @$devices, '-device', "virtserialport,chardev=foldershare,name=org.spice-space.webdav.0";
4030         }
4031
4032         my $spice_opts = "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";
4033         $spice_opts .= ",streaming-video=$spice_enhancement->{videostreaming}" if $spice_enhancement->{videostreaming};
4034         push @$devices, '-spice', "$spice_opts";
4035     }
4036
4037     # enable balloon by default, unless explicitly disabled
4038     if (!defined($conf->{balloon}) || $conf->{balloon}) {
4039         $pciaddr = print_pci_addr("balloon0", $bridges, $arch, $machine_type);
4040         push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
4041     }
4042
4043     if ($conf->{watchdog}) {
4044         my $wdopts = parse_watchdog($conf->{watchdog});
4045         $pciaddr = print_pci_addr("watchdog", $bridges, $arch, $machine_type);
4046         my $watchdog = $wdopts->{model} || 'i6300esb';
4047         push @$devices, '-device', "$watchdog$pciaddr";
4048         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
4049     }
4050
4051     my $vollist = [];
4052     my $scsicontroller = {};
4053     my $ahcicontroller = {};
4054     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
4055
4056     # Add iscsi initiator name if available
4057     if (my $initiator = get_initiator_name()) {
4058         push @$devices, '-iscsi', "initiator-name=$initiator";
4059     }
4060
4061     foreach_drive($conf, sub {
4062         my ($ds, $drive) = @_;
4063
4064         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
4065             push @$vollist, $drive->{file};
4066         }
4067
4068         # ignore efidisk here, already added in bios/fw handling code above
4069         return if $drive->{interface} eq 'efidisk';
4070
4071         $use_virtio = 1 if $ds =~ m/^virtio/;
4072
4073         if (drive_is_cdrom ($drive)) {
4074             if ($bootindex_hash->{d}) {
4075                 $drive->{bootindex} = $bootindex_hash->{d};
4076                 $bootindex_hash->{d} += 1;
4077             }
4078         } else {
4079             if ($bootindex_hash->{c}) {
4080                 $drive->{bootindex} = $bootindex_hash->{c} if $conf->{bootdisk} && ($conf->{bootdisk} eq $ds);
4081                 $bootindex_hash->{c} += 1;
4082             }
4083         }
4084
4085         if($drive->{interface} eq 'virtio'){
4086            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
4087         }
4088
4089         if ($drive->{interface} eq 'scsi') {
4090
4091             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
4092
4093             $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges, $arch, $machine_type);
4094             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
4095
4096             my $iothread = '';
4097             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
4098                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
4099                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
4100             } elsif ($drive->{iothread}) {
4101                 warn "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n";
4102             }
4103
4104             my $queues = '';
4105             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
4106                 $queues = ",num_queues=$drive->{queues}";
4107             }
4108
4109             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues" if !$scsicontroller->{$controller};
4110             $scsicontroller->{$controller}=1;
4111         }
4112
4113         if ($drive->{interface} eq 'sata') {
4114            my $controller = int($drive->{index} / $MAX_SATA_DISKS);
4115            $pciaddr = print_pci_addr("ahci$controller", $bridges, $arch, $machine_type);
4116            push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr" if !$ahcicontroller->{$controller};
4117            $ahcicontroller->{$controller}=1;
4118         }
4119
4120         my $drive_cmd = print_drive_full($storecfg, $vmid, $drive);
4121         push @$devices, '-drive',$drive_cmd;
4122         push @$devices, '-device', print_drivedevice_full($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type);
4123     });
4124
4125     for (my $i = 0; $i < $MAX_NETS; $i++) {
4126          next if !$conf->{"net$i"};
4127          my $d = parse_net($conf->{"net$i"});
4128          next if !$d;
4129
4130          $use_virtio = 1 if $d->{model} eq 'virtio';
4131
4132          if ($bootindex_hash->{n}) {
4133             $d->{bootindex} = $bootindex_hash->{n};
4134             $bootindex_hash->{n} += 1;
4135          }
4136
4137          my $netdevfull = print_netdev_full($vmid, $conf, $arch, $d, "net$i");
4138          push @$devices, '-netdev', $netdevfull;
4139
4140          my $netdevicefull = print_netdevice_full($vmid, $conf, $d, "net$i", $bridges, $use_old_bios_files, $arch, $machine_type);
4141          push @$devices, '-device', $netdevicefull;
4142     }
4143
4144     if ($conf->{ivshmem}) {
4145         my $ivshmem = PVE::JSONSchema::parse_property_string($ivshmem_fmt, $conf->{ivshmem});
4146
4147         my $bus;
4148         if ($q35) {
4149             $bus = print_pcie_addr("ivshmem");
4150         } else {
4151             $bus = print_pci_addr("ivshmem", $bridges, $arch, $machine_type);
4152         }
4153
4154         my $ivshmem_name = $ivshmem->{name} // $vmid;
4155         my $path = '/dev/shm/pve-shm-' . $ivshmem_name;
4156
4157         push @$devices, '-device', "ivshmem-plain,memdev=ivshmem$bus,";
4158         push @$devices, '-object', "memory-backend-file,id=ivshmem,share=on,mem-path=$path,size=$ivshmem->{size}M";
4159     }
4160
4161     if (!$q35) {
4162         # add pci bridges
4163         if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
4164            $bridges->{1} = 1;
4165            $bridges->{2} = 1;
4166         }
4167
4168         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
4169
4170         for my $k (sort {$b cmp $a} keys %$bridges) {
4171             $pciaddr = print_pci_addr("pci.$k", undef, $arch, $machine_type);
4172             unshift @$devices, '-device', "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr" if $k > 0;
4173         }
4174     }
4175
4176     push @$cmd, @$devices;
4177     push @$cmd, '-rtc', join(',', @$rtcFlags)
4178         if scalar(@$rtcFlags);
4179     push @$cmd, '-machine', join(',', @$machineFlags)
4180         if scalar(@$machineFlags);
4181     push @$cmd, '-global', join(',', @$globalFlags)
4182         if scalar(@$globalFlags);
4183
4184     if (my $vmstate = $conf->{vmstate}) {
4185         my $statepath = PVE::Storage::path($storecfg, $vmstate);
4186         push @$vollist, $statepath;
4187         push @$cmd, '-loadstate', $statepath;
4188     }
4189
4190     # add custom args
4191     if ($conf->{args}) {
4192         my $aa = PVE::Tools::split_args($conf->{args});
4193         push @$cmd, @$aa;
4194     }
4195
4196     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
4197 }
4198
4199 sub vnc_socket {
4200     my ($vmid) = @_;
4201     return "${var_run_tmpdir}/$vmid.vnc";
4202 }
4203
4204 sub spice_port {
4205     my ($vmid) = @_;
4206
4207     my $res = vm_mon_cmd($vmid, 'query-spice');
4208
4209     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
4210 }
4211
4212 sub qmp_socket {
4213     my ($vmid, $qga, $name) = @_;
4214     my $sockettype = $qga ? 'qga' : 'qmp';
4215     my $ext = $name ? '-'.$name : '';
4216     return "${var_run_tmpdir}/$vmid$ext.$sockettype";
4217 }
4218
4219 sub pidfile_name {
4220     my ($vmid) = @_;
4221     return "${var_run_tmpdir}/$vmid.pid";
4222 }
4223
4224 sub vm_devices_list {
4225     my ($vmid) = @_;
4226
4227     my $res = vm_mon_cmd($vmid, 'query-pci');
4228     my $devices_to_check = [];
4229     my $devices = {};
4230     foreach my $pcibus (@$res) {
4231         push @$devices_to_check, @{$pcibus->{devices}},
4232     }
4233
4234     while (@$devices_to_check) {
4235         my $to_check = [];
4236         for my $d (@$devices_to_check) {
4237             $devices->{$d->{'qdev_id'}} = 1 if $d->{'qdev_id'};
4238             next if !$d->{'pci_bridge'};
4239
4240             $devices->{$d->{'qdev_id'}} += scalar(@{$d->{'pci_bridge'}->{devices}});
4241             push @$to_check, @{$d->{'pci_bridge'}->{devices}};
4242         }
4243         $devices_to_check = $to_check;
4244     }
4245
4246     my $resblock = vm_mon_cmd($vmid, 'query-block');
4247     foreach my $block (@$resblock) {
4248         if($block->{device} =~ m/^drive-(\S+)/){
4249                 $devices->{$1} = 1;
4250         }
4251     }
4252
4253     my $resmice = vm_mon_cmd($vmid, 'query-mice');
4254     foreach my $mice (@$resmice) {
4255         if ($mice->{name} eq 'QEMU HID Tablet') {
4256             $devices->{tablet} = 1;
4257             last;
4258         }
4259     }
4260
4261     # for usb devices there is no query-usb
4262     # but we can iterate over the entries in
4263     # qom-list path=/machine/peripheral
4264     my $resperipheral = vm_mon_cmd($vmid, 'qom-list', path => '/machine/peripheral');
4265     foreach my $per (@$resperipheral) {
4266         if ($per->{name} =~ m/^usb\d+$/) {
4267             $devices->{$per->{name}} = 1;
4268         }
4269     }
4270
4271     return $devices;
4272 }
4273
4274 sub vm_deviceplug {
4275     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4276
4277     my $q35 = machine_type_is_q35($conf);
4278
4279     my $devices_list = vm_devices_list($vmid);
4280     return 1 if defined($devices_list->{$deviceid});
4281
4282     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid, $arch, $machine_type); # add PCI bridge if we need it for the device
4283
4284     if ($deviceid eq 'tablet') {
4285
4286         qemu_deviceadd($vmid, print_tabletdevice_full($conf, $arch));
4287
4288     } elsif ($deviceid eq 'keyboard') {
4289
4290         qemu_deviceadd($vmid, print_keyboarddevice_full($conf, $arch));
4291
4292     } elsif ($deviceid =~ m/^usb(\d+)$/) {
4293
4294         die "usb hotplug currently not reliable\n";
4295         # since we can't reliably hot unplug all added usb devices
4296         # and usb passthrough disables live migration
4297         # we disable usb hotplugging for now
4298         qemu_deviceadd($vmid, PVE::QemuServer::USB::print_usbdevice_full($conf, $deviceid, $device));
4299
4300     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4301
4302         qemu_iothread_add($vmid, $deviceid, $device);
4303
4304         qemu_driveadd($storecfg, $vmid, $device);
4305         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, $arch, $machine_type);
4306
4307         qemu_deviceadd($vmid, $devicefull);
4308         eval { qemu_deviceaddverify($vmid, $deviceid); };
4309         if (my $err = $@) {
4310             eval { qemu_drivedel($vmid, $deviceid); };
4311             warn $@ if $@;
4312             die $err;
4313         }
4314
4315     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4316
4317
4318         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
4319         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
4320         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
4321
4322         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
4323
4324         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
4325             qemu_iothread_add($vmid, $deviceid, $device);
4326             $devicefull .= ",iothread=iothread-$deviceid";
4327         }
4328
4329         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
4330             $devicefull .= ",num_queues=$device->{queues}";
4331         }
4332
4333         qemu_deviceadd($vmid, $devicefull);
4334         qemu_deviceaddverify($vmid, $deviceid);
4335
4336     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4337
4338         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device, $arch, $machine_type);
4339         qemu_driveadd($storecfg, $vmid, $device);
4340
4341         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, $arch, $machine_type);
4342         eval { qemu_deviceadd($vmid, $devicefull); };
4343         if (my $err = $@) {
4344             eval { qemu_drivedel($vmid, $deviceid); };
4345             warn $@ if $@;
4346             die $err;
4347         }
4348
4349     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4350
4351         return undef if !qemu_netdevadd($vmid, $conf, $arch, $device, $deviceid);
4352
4353         my $machine_type = PVE::QemuServer::qemu_machine_pxe($vmid, $conf);
4354         my $use_old_bios_files = undef;
4355         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
4356
4357         my $netdevicefull = print_netdevice_full($vmid, $conf, $device, $deviceid, undef, $use_old_bios_files, $arch, $machine_type);
4358         qemu_deviceadd($vmid, $netdevicefull);
4359         eval {
4360             qemu_deviceaddverify($vmid, $deviceid);
4361             qemu_set_link_status($vmid, $deviceid, !$device->{link_down});
4362         };
4363         if (my $err = $@) {
4364             eval { qemu_netdevdel($vmid, $deviceid); };
4365             warn $@ if $@;
4366             die $err;
4367         }
4368
4369     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
4370
4371         my $bridgeid = $2;
4372         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
4373         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
4374
4375         qemu_deviceadd($vmid, $devicefull);
4376         qemu_deviceaddverify($vmid, $deviceid);
4377
4378     } else {
4379         die "can't hotplug device '$deviceid'\n";
4380     }
4381
4382     return 1;
4383 }
4384
4385 # fixme: this should raise exceptions on error!
4386 sub vm_deviceunplug {
4387     my ($vmid, $conf, $deviceid) = @_;
4388
4389     my $devices_list = vm_devices_list($vmid);
4390     return 1 if !defined($devices_list->{$deviceid});
4391
4392     die "can't unplug bootdisk" if $conf->{bootdisk} && $conf->{bootdisk} eq $deviceid;
4393
4394     if ($deviceid eq 'tablet' || $deviceid eq 'keyboard') {
4395
4396         qemu_devicedel($vmid, $deviceid);
4397
4398     } elsif ($deviceid =~ m/^usb\d+$/) {
4399
4400         die "usb hotplug currently not reliable\n";
4401         # when unplugging usb devices this way,
4402         # there may be remaining usb controllers/hubs
4403         # so we disable it for now
4404         qemu_devicedel($vmid, $deviceid);
4405         qemu_devicedelverify($vmid, $deviceid);
4406
4407     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4408
4409         qemu_devicedel($vmid, $deviceid);
4410         qemu_devicedelverify($vmid, $deviceid);
4411         qemu_drivedel($vmid, $deviceid);
4412         qemu_iothread_del($conf, $vmid, $deviceid);
4413
4414     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4415
4416         qemu_devicedel($vmid, $deviceid);
4417         qemu_devicedelverify($vmid, $deviceid);
4418         qemu_iothread_del($conf, $vmid, $deviceid);
4419
4420     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4421
4422         qemu_devicedel($vmid, $deviceid);
4423         qemu_drivedel($vmid, $deviceid);
4424         qemu_deletescsihw($conf, $vmid, $deviceid);
4425
4426     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4427
4428         qemu_devicedel($vmid, $deviceid);
4429         qemu_devicedelverify($vmid, $deviceid);
4430         qemu_netdevdel($vmid, $deviceid);
4431
4432     } else {
4433         die "can't unplug device '$deviceid'\n";
4434     }
4435
4436     return 1;
4437 }
4438
4439 sub qemu_deviceadd {
4440     my ($vmid, $devicefull) = @_;
4441
4442     $devicefull = "driver=".$devicefull;
4443     my %options =  split(/[=,]/, $devicefull);
4444
4445     vm_mon_cmd($vmid, "device_add" , %options);
4446 }
4447
4448 sub qemu_devicedel {
4449     my ($vmid, $deviceid) = @_;
4450
4451     my $ret = vm_mon_cmd($vmid, "device_del", id => $deviceid);
4452 }
4453
4454 sub qemu_iothread_add {
4455     my($vmid, $deviceid, $device) = @_;
4456
4457     if ($device->{iothread}) {
4458         my $iothreads = vm_iothreads_list($vmid);
4459         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
4460     }
4461 }
4462
4463 sub qemu_iothread_del {
4464     my($conf, $vmid, $deviceid) = @_;
4465
4466     my $confid = $deviceid;
4467     if ($deviceid =~ m/^(?:virtioscsi|scsihw)(\d+)$/) {
4468         $confid = 'scsi' . $1;
4469     }
4470     my $device = parse_drive($confid, $conf->{$confid});
4471     if ($device->{iothread}) {
4472         my $iothreads = vm_iothreads_list($vmid);
4473         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
4474     }
4475 }
4476
4477 sub qemu_objectadd {
4478     my($vmid, $objectid, $qomtype) = @_;
4479
4480     vm_mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
4481
4482     return 1;
4483 }
4484
4485 sub qemu_objectdel {
4486     my($vmid, $objectid) = @_;
4487
4488     vm_mon_cmd($vmid, "object-del", id => $objectid);
4489
4490     return 1;
4491 }
4492
4493 sub qemu_driveadd {
4494     my ($storecfg, $vmid, $device) = @_;
4495
4496     my $drive = print_drive_full($storecfg, $vmid, $device);
4497     $drive =~ s/\\/\\\\/g;
4498     my $ret = vm_human_monitor_command($vmid, "drive_add auto \"$drive\"");
4499
4500     # If the command succeeds qemu prints: "OK"
4501     return 1 if $ret =~ m/OK/s;
4502
4503     die "adding drive failed: $ret\n";
4504 }
4505
4506 sub qemu_drivedel {
4507     my($vmid, $deviceid) = @_;
4508
4509     my $ret = vm_human_monitor_command($vmid, "drive_del drive-$deviceid");
4510     $ret =~ s/^\s+//;
4511
4512     return 1 if $ret eq "";
4513
4514     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
4515     return 1 if $ret =~ m/Device \'.*?\' not found/s;
4516
4517     die "deleting drive $deviceid failed : $ret\n";
4518 }
4519
4520 sub qemu_deviceaddverify {
4521     my ($vmid, $deviceid) = @_;
4522
4523     for (my $i = 0; $i <= 5; $i++) {
4524          my $devices_list = vm_devices_list($vmid);
4525          return 1 if defined($devices_list->{$deviceid});
4526          sleep 1;
4527     }
4528
4529     die "error on hotplug device '$deviceid'\n";
4530 }
4531
4532
4533 sub qemu_devicedelverify {
4534     my ($vmid, $deviceid) = @_;
4535
4536     # need to verify that the device is correctly removed as device_del
4537     # is async and empty return is not reliable
4538
4539     for (my $i = 0; $i <= 5; $i++) {
4540          my $devices_list = vm_devices_list($vmid);
4541          return 1 if !defined($devices_list->{$deviceid});
4542          sleep 1;
4543     }
4544
4545     die "error on hot-unplugging device '$deviceid'\n";
4546 }
4547
4548 sub qemu_findorcreatescsihw {
4549     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4550
4551     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4552
4553     my $scsihwid="$controller_prefix$controller";
4554     my $devices_list = vm_devices_list($vmid);
4555
4556     if(!defined($devices_list->{$scsihwid})) {
4557         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device, $arch, $machine_type);
4558     }
4559
4560     return 1;
4561 }
4562
4563 sub qemu_deletescsihw {
4564     my ($conf, $vmid, $opt) = @_;
4565
4566     my $device = parse_drive($opt, $conf->{$opt});
4567
4568     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
4569         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
4570         return 1;
4571     }
4572
4573     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4574
4575     my $devices_list = vm_devices_list($vmid);
4576     foreach my $opt (keys %{$devices_list}) {
4577         if (PVE::QemuServer::is_valid_drivename($opt)) {
4578             my $drive = PVE::QemuServer::parse_drive($opt, $conf->{$opt});
4579             if($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
4580                 return 1;
4581             }
4582         }
4583     }
4584
4585     my $scsihwid="scsihw$controller";
4586
4587     vm_deviceunplug($vmid, $conf, $scsihwid);
4588
4589     return 1;
4590 }
4591
4592 sub qemu_add_pci_bridge {
4593     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4594
4595     my $bridges = {};
4596
4597     my $bridgeid;
4598
4599     print_pci_addr($device, $bridges, $arch, $machine_type);
4600
4601     while (my ($k, $v) = each %$bridges) {
4602         $bridgeid = $k;
4603     }
4604     return 1 if !defined($bridgeid) || $bridgeid < 1;
4605
4606     my $bridge = "pci.$bridgeid";
4607     my $devices_list = vm_devices_list($vmid);
4608
4609     if (!defined($devices_list->{$bridge})) {
4610         vm_deviceplug($storecfg, $conf, $vmid, $bridge, $arch, $machine_type);
4611     }
4612
4613     return 1;
4614 }
4615
4616 sub qemu_set_link_status {
4617     my ($vmid, $device, $up) = @_;
4618
4619     vm_mon_cmd($vmid, "set_link", name => $device,
4620                up => $up ? JSON::true : JSON::false);
4621 }
4622
4623 sub qemu_netdevadd {
4624     my ($vmid, $conf, $arch, $device, $deviceid) = @_;
4625
4626     my $netdev = print_netdev_full($vmid, $conf, $arch, $device, $deviceid, 1);
4627     my %options =  split(/[=,]/, $netdev);
4628
4629     vm_mon_cmd($vmid, "netdev_add",  %options);
4630     return 1;
4631 }
4632
4633 sub qemu_netdevdel {
4634     my ($vmid, $deviceid) = @_;
4635
4636     vm_mon_cmd($vmid, "netdev_del", id => $deviceid);
4637 }
4638
4639 sub qemu_usb_hotplug {
4640     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4641
4642     return if !$device;
4643
4644     # remove the old one first
4645     vm_deviceunplug($vmid, $conf, $deviceid);
4646
4647     # check if xhci controller is necessary and available
4648     if ($device->{usb3}) {
4649
4650         my $devicelist = vm_devices_list($vmid);
4651
4652         if (!$devicelist->{xhci}) {
4653             my $pciaddr = print_pci_addr("xhci", undef, $arch, $machine_type);
4654             qemu_deviceadd($vmid, "nec-usb-xhci,id=xhci$pciaddr");
4655         }
4656     }
4657     my $d = parse_usb_device($device->{host});
4658     $d->{usb3} = $device->{usb3};
4659
4660     # add the new one
4661     vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $d, $arch, $machine_type);
4662 }
4663
4664 sub qemu_cpu_hotplug {
4665     my ($vmid, $conf, $vcpus) = @_;
4666
4667     my $machine_type = PVE::QemuServer::get_current_qemu_machine($vmid);
4668
4669     my $sockets = 1;
4670     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
4671     $sockets = $conf->{sockets} if  $conf->{sockets};
4672     my $cores = $conf->{cores} || 1;
4673     my $maxcpus = $sockets * $cores;
4674
4675     $vcpus = $maxcpus if !$vcpus;
4676
4677     die "you can't add more vcpus than maxcpus\n"
4678         if $vcpus > $maxcpus;
4679
4680     my $currentvcpus = $conf->{vcpus} || $maxcpus;
4681
4682     if ($vcpus < $currentvcpus) {
4683
4684         if (qemu_machine_feature_enabled ($machine_type, undef, 2, 7)) {
4685
4686             for (my $i = $currentvcpus; $i > $vcpus; $i--) {
4687                 qemu_devicedel($vmid, "cpu$i");
4688                 my $retry = 0;
4689                 my $currentrunningvcpus = undef;
4690                 while (1) {
4691                     $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
4692                     last if scalar(@{$currentrunningvcpus}) == $i-1;
4693                     raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
4694                     $retry++;
4695                     sleep 1;
4696                 }
4697                 #update conf after each succesfull cpu unplug
4698                 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4699                 PVE::QemuConfig->write_config($vmid, $conf);
4700             }
4701         } else {
4702             die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
4703         }
4704
4705         return;
4706     }
4707
4708     my $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
4709     die "vcpus in running vm does not match its configuration\n"
4710         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
4711
4712     if (qemu_machine_feature_enabled ($machine_type, undef, 2, 7)) {
4713
4714         for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
4715             my $cpustr = print_cpu_device($conf, $i);
4716             qemu_deviceadd($vmid, $cpustr);
4717
4718             my $retry = 0;
4719             my $currentrunningvcpus = undef;
4720             while (1) {
4721                 $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
4722                 last if scalar(@{$currentrunningvcpus}) == $i;
4723                 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
4724                 sleep 1;
4725                 $retry++;
4726             }
4727             #update conf after each succesfull cpu hotplug
4728             $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4729             PVE::QemuConfig->write_config($vmid, $conf);
4730         }
4731     } else {
4732
4733         for (my $i = $currentvcpus; $i < $vcpus; $i++) {
4734             vm_mon_cmd($vmid, "cpu-add", id => int($i));
4735         }
4736     }
4737 }
4738
4739 sub qemu_block_set_io_throttle {
4740     my ($vmid, $deviceid,
4741         $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
4742         $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max,
4743         $bps_max_length, $bps_rd_max_length, $bps_wr_max_length,
4744         $iops_max_length, $iops_rd_max_length, $iops_wr_max_length) = @_;
4745
4746     return if !check_running($vmid) ;
4747
4748     vm_mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
4749         bps => int($bps),
4750         bps_rd => int($bps_rd),
4751         bps_wr => int($bps_wr),
4752         iops => int($iops),
4753         iops_rd => int($iops_rd),
4754         iops_wr => int($iops_wr),
4755         bps_max => int($bps_max),
4756         bps_rd_max => int($bps_rd_max),
4757         bps_wr_max => int($bps_wr_max),
4758         iops_max => int($iops_max),
4759         iops_rd_max => int($iops_rd_max),
4760         iops_wr_max => int($iops_wr_max),
4761         bps_max_length => int($bps_max_length),
4762         bps_rd_max_length => int($bps_rd_max_length),
4763         bps_wr_max_length => int($bps_wr_max_length),
4764         iops_max_length => int($iops_max_length),
4765         iops_rd_max_length => int($iops_rd_max_length),
4766         iops_wr_max_length => int($iops_wr_max_length),
4767     );
4768
4769 }
4770
4771 # old code, only used to shutdown old VM after update
4772 sub __read_avail {
4773     my ($fh, $timeout) = @_;
4774
4775     my $sel = new IO::Select;
4776     $sel->add($fh);
4777
4778     my $res = '';
4779     my $buf;
4780
4781     my @ready;
4782     while (scalar (@ready = $sel->can_read($timeout))) {
4783         my $count;
4784         if ($count = $fh->sysread($buf, 8192)) {
4785             if ($buf =~ /^(.*)\(qemu\) $/s) {
4786                 $res .= $1;
4787                 last;
4788             } else {
4789                 $res .= $buf;
4790             }
4791         } else {
4792             if (!defined($count)) {
4793                 die "$!\n";
4794             }
4795             last;
4796         }
4797     }
4798
4799     die "monitor read timeout\n" if !scalar(@ready);
4800
4801     return $res;
4802 }
4803
4804 sub qemu_block_resize {
4805     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4806
4807     my $running = check_running($vmid);
4808
4809     $size = 0 if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4810
4811     return if !$running;
4812
4813     vm_mon_cmd($vmid, "block_resize", device => $deviceid, size => int($size));
4814
4815 }
4816
4817 sub qemu_volume_snapshot {
4818     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4819
4820     my $running = check_running($vmid);
4821
4822     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4823         vm_mon_cmd($vmid, 'blockdev-snapshot-internal-sync', device => $deviceid, name => $snap);
4824     } else {
4825         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4826     }
4827 }
4828
4829 sub qemu_volume_snapshot_delete {
4830     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4831
4832     my $running = check_running($vmid);
4833
4834     if($running) {
4835
4836         $running = undef;
4837         my $conf = PVE::QemuConfig->load_config($vmid);
4838         foreach_drive($conf, sub {
4839             my ($ds, $drive) = @_;
4840             $running = 1 if $drive->{file} eq $volid;
4841         });
4842     }
4843
4844     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4845         vm_mon_cmd($vmid, 'blockdev-snapshot-delete-internal-sync', device => $deviceid, name => $snap);
4846     } else {
4847         PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4848     }
4849 }
4850
4851 sub set_migration_caps {
4852     my ($vmid) = @_;
4853
4854     my $cap_ref = [];
4855
4856     my $enabled_cap = {
4857         "auto-converge" => 1,
4858         "xbzrle" => 1,
4859         "x-rdma-pin-all" => 0,
4860         "zero-blocks" => 0,
4861         "compress" => 0
4862     };
4863
4864     my $supported_capabilities = vm_mon_cmd_nocheck($vmid, "query-migrate-capabilities");
4865
4866     for my $supported_capability (@$supported_capabilities) {
4867         push @$cap_ref, {
4868             capability => $supported_capability->{capability},
4869             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4870         };
4871     }
4872
4873     vm_mon_cmd_nocheck($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4874 }
4875
4876 my $fast_plug_option = {
4877     'lock' => 1,
4878     'name' => 1,
4879     'onboot' => 1,
4880     'shares' => 1,
4881     'startup' => 1,
4882     'description' => 1,
4883     'protection' => 1,
4884     'vmstatestorage' => 1,
4885     'hookscript' => 1,
4886 };
4887
4888 # hotplug changes in [PENDING]
4889 # $selection hash can be used to only apply specified options, for
4890 # example: { cores => 1 } (only apply changed 'cores')
4891 # $errors ref is used to return error messages
4892 sub vmconfig_hotplug_pending {
4893     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4894
4895     my $defaults = load_defaults();
4896     my ($arch, $machine_type) = get_basic_machine_info($conf, undef);
4897
4898     # commit values which do not have any impact on running VM first
4899     # Note: those option cannot raise errors, we we do not care about
4900     # $selection and always apply them.
4901
4902     my $add_error = sub {
4903         my ($opt, $msg) = @_;
4904         $errors->{$opt} = "hotplug problem - $msg";
4905     };
4906
4907     my $changes = 0;
4908     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4909         if ($fast_plug_option->{$opt}) {
4910             $conf->{$opt} = $conf->{pending}->{$opt};
4911             delete $conf->{pending}->{$opt};
4912             $changes = 1;
4913         }
4914     }
4915
4916     if ($changes) {
4917         PVE::QemuConfig->write_config($vmid, $conf);
4918         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4919     }
4920
4921     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4922
4923     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4924     while (my ($opt, $force) = each %$pending_delete_hash) {
4925         next if $selection && !$selection->{$opt};
4926         eval {
4927             if ($opt eq 'hotplug') {
4928                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
4929             } elsif ($opt eq 'tablet') {
4930                 die "skip\n" if !$hotplug_features->{usb};
4931                 if ($defaults->{tablet}) {
4932                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4933                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4934                         if $arch eq 'aarch64';
4935                 } else {
4936                     vm_deviceunplug($vmid, $conf, 'tablet');
4937                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4938                 }
4939             } elsif ($opt =~ m/^usb\d+/) {
4940                 die "skip\n";
4941                 # since we cannot reliably hot unplug usb devices
4942                 # we are disabling it
4943                 die "skip\n" if !$hotplug_features->{usb} || $conf->{$opt} =~ m/spice/i;
4944                 vm_deviceunplug($vmid, $conf, $opt);
4945             } elsif ($opt eq 'vcpus') {
4946                 die "skip\n" if !$hotplug_features->{cpu};
4947                 qemu_cpu_hotplug($vmid, $conf, undef);
4948             } elsif ($opt eq 'balloon') {
4949                 # enable balloon device is not hotpluggable
4950                 die "skip\n" if defined($conf->{balloon}) && $conf->{balloon} == 0;
4951                 # here we reset the ballooning value to memory
4952                 my $balloon = $conf->{memory} || $defaults->{memory};
4953                 vm_mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4954             } elsif ($fast_plug_option->{$opt}) {
4955                 # do nothing
4956             } elsif ($opt =~ m/^net(\d+)$/) {
4957                 die "skip\n" if !$hotplug_features->{network};
4958                 vm_deviceunplug($vmid, $conf, $opt);
4959             } elsif (is_valid_drivename($opt)) {
4960                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4961                 vm_deviceunplug($vmid, $conf, $opt);
4962                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4963             } elsif ($opt =~ m/^memory$/) {
4964                 die "skip\n" if !$hotplug_features->{memory};
4965                 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
4966             } elsif ($opt eq 'cpuunits') {
4967                 cgroups_write("cpu", $vmid, "cpu.shares", $defaults->{cpuunits});
4968             } elsif ($opt eq 'cpulimit') {
4969                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", -1);
4970             } else {
4971                 die "skip\n";
4972             }
4973         };
4974         if (my $err = $@) {
4975             &$add_error($opt, $err) if $err ne "skip\n";
4976         } else {
4977             # save new config if hotplug was successful
4978             delete $conf->{$opt};
4979             vmconfig_undelete_pending_option($conf, $opt);
4980             PVE::QemuConfig->write_config($vmid, $conf);
4981             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4982         }
4983     }
4984
4985     my $apply_pending_cloudinit;
4986     $apply_pending_cloudinit = sub {
4987         my ($key, $value) = @_;
4988         $apply_pending_cloudinit = sub {}; # once is enough
4989
4990         my @cloudinit_opts = keys %$confdesc_cloudinit;
4991         foreach my $opt (keys %{$conf->{pending}}) {
4992             next if !grep { $_ eq $opt } @cloudinit_opts;
4993             $conf->{$opt} = delete $conf->{pending}->{$opt};
4994         }
4995
4996         my $new_conf = { %$conf };
4997         $new_conf->{$key} = $value;
4998         PVE::QemuServer::Cloudinit::generate_cloudinitconfig($new_conf, $vmid);
4999     };
5000
5001     foreach my $opt (keys %{$conf->{pending}}) {
5002         next if $selection && !$selection->{$opt};
5003         my $value = $conf->{pending}->{$opt};
5004         eval {
5005             if ($opt eq 'hotplug') {
5006                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
5007             } elsif ($opt eq 'tablet') {
5008                 die "skip\n" if !$hotplug_features->{usb};
5009                 if ($value == 1) {
5010                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
5011                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
5012                         if $arch eq 'aarch64';
5013                 } elsif ($value == 0) {
5014                     vm_deviceunplug($vmid, $conf, 'tablet');
5015                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
5016                 }
5017             } elsif ($opt =~ m/^usb\d+$/) {
5018                 die "skip\n";
5019                 # since we cannot reliably hot unplug usb devices
5020                 # we are disabling it
5021                 die "skip\n" if !$hotplug_features->{usb} || $value =~ m/spice/i;
5022                 my $d = eval { PVE::JSONSchema::parse_property_string($usbdesc->{format}, $value) };
5023                 die "skip\n" if !$d;
5024                 qemu_usb_hotplug($storecfg, $conf, $vmid, $opt, $d, $arch, $machine_type);
5025             } elsif ($opt eq 'vcpus') {
5026                 die "skip\n" if !$hotplug_features->{cpu};
5027                 qemu_cpu_hotplug($vmid, $conf, $value);
5028             } elsif ($opt eq 'balloon') {
5029                 # enable/disable balloning device is not hotpluggable
5030                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
5031                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
5032                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
5033
5034                 # allow manual ballooning if shares is set to zero
5035                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
5036                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
5037                     vm_mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
5038                 }
5039             } elsif ($opt =~ m/^net(\d+)$/) {
5040                 # some changes can be done without hotplug
5041                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
5042                                     $vmid, $opt, $value, $arch, $machine_type);
5043             } elsif (is_valid_drivename($opt)) {
5044                 # some changes can be done without hotplug
5045                 my $drive = parse_drive($opt, $value);
5046                 if (drive_is_cloudinit($drive)) {
5047                     &$apply_pending_cloudinit($opt, $value);
5048                 }
5049                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
5050                                      $vmid, $opt, $value, 1, $arch, $machine_type);
5051             } elsif ($opt =~ m/^memory$/) { #dimms
5052                 die "skip\n" if !$hotplug_features->{memory};
5053                 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
5054             } elsif ($opt eq 'cpuunits') {
5055                 cgroups_write("cpu", $vmid, "cpu.shares", $conf->{pending}->{$opt});
5056             } elsif ($opt eq 'cpulimit') {
5057                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
5058                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", $cpulimit);
5059             } else {
5060                 die "skip\n";  # skip non-hot-pluggable options
5061             }
5062         };
5063         if (my $err = $@) {
5064             &$add_error($opt, $err) if $err ne "skip\n";
5065         } else {
5066             # save new config if hotplug was successful
5067             $conf->{$opt} = $value;
5068             delete $conf->{pending}->{$opt};
5069             PVE::QemuConfig->write_config($vmid, $conf);
5070             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5071         }
5072     }
5073 }
5074
5075 sub try_deallocate_drive {
5076     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
5077
5078     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
5079         my $volid = $drive->{file};
5080         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
5081             my $sid = PVE::Storage::parse_volume_id($volid);
5082             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
5083
5084             # check if the disk is really unused
5085             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
5086                 if is_volume_in_use($storecfg, $conf, $key, $volid);
5087             PVE::Storage::vdisk_free($storecfg, $volid);
5088             return 1;
5089         } else {
5090             # If vm is not owner of this disk remove from config
5091             return 1;
5092         }
5093     }
5094
5095     return undef;
5096 }
5097
5098 sub vmconfig_delete_or_detach_drive {
5099     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
5100
5101     my $drive = parse_drive($opt, $conf->{$opt});
5102
5103     my $rpcenv = PVE::RPCEnvironment::get();
5104     my $authuser = $rpcenv->get_user();
5105
5106     if ($force) {
5107         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
5108         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
5109     } else {
5110         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
5111     }
5112 }
5113
5114 sub vmconfig_apply_pending {
5115     my ($vmid, $conf, $storecfg) = @_;
5116
5117     # cold plug
5118
5119     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
5120     while (my ($opt, $force) = each %$pending_delete_hash) {
5121         die "internal error" if $opt =~ m/^unused/;
5122         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5123         if (!defined($conf->{$opt})) {
5124             vmconfig_undelete_pending_option($conf, $opt);
5125             PVE::QemuConfig->write_config($vmid, $conf);
5126         } elsif (is_valid_drivename($opt)) {
5127             vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
5128             vmconfig_undelete_pending_option($conf, $opt);
5129             delete $conf->{$opt};
5130             PVE::QemuConfig->write_config($vmid, $conf);
5131         } else {
5132             vmconfig_undelete_pending_option($conf, $opt);
5133             delete $conf->{$opt};
5134             PVE::QemuConfig->write_config($vmid, $conf);
5135         }
5136     }
5137
5138     $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5139
5140     foreach my $opt (keys %{$conf->{pending}}) { # add/change
5141         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5142
5143         if (defined($conf->{$opt}) && ($conf->{$opt} eq $conf->{pending}->{$opt})) {
5144             # skip if nothing changed
5145         } elsif (is_valid_drivename($opt)) {
5146             vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
5147                 if defined($conf->{$opt});
5148             $conf->{$opt} = $conf->{pending}->{$opt};
5149         } else {
5150             $conf->{$opt} = $conf->{pending}->{$opt};
5151         }
5152
5153         delete $conf->{pending}->{$opt};
5154         PVE::QemuConfig->write_config($vmid, $conf);
5155     }
5156 }
5157
5158 my $safe_num_ne = sub {
5159     my ($a, $b) = @_;
5160
5161     return 0 if !defined($a) && !defined($b);
5162     return 1 if !defined($a);
5163     return 1 if !defined($b);
5164
5165     return $a != $b;
5166 };
5167
5168 my $safe_string_ne = sub {
5169     my ($a, $b) = @_;
5170
5171     return 0 if !defined($a) && !defined($b);
5172     return 1 if !defined($a);
5173     return 1 if !defined($b);
5174
5175     return $a ne $b;
5176 };
5177
5178 sub vmconfig_update_net {
5179     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
5180
5181     my $newnet = parse_net($value);
5182
5183     if ($conf->{$opt}) {
5184         my $oldnet = parse_net($conf->{$opt});
5185
5186         if (&$safe_string_ne($oldnet->{model}, $newnet->{model}) ||
5187             &$safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
5188             &$safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
5189             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
5190
5191             # for non online change, we try to hot-unplug
5192             die "skip\n" if !$hotplug;
5193             vm_deviceunplug($vmid, $conf, $opt);
5194         } else {
5195
5196             die "internal error" if $opt !~ m/net(\d+)/;
5197             my $iface = "tap${vmid}i$1";
5198
5199             if (&$safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
5200                 &$safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
5201                 &$safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
5202                 &$safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
5203                 PVE::Network::tap_unplug($iface);
5204                 PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
5205             } elsif (&$safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
5206                 # Rate can be applied on its own but any change above needs to
5207                 # include the rate in tap_plug since OVS resets everything.
5208                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
5209             }
5210
5211             if (&$safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
5212                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
5213             }
5214
5215             return 1;
5216         }
5217     }
5218
5219     if ($hotplug) {
5220         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet, $arch, $machine_type);
5221     } else {
5222         die "skip\n";
5223     }
5224 }
5225
5226 sub vmconfig_update_disk {
5227     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $force, $arch, $machine_type) = @_;
5228
5229     # fixme: do we need force?
5230
5231     my $drive = parse_drive($opt, $value);
5232
5233     if ($conf->{$opt}) {
5234
5235         if (my $old_drive = parse_drive($opt, $conf->{$opt}))  {
5236
5237             my $media = $drive->{media} || 'disk';
5238             my $oldmedia = $old_drive->{media} || 'disk';
5239             die "unable to change media type\n" if $media ne $oldmedia;
5240
5241             if (!drive_is_cdrom($old_drive)) {
5242
5243                 if ($drive->{file} ne $old_drive->{file}) {
5244
5245                     die "skip\n" if !$hotplug;
5246
5247                     # unplug and register as unused
5248                     vm_deviceunplug($vmid, $conf, $opt);
5249                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
5250
5251                 } else {
5252                     # update existing disk
5253
5254                     # skip non hotpluggable value
5255                     if (&$safe_string_ne($drive->{discard}, $old_drive->{discard}) ||
5256                         &$safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
5257                         &$safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
5258                         &$safe_string_ne($drive->{cache}, $old_drive->{cache})) {
5259                         die "skip\n";
5260                     }
5261
5262                     # apply throttle
5263                     if (&$safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
5264                         &$safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
5265                         &$safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
5266                         &$safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
5267                         &$safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
5268                         &$safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
5269                         &$safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
5270                         &$safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
5271                         &$safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
5272                         &$safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
5273                         &$safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
5274                         &$safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max}) ||
5275                         &$safe_num_ne($drive->{bps_max_length}, $old_drive->{bps_max_length}) ||
5276                         &$safe_num_ne($drive->{bps_rd_max_length}, $old_drive->{bps_rd_max_length}) ||
5277                         &$safe_num_ne($drive->{bps_wr_max_length}, $old_drive->{bps_wr_max_length}) ||
5278                         &$safe_num_ne($drive->{iops_max_length}, $old_drive->{iops_max_length}) ||
5279                         &$safe_num_ne($drive->{iops_rd_max_length}, $old_drive->{iops_rd_max_length}) ||
5280                         &$safe_num_ne($drive->{iops_wr_max_length}, $old_drive->{iops_wr_max_length})) {
5281
5282                         qemu_block_set_io_throttle($vmid,"drive-$opt",
5283                                                    ($drive->{mbps} || 0)*1024*1024,
5284                                                    ($drive->{mbps_rd} || 0)*1024*1024,
5285                                                    ($drive->{mbps_wr} || 0)*1024*1024,
5286                                                    $drive->{iops} || 0,
5287                                                    $drive->{iops_rd} || 0,
5288                                                    $drive->{iops_wr} || 0,
5289                                                    ($drive->{mbps_max} || 0)*1024*1024,
5290                                                    ($drive->{mbps_rd_max} || 0)*1024*1024,
5291                                                    ($drive->{mbps_wr_max} || 0)*1024*1024,
5292                                                    $drive->{iops_max} || 0,
5293                                                    $drive->{iops_rd_max} || 0,
5294                                                    $drive->{iops_wr_max} || 0,
5295                                                    $drive->{bps_max_length} || 1,
5296                                                    $drive->{bps_rd_max_length} || 1,
5297                                                    $drive->{bps_wr_max_length} || 1,
5298                                                    $drive->{iops_max_length} || 1,
5299                                                    $drive->{iops_rd_max_length} || 1,
5300                                                    $drive->{iops_wr_max_length} || 1);
5301
5302                     }
5303
5304                     return 1;
5305                 }
5306
5307             } else { # cdrom
5308
5309                 if ($drive->{file} eq 'none') {
5310                     vm_mon_cmd($vmid, "eject",force => JSON::true,device => "drive-$opt");
5311                     if (drive_is_cloudinit($old_drive)) {
5312                         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive);
5313                     }
5314                 } else {
5315                     my $path = get_iso_path($storecfg, $vmid, $drive->{file});
5316                     vm_mon_cmd($vmid, "eject", force => JSON::true,device => "drive-$opt"); # force eject if locked
5317                     vm_mon_cmd($vmid, "change", device => "drive-$opt",target => "$path") if $path;
5318                 }
5319
5320                 return 1;
5321             }
5322         }
5323     }
5324
5325     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
5326     # hotplug new disks
5327     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
5328     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive, $arch, $machine_type);
5329 }
5330
5331 sub vm_start {
5332     my ($storecfg, $vmid, $statefile, $skiplock, $migratedfrom, $paused,
5333         $forcemachine, $spice_ticket, $migration_network, $migration_type, $targetstorage) = @_;
5334
5335     PVE::QemuConfig->lock_config($vmid, sub {
5336         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
5337
5338         die "you can't start a vm if it's a template\n" if PVE::QemuConfig->is_template($conf);
5339
5340         my $is_suspended = PVE::QemuConfig->has_lock($conf, 'suspended');
5341
5342         PVE::QemuConfig->check_lock($conf)
5343             if !($skiplock || $is_suspended);
5344
5345         die "VM $vmid already running\n" if check_running($vmid, undef, $migratedfrom);
5346
5347         # clean up leftover reboot request files
5348         eval { clear_reboot_request($vmid); };
5349         warn $@ if $@;
5350
5351         if (!$statefile && scalar(keys %{$conf->{pending}})) {
5352             vmconfig_apply_pending($vmid, $conf, $storecfg);
5353             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5354         }
5355
5356         PVE::QemuServer::Cloudinit::generate_cloudinitconfig($conf, $vmid);
5357
5358         my $defaults = load_defaults();
5359
5360         # set environment variable useful inside network script
5361         $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
5362
5363         my $local_volumes = {};
5364
5365         if ($targetstorage) {
5366             foreach_drive($conf, sub {
5367                 my ($ds, $drive) = @_;
5368
5369                 return if drive_is_cdrom($drive);
5370
5371                 my $volid = $drive->{file};
5372
5373                 return if !$volid;
5374
5375                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
5376
5377                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5378                 return if $scfg->{shared};
5379                 $local_volumes->{$ds} = [$volid, $storeid, $volname];
5380             });
5381
5382             my $format = undef;
5383
5384             foreach my $opt (sort keys %$local_volumes) {
5385
5386                 my ($volid, $storeid, $volname) = @{$local_volumes->{$opt}};
5387                 my $drive = parse_drive($opt, $conf->{$opt});
5388
5389                 #if remote storage is specified, use default format
5390                 if ($targetstorage && $targetstorage ne "1") {
5391                     $storeid = $targetstorage;
5392                     my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
5393                     $format = $defFormat;
5394                 } else {
5395                     #else we use same format than original
5396                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5397                     $format = qemu_img_format($scfg, $volid);
5398                 }
5399
5400                 my $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $format, undef, ($drive->{size}/1024));
5401                 my $newdrive = $drive;
5402                 $newdrive->{format} = $format;
5403                 $newdrive->{file} = $newvolid;
5404                 my $drivestr = PVE::QemuServer::print_drive($vmid, $newdrive);
5405                 $local_volumes->{$opt} = $drivestr;
5406                 #pass drive to conf for command line
5407                 $conf->{$opt} = $drivestr;
5408             }
5409         }
5410
5411         PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-start', 1);
5412
5413         if ($is_suspended) {
5414             # enforce machine type on suspended vm to ensure HW compatibility
5415             $forcemachine = $conf->{runningmachine};
5416             print "Resuming suspended VM\n";
5417         }
5418
5419         my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine);
5420
5421         my $migrate_uri;
5422         if ($statefile) {
5423             if ($statefile eq 'tcp') {
5424                 my $localip = "localhost";
5425                 my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
5426                 my $nodename = PVE::INotify::nodename();
5427
5428                 if (!defined($migration_type)) {
5429                     if (defined($datacenterconf->{migration}->{type})) {
5430                         $migration_type = $datacenterconf->{migration}->{type};
5431                     } else {
5432                         $migration_type = 'secure';
5433                     }
5434                 }
5435
5436                 if ($migration_type eq 'insecure') {
5437                     my $migrate_network_addr = PVE::Cluster::get_local_migration_ip($migration_network);
5438                     if ($migrate_network_addr) {
5439                         $localip = $migrate_network_addr;
5440                     } else {
5441                         $localip = PVE::Cluster::remote_node_ip($nodename, 1);
5442                     }
5443
5444                     $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5445                 }
5446
5447                 my $pfamily = PVE::Tools::get_host_address_family($nodename);
5448                 my $migrate_port = PVE::Tools::next_migrate_port($pfamily);
5449                 $migrate_uri = "tcp:${localip}:${migrate_port}";
5450                 push @$cmd, '-incoming', $migrate_uri;
5451                 push @$cmd, '-S';
5452
5453             } elsif ($statefile eq 'unix') {
5454                 # should be default for secure migrations as a ssh TCP forward
5455                 # tunnel is not deterministic reliable ready and fails regurarly
5456                 # to set up in time, so use UNIX socket forwards
5457                 my $socket_addr = "/run/qemu-server/$vmid.migrate";
5458                 unlink $socket_addr;
5459
5460                 $migrate_uri = "unix:$socket_addr";
5461
5462                 push @$cmd, '-incoming', $migrate_uri;
5463                 push @$cmd, '-S';
5464
5465             } elsif (-e $statefile) {
5466                 push @$cmd, '-loadstate', $statefile;
5467             } else {
5468                 my $statepath = PVE::Storage::path($storecfg, $statefile);
5469                 push @$vollist, $statepath;
5470                 push @$cmd, '-loadstate', $statepath;
5471             }
5472         } elsif ($paused) {
5473             push @$cmd, '-S';
5474         }
5475
5476         # host pci devices
5477         for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
5478           my $d = parse_hostpci($conf->{"hostpci$i"});
5479           next if !$d;
5480           my $pcidevices = $d->{pciid};
5481           foreach my $pcidevice (@$pcidevices) {
5482                 my $pciid = $pcidevice->{id};
5483
5484                 my $info = PVE::SysFSTools::pci_device_info("0000:$pciid");
5485                 die "IOMMU not present\n" if !PVE::SysFSTools::check_iommu_support();
5486                 die "no pci device info for device '$pciid'\n" if !$info;
5487
5488                 if ($d->{mdev}) {
5489                     my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $i);
5490                     PVE::SysFSTools::pci_create_mdev_device($pciid, $uuid, $d->{mdev});
5491                 } else {
5492                     die "can't unbind/bind pci group to vfio '$pciid'\n"
5493                         if !PVE::SysFSTools::pci_dev_group_bind_to_vfio($pciid);
5494                     die "can't reset pci device '$pciid'\n"
5495                         if $info->{has_fl_reset} and !PVE::SysFSTools::pci_dev_reset($info);
5496                 }
5497           }
5498         }
5499
5500         PVE::Storage::activate_volumes($storecfg, $vollist);
5501
5502         eval {
5503             run_command(['/bin/systemctl', 'stop', "$vmid.scope"],
5504                 outfunc => sub {}, errfunc => sub {});
5505         };
5506         # Issues with the above 'stop' not being fully completed are extremely rare, a very low
5507         # timeout should be more than enough here...
5508         PVE::Systemd::wait_for_unit_removed("$vmid.scope", 5);
5509
5510         my $cpuunits = defined($conf->{cpuunits}) ? $conf->{cpuunits}
5511                                                   : $defaults->{cpuunits};
5512
5513         my $start_timeout = ($conf->{hugepages} || $is_suspended) ? 300 : 30;
5514         my %run_params = (timeout => $statefile ? undef : $start_timeout, umask => 0077);
5515
5516         my %properties = (
5517             Slice => 'qemu.slice',
5518             KillMode => 'none',
5519             CPUShares => $cpuunits
5520         );
5521
5522         if (my $cpulimit = $conf->{cpulimit}) {
5523             $properties{CPUQuota} = int($cpulimit * 100);
5524         }
5525         $properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
5526
5527         my $run_qemu = sub {
5528             PVE::Tools::run_fork sub {
5529                 PVE::Systemd::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
5530                 run_command($cmd, %run_params);
5531             };
5532         };
5533
5534         if ($conf->{hugepages}) {
5535
5536             my $code = sub {
5537                 my $hugepages_topology = PVE::QemuServer::Memory::hugepages_topology($conf);
5538                 my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
5539
5540                 PVE::QemuServer::Memory::hugepages_mount();
5541                 PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
5542
5543                 eval { $run_qemu->() };
5544                 if (my $err = $@) {
5545                     PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology);
5546                     die $err;
5547                 }
5548
5549                 PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology);
5550             };
5551             eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
5552
5553         } else {
5554             eval { $run_qemu->() };
5555         }
5556
5557         if (my $err = $@) {
5558             # deactivate volumes if start fails
5559             eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
5560             die "start failed: $err";
5561         }
5562
5563         print "migration listens on $migrate_uri\n" if $migrate_uri;
5564
5565         if ($statefile && $statefile ne 'tcp' && $statefile ne 'unix')  {
5566             eval { vm_mon_cmd_nocheck($vmid, "cont"); };
5567             warn $@ if $@;
5568         }
5569
5570         #start nbd server for storage migration
5571         if ($targetstorage) {
5572             my $nodename = PVE::INotify::nodename();
5573             my $migrate_network_addr = PVE::Cluster::get_local_migration_ip($migration_network);
5574             my $localip = $migrate_network_addr ? $migrate_network_addr : PVE::Cluster::remote_node_ip($nodename, 1);
5575             my $pfamily = PVE::Tools::get_host_address_family($nodename);
5576             my $storage_migrate_port = PVE::Tools::next_migrate_port($pfamily);
5577
5578             vm_mon_cmd_nocheck($vmid, "nbd-server-start", addr => { type => 'inet', data => { host => "${localip}", port => "${storage_migrate_port}" } } );
5579
5580             $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5581
5582             foreach my $opt (sort keys %$local_volumes) {
5583                 my $volid = $local_volumes->{$opt};
5584                 vm_mon_cmd_nocheck($vmid, "nbd-server-add", device => "drive-$opt", writable => JSON::true );
5585                 my $migrate_storage_uri = "nbd:${localip}:${storage_migrate_port}:exportname=drive-$opt";
5586                 print "storage migration listens on $migrate_storage_uri volume:$volid\n";
5587             }
5588         }
5589
5590         if ($migratedfrom) {
5591             eval {
5592                 set_migration_caps($vmid);
5593             };
5594             warn $@ if $@;
5595
5596             if ($spice_port) {
5597                 print "spice listens on port $spice_port\n";
5598                 if ($spice_ticket) {
5599                     vm_mon_cmd_nocheck($vmid, "set_password", protocol => 'spice', password => $spice_ticket);
5600                     vm_mon_cmd_nocheck($vmid, "expire_password", protocol => 'spice', time => "+30");
5601                 }
5602             }
5603
5604         } else {
5605             vm_mon_cmd_nocheck($vmid, "balloon", value => $conf->{balloon}*1024*1024)
5606                 if !$statefile && $conf->{balloon};
5607
5608             foreach my $opt (keys %$conf) {
5609                 next if $opt !~  m/^net\d+$/;
5610                 my $nicconf = parse_net($conf->{$opt});
5611                 qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
5612             }
5613         }
5614
5615         vm_mon_cmd_nocheck($vmid, 'qom-set',
5616                     path => "machine/peripheral/balloon0",
5617                     property => "guest-stats-polling-interval",
5618                     value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
5619
5620         if ($is_suspended && (my $vmstate = $conf->{vmstate})) {
5621             print "Resumed VM, removing state\n";
5622             delete $conf->@{qw(lock vmstate runningmachine)};
5623             PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
5624             PVE::Storage::vdisk_free($storecfg, $vmstate);
5625             PVE::QemuConfig->write_config($vmid, $conf);
5626         }
5627
5628         PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'post-start');
5629     });
5630 }
5631
5632 sub vm_mon_cmd {
5633     my ($vmid, $execute, %params) = @_;
5634
5635     my $cmd = { execute => $execute, arguments => \%params };
5636     vm_qmp_command($vmid, $cmd);
5637 }
5638
5639 sub vm_mon_cmd_nocheck {
5640     my ($vmid, $execute, %params) = @_;
5641
5642     my $cmd = { execute => $execute, arguments => \%params };
5643     vm_qmp_command($vmid, $cmd, 1);
5644 }
5645
5646 sub vm_qmp_command {
5647     my ($vmid, $cmd, $nocheck) = @_;
5648
5649     my $res;
5650
5651     my $timeout;
5652     if ($cmd->{arguments}) {
5653         $timeout = delete $cmd->{arguments}->{timeout};
5654     }
5655
5656     eval {
5657         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
5658         my $sname = qmp_socket($vmid);
5659         if (-e $sname) { # test if VM is reasonambe new and supports qmp/qga
5660             my $qmpclient = PVE::QMPClient->new();
5661
5662             $res = $qmpclient->cmd($vmid, $cmd, $timeout);
5663         } else {
5664             die "unable to open monitor socket\n";
5665         }
5666     };
5667     if (my $err = $@) {
5668         syslog("err", "VM $vmid qmp command failed - $err");
5669         die $err;
5670     }
5671
5672     return $res;
5673 }
5674
5675 sub vm_human_monitor_command {
5676     my ($vmid, $cmdline) = @_;
5677
5678     my $cmd = {
5679         execute => 'human-monitor-command',
5680         arguments => { 'command-line' => $cmdline},
5681     };
5682
5683     return vm_qmp_command($vmid, $cmd);
5684 }
5685
5686 sub vm_commandline {
5687     my ($storecfg, $vmid, $snapname) = @_;
5688
5689     my $conf = PVE::QemuConfig->load_config($vmid);
5690
5691     if ($snapname) {
5692         my $snapshot = $conf->{snapshots}->{$snapname};
5693         die "snapshot '$snapname' does not exist\n" if !defined($snapshot);
5694
5695         $snapshot->{digest} = $conf->{digest}; # keep file digest for API
5696
5697         $conf = $snapshot;
5698     }
5699
5700     my $defaults = load_defaults();
5701
5702     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults);
5703
5704     return PVE::Tools::cmd2string($cmd);
5705 }
5706
5707 sub vm_reset {
5708     my ($vmid, $skiplock) = @_;
5709
5710     PVE::QemuConfig->lock_config($vmid, sub {
5711
5712         my $conf = PVE::QemuConfig->load_config($vmid);
5713
5714         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5715
5716         vm_mon_cmd($vmid, "system_reset");
5717     });
5718 }
5719
5720 sub get_vm_volumes {
5721     my ($conf) = @_;
5722
5723     my $vollist = [];
5724     foreach_volid($conf, sub {
5725         my ($volid, $attr) = @_;
5726
5727         return if $volid =~ m|^/|;
5728
5729         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5730         return if !$sid;
5731
5732         push @$vollist, $volid;
5733     });
5734
5735     return $vollist;
5736 }
5737
5738 sub vm_stop_cleanup {
5739     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
5740
5741     eval {
5742
5743         if (!$keepActive) {
5744             my $vollist = get_vm_volumes($conf);
5745             PVE::Storage::deactivate_volumes($storecfg, $vollist);
5746         }
5747
5748         foreach my $ext (qw(mon qmp pid vnc qga)) {
5749             unlink "/var/run/qemu-server/${vmid}.$ext";
5750         }
5751
5752         if ($conf->{ivshmem}) {
5753             my $ivshmem = PVE::JSONSchema::parse_property_string($ivshmem_fmt, $conf->{ivshmem});
5754             # just delete it for now, VMs which have this already open do not
5755             # are affected, but new VMs will get a separated one. If this
5756             # becomes an issue we either add some sort of ref-counting or just
5757             # add a "don't delete on stop" flag to the ivshmem format.
5758             unlink '/dev/shm/pve-shm-' . ($ivshmem->{name} // $vmid);
5759         }
5760
5761         foreach my $key (keys %$conf) {
5762             next if $key !~ m/^hostpci(\d+)$/;
5763             my $hostpciindex = $1;
5764             my $d = parse_hostpci($conf->{$key});
5765             my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $hostpciindex);
5766
5767             foreach my $pci (@{$d->{pciid}}) {
5768                 my $pciid = $pci->{id};
5769                 PVE::SysFSTools::pci_cleanup_mdev_device($pciid, $uuid);
5770             }
5771         }
5772
5773         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
5774     };
5775     warn $@ if $@; # avoid errors - just warn
5776 }
5777
5778 # call only in locked context
5779 sub _do_vm_stop {
5780     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive) = @_;
5781
5782     my $pid = check_running($vmid, $nocheck);
5783     return if !$pid;
5784
5785     my $conf;
5786     if (!$nocheck) {
5787         $conf = PVE::QemuConfig->load_config($vmid);
5788         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5789         if (!defined($timeout) && $shutdown && $conf->{startup}) {
5790             my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
5791             $timeout = $opts->{down} if $opts->{down};
5792         }
5793         PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-stop');
5794     }
5795
5796     eval {
5797         if ($shutdown) {
5798             if (defined($conf) && parse_guest_agent($conf)->{enabled}) {
5799                 vm_qmp_command($vmid, {
5800                         execute => "guest-shutdown",
5801                         arguments => { timeout => $timeout }
5802                     }, $nocheck);
5803             } else {
5804                 vm_qmp_command($vmid, { execute => "system_powerdown" }, $nocheck);
5805             }
5806         } else {
5807             vm_qmp_command($vmid, { execute => "quit" }, $nocheck);
5808         }
5809     };
5810     my $err = $@;
5811
5812     if (!$err) {
5813         $timeout = 60 if !defined($timeout);
5814
5815         my $count = 0;
5816         while (($count < $timeout) && check_running($vmid, $nocheck)) {
5817             $count++;
5818             sleep 1;
5819         }
5820
5821         if ($count >= $timeout) {
5822             if ($force) {
5823                 warn "VM still running - terminating now with SIGTERM\n";
5824                 kill 15, $pid;
5825             } else {
5826                 die "VM quit/powerdown failed - got timeout\n";
5827             }
5828         } else {
5829             vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5830             return;
5831         }
5832     } else {
5833         if ($force) {
5834             warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
5835             kill 15, $pid;
5836         } else {
5837             die "VM quit/powerdown failed\n";
5838         }
5839     }
5840
5841     # wait again
5842     $timeout = 10;
5843
5844     my $count = 0;
5845     while (($count < $timeout) && check_running($vmid, $nocheck)) {
5846         $count++;
5847         sleep 1;
5848     }
5849
5850     if ($count >= $timeout) {
5851         warn "VM still running - terminating now with SIGKILL\n";
5852         kill 9, $pid;
5853         sleep 1;
5854     }
5855
5856     vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5857 }
5858
5859 # Note: use $nocheck to skip tests if VM configuration file exists.
5860 # We need that when migration VMs to other nodes (files already moved)
5861 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
5862 sub vm_stop {
5863     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
5864
5865     $force = 1 if !defined($force) && !$shutdown;
5866
5867     if ($migratedfrom){
5868         my $pid = check_running($vmid, $nocheck, $migratedfrom);
5869         kill 15, $pid if $pid;
5870         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
5871         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
5872         return;
5873     }
5874
5875     PVE::QemuConfig->lock_config($vmid, sub {
5876         _do_vm_stop($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive);
5877    });
5878 }
5879
5880 sub vm_reboot {
5881     my ($vmid, $timeout) = @_;
5882
5883     PVE::QemuConfig->lock_config($vmid, sub {
5884
5885         # only reboot if running, as qmeventd starts it again on a stop event
5886         return if !check_running($vmid);
5887
5888         create_reboot_request($vmid);
5889
5890         my $storecfg = PVE::Storage::config();
5891         _do_vm_stop($storecfg, $vmid, undef, undef, $timeout, 1);
5892
5893    });
5894 }
5895
5896 sub vm_suspend {
5897     my ($vmid, $skiplock, $includestate, $statestorage) = @_;
5898
5899     my $conf;
5900     my $path;
5901     my $storecfg;
5902     my $vmstate;
5903
5904     PVE::QemuConfig->lock_config($vmid, sub {
5905
5906         $conf = PVE::QemuConfig->load_config($vmid);
5907
5908         my $is_backing_up = PVE::QemuConfig->has_lock($conf, 'backup');
5909         PVE::QemuConfig->check_lock($conf)
5910             if !($skiplock || $is_backing_up);
5911
5912         die "cannot suspend to disk during backup\n"
5913             if $is_backing_up && $includestate;
5914
5915         if ($includestate) {
5916             $conf->{lock} = 'suspending';
5917             my $date = strftime("%Y-%m-%d", localtime(time()));
5918             $storecfg = PVE::Storage::config();
5919             $vmstate = PVE::QemuConfig->__snapshot_save_vmstate($vmid, $conf, "suspend-$date", $storecfg, $statestorage, 1);
5920             $path = PVE::Storage::path($storecfg, $vmstate);
5921             PVE::QemuConfig->write_config($vmid, $conf);
5922         } else {
5923             vm_mon_cmd($vmid, "stop");
5924         }
5925     });
5926
5927     if ($includestate) {
5928         # save vm state
5929         PVE::Storage::activate_volumes($storecfg, [$vmstate]);
5930
5931         eval {
5932             vm_mon_cmd($vmid, "savevm-start", statefile => $path);
5933             for(;;) {
5934                 my $state = vm_mon_cmd_nocheck($vmid, "query-savevm");
5935                 if (!$state->{status}) {
5936                     die "savevm not active\n";
5937                 } elsif ($state->{status} eq 'active') {
5938                     sleep(1);
5939                     next;
5940                 } elsif ($state->{status} eq 'completed') {
5941                     print "State saved, quitting\n";
5942                     last;
5943                 } elsif ($state->{status} eq 'failed' && $state->{error}) {
5944                     die "query-savevm failed with error '$state->{error}'\n"
5945                 } else {
5946                     die "query-savevm returned status '$state->{status}'\n";
5947                 }
5948             }
5949         };
5950         my $err = $@;
5951
5952         PVE::QemuConfig->lock_config($vmid, sub {
5953             $conf = PVE::QemuConfig->load_config($vmid);
5954             if ($err) {
5955                 # cleanup, but leave suspending lock, to indicate something went wrong
5956                 eval {
5957                     vm_mon_cmd($vmid, "savevm-end");
5958                     PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
5959                     PVE::Storage::vdisk_free($storecfg, $vmstate);
5960                     delete $conf->@{qw(vmstate runningmachine)};
5961                     PVE::QemuConfig->write_config($vmid, $conf);
5962                 };
5963                 warn $@ if $@;
5964                 die $err;
5965             }
5966
5967             die "lock changed unexpectedly\n"
5968                 if !PVE::QemuConfig->has_lock($conf, 'suspending');
5969
5970             vm_qmp_command($vmid, { execute => "quit" });
5971             $conf->{lock} = 'suspended';
5972             PVE::QemuConfig->write_config($vmid, $conf);
5973         });
5974     }
5975 }
5976
5977 sub vm_resume {
5978     my ($vmid, $skiplock, $nocheck) = @_;
5979
5980     PVE::QemuConfig->lock_config($vmid, sub {
5981         my $vm_mon_cmd = $nocheck ? \&vm_mon_cmd_nocheck : \&vm_mon_cmd;
5982         my $res = $vm_mon_cmd->($vmid, 'query-status');
5983         my $resume_cmd = 'cont';
5984
5985         if ($res->{status} && $res->{status} eq 'suspended') {
5986             $resume_cmd = 'system_wakeup';
5987         }
5988
5989         if (!$nocheck) {
5990
5991             my $conf = PVE::QemuConfig->load_config($vmid);
5992
5993             PVE::QemuConfig->check_lock($conf)
5994                 if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
5995         }
5996
5997         $vm_mon_cmd->($vmid, $resume_cmd);
5998     });
5999 }
6000
6001 sub vm_sendkey {
6002     my ($vmid, $skiplock, $key) = @_;
6003
6004     PVE::QemuConfig->lock_config($vmid, sub {
6005
6006         my $conf = PVE::QemuConfig->load_config($vmid);
6007
6008         # there is no qmp command, so we use the human monitor command
6009         my $res = vm_human_monitor_command($vmid, "sendkey $key");
6010         die $res if $res ne '';
6011     });
6012 }
6013
6014 sub vm_destroy {
6015     my ($storecfg, $vmid, $skiplock) = @_;
6016
6017     PVE::QemuConfig->lock_config($vmid, sub {
6018
6019         my $conf = PVE::QemuConfig->load_config($vmid);
6020
6021         if (!check_running($vmid)) {
6022             destroy_vm($storecfg, $vmid, undef, $skiplock);
6023         } else {
6024             die "VM $vmid is running - destroy failed\n";
6025         }
6026     });
6027 }
6028
6029 # vzdump restore implementaion
6030
6031 sub tar_archive_read_firstfile {
6032     my $archive = shift;
6033
6034     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
6035
6036     # try to detect archive type first
6037     my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
6038         die "unable to open file '$archive'\n";
6039     my $firstfile = <$fh>;
6040     kill 15, $pid;
6041     close $fh;
6042
6043     die "ERROR: archive contaions no data\n" if !$firstfile;
6044     chomp $firstfile;
6045
6046     return $firstfile;
6047 }
6048
6049 sub tar_restore_cleanup {
6050     my ($storecfg, $statfile) = @_;
6051
6052     print STDERR "starting cleanup\n";
6053
6054     if (my $fd = IO::File->new($statfile, "r")) {
6055         while (defined(my $line = <$fd>)) {
6056             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
6057                 my $volid = $2;
6058                 eval {
6059                     if ($volid =~ m|^/|) {
6060                         unlink $volid || die 'unlink failed\n';
6061                     } else {
6062                         PVE::Storage::vdisk_free($storecfg, $volid);
6063                     }
6064                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
6065                 };
6066                 print STDERR "unable to cleanup '$volid' - $@" if $@;
6067             } else {
6068                 print STDERR "unable to parse line in statfile - $line";
6069             }
6070         }
6071         $fd->close();
6072     }
6073 }
6074
6075 sub restore_archive {
6076     my ($archive, $vmid, $user, $opts) = @_;
6077
6078     my $format = $opts->{format};
6079     my $comp;
6080
6081     if ($archive =~ m/\.tgz$/ || $archive =~ m/\.tar\.gz$/) {
6082         $format = 'tar' if !$format;
6083         $comp = 'gzip';
6084     } elsif ($archive =~ m/\.tar$/) {
6085         $format = 'tar' if !$format;
6086     } elsif ($archive =~ m/.tar.lzo$/) {
6087         $format = 'tar' if !$format;
6088         $comp = 'lzop';
6089     } elsif ($archive =~ m/\.vma$/) {
6090         $format = 'vma' if !$format;
6091     } elsif ($archive =~ m/\.vma\.gz$/) {
6092         $format = 'vma' if !$format;
6093         $comp = 'gzip';
6094     } elsif ($archive =~ m/\.vma\.lzo$/) {
6095         $format = 'vma' if !$format;
6096         $comp = 'lzop';
6097     } else {
6098         $format = 'vma' if !$format; # default
6099     }
6100
6101     # try to detect archive format
6102     if ($format eq 'tar') {
6103         return restore_tar_archive($archive, $vmid, $user, $opts);
6104     } else {
6105         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
6106     }
6107 }
6108
6109 sub restore_update_config_line {
6110     my ($outfd, $cookie, $vmid, $map, $line, $unique) = @_;
6111
6112     return if $line =~ m/^\#qmdump\#/;
6113     return if $line =~ m/^\#vzdump\#/;
6114     return if $line =~ m/^lock:/;
6115     return if $line =~ m/^unused\d+:/;
6116     return if $line =~ m/^parent:/;
6117
6118     my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
6119     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
6120         # try to convert old 1.X settings
6121         my ($id, $ind, $ethcfg) = ($1, $2, $3);
6122         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
6123             my ($model, $macaddr) = split(/\=/, $devconfig);
6124             $macaddr = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if !$macaddr || $unique;
6125             my $net = {
6126                 model => $model,
6127                 bridge => "vmbr$ind",
6128                 macaddr => $macaddr,
6129             };
6130             my $netstr = print_net($net);
6131
6132             print $outfd "net$cookie->{netcount}: $netstr\n";
6133             $cookie->{netcount}++;
6134         }
6135     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
6136         my ($id, $netstr) = ($1, $2);
6137         my $net = parse_net($netstr);
6138         $net->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if $net->{macaddr};
6139         $netstr = print_net($net);
6140         print $outfd "$id: $netstr\n";
6141     } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk)\d+):\s*(\S+)\s*$/) {
6142         my $virtdev = $1;
6143         my $value = $3;
6144         my $di = parse_drive($virtdev, $value);
6145         if (defined($di->{backup}) && !$di->{backup}) {
6146             print $outfd "#$line";
6147         } elsif ($map->{$virtdev}) {
6148             delete $di->{format}; # format can change on restore
6149             $di->{file} = $map->{$virtdev};
6150             $value = print_drive($vmid, $di);
6151             print $outfd "$virtdev: $value\n";
6152         } else {
6153             print $outfd $line;
6154         }
6155     } elsif (($line =~ m/^vmgenid: (.*)/)) {
6156         my $vmgenid = $1;
6157         if ($vmgenid ne '0') {
6158             # always generate a new vmgenid if there was a valid one setup
6159             $vmgenid = generate_uuid();
6160         }
6161         print $outfd "vmgenid: $vmgenid\n";
6162     } elsif (($line =~ m/^(smbios1: )(.*)/) && $unique) {
6163         my ($uuid, $uuid_str);
6164         UUID::generate($uuid);
6165         UUID::unparse($uuid, $uuid_str);
6166         my $smbios1 = parse_smbios1($2);
6167         $smbios1->{uuid} = $uuid_str;
6168         print $outfd $1.print_smbios1($smbios1)."\n";
6169     } else {
6170         print $outfd $line;
6171     }
6172 }
6173
6174 sub scan_volids {
6175     my ($cfg, $vmid) = @_;
6176
6177     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid);
6178
6179     my $volid_hash = {};
6180     foreach my $storeid (keys %$info) {
6181         foreach my $item (@{$info->{$storeid}}) {
6182             next if !($item->{volid} && $item->{size});
6183             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
6184             $volid_hash->{$item->{volid}} = $item;
6185         }
6186     }
6187
6188     return $volid_hash;
6189 }
6190
6191 sub is_volume_in_use {
6192     my ($storecfg, $conf, $skip_drive, $volid) = @_;
6193
6194     my $path = PVE::Storage::path($storecfg, $volid);
6195
6196     my $scan_config = sub {
6197         my ($cref, $snapname) = @_;
6198
6199         foreach my $key (keys %$cref) {
6200             my $value = $cref->{$key};
6201             if (is_valid_drivename($key)) {
6202                 next if $skip_drive && $key eq $skip_drive;
6203                 my $drive = parse_drive($key, $value);
6204                 next if !$drive || !$drive->{file} || drive_is_cdrom($drive);
6205                 return 1 if $volid eq $drive->{file};
6206                 if ($drive->{file} =~ m!^/!) {
6207                     return 1 if $drive->{file} eq $path;
6208                 } else {
6209                     my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file}, 1);
6210                     next if !$storeid;
6211                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid, 1);
6212                     next if !$scfg;
6213                     return 1 if $path eq PVE::Storage::path($storecfg, $drive->{file}, $snapname);
6214                 }
6215             }
6216         }
6217
6218         return 0;
6219     };
6220
6221     return 1 if &$scan_config($conf);
6222
6223     undef $skip_drive;
6224
6225     foreach my $snapname (keys %{$conf->{snapshots}}) {
6226         return 1 if &$scan_config($conf->{snapshots}->{$snapname}, $snapname);
6227     }
6228
6229     return 0;
6230 }
6231
6232 sub update_disksize {
6233     my ($vmid, $conf, $volid_hash) = @_;
6234
6235     my $changes;
6236     my $prefix = "VM $vmid:";
6237
6238     # used and unused disks
6239     my $referenced = {};
6240
6241     # Note: it is allowed to define multiple storages with same path (alias), so
6242     # we need to check both 'volid' and real 'path' (two different volid can point
6243     # to the same path).
6244
6245     my $referencedpath = {};
6246
6247     # update size info
6248     foreach my $opt (keys %$conf) {
6249         if (is_valid_drivename($opt)) {
6250             my $drive = parse_drive($opt, $conf->{$opt});
6251             my $volid = $drive->{file};
6252             next if !$volid;
6253
6254             $referenced->{$volid} = 1;
6255             if ($volid_hash->{$volid} &&
6256                 (my $path = $volid_hash->{$volid}->{path})) {
6257                 $referencedpath->{$path} = 1;
6258             }
6259
6260             next if drive_is_cdrom($drive);
6261             next if !$volid_hash->{$volid};
6262
6263             $drive->{size} = $volid_hash->{$volid}->{size};
6264             my $new = print_drive($vmid, $drive);
6265             if ($new ne $conf->{$opt}) {
6266                 $changes = 1;
6267                 $conf->{$opt} = $new;
6268                 print "$prefix update disk '$opt' information.\n";
6269             }
6270         }
6271     }
6272
6273     # remove 'unusedX' entry if volume is used
6274     foreach my $opt (keys %$conf) {
6275         next if $opt !~ m/^unused\d+$/;
6276         my $volid = $conf->{$opt};
6277         my $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
6278         if ($referenced->{$volid} || ($path && $referencedpath->{$path})) {
6279             print "$prefix remove entry '$opt', its volume '$volid' is in use.\n";
6280             $changes = 1;
6281             delete $conf->{$opt};
6282         }
6283
6284         $referenced->{$volid} = 1;
6285         $referencedpath->{$path} = 1 if $path;
6286     }
6287
6288     foreach my $volid (sort keys %$volid_hash) {
6289         next if $volid =~ m/vm-$vmid-state-/;
6290         next if $referenced->{$volid};
6291         my $path = $volid_hash->{$volid}->{path};
6292         next if !$path; # just to be sure
6293         next if $referencedpath->{$path};
6294         $changes = 1;
6295         my $key = PVE::QemuConfig->add_unused_volume($conf, $volid);
6296         print "$prefix add unreferenced volume '$volid' as '$key' to config.\n";
6297         $referencedpath->{$path} = 1; # avoid to add more than once (aliases)
6298     }
6299
6300     return $changes;
6301 }
6302
6303 sub rescan {
6304     my ($vmid, $nolock, $dryrun) = @_;
6305
6306     my $cfg = PVE::Storage::config();
6307
6308     # FIXME: Remove once our RBD plugin can handle CT and VM on a single storage
6309     # see: https://pve.proxmox.com/pipermail/pve-devel/2018-July/032900.html
6310     foreach my $stor (keys %{$cfg->{ids}}) {
6311         delete($cfg->{ids}->{$stor}) if ! $cfg->{ids}->{$stor}->{content}->{images};
6312     }
6313
6314     print "rescan volumes...\n";
6315     my $volid_hash = scan_volids($cfg, $vmid);
6316
6317     my $updatefn =  sub {
6318         my ($vmid) = @_;
6319
6320         my $conf = PVE::QemuConfig->load_config($vmid);
6321
6322         PVE::QemuConfig->check_lock($conf);
6323
6324         my $vm_volids = {};
6325         foreach my $volid (keys %$volid_hash) {
6326             my $info = $volid_hash->{$volid};
6327             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
6328         }
6329
6330         my $changes = update_disksize($vmid, $conf, $vm_volids);
6331
6332         PVE::QemuConfig->write_config($vmid, $conf) if $changes && !$dryrun;
6333     };
6334
6335     if (defined($vmid)) {
6336         if ($nolock) {
6337             &$updatefn($vmid);
6338         } else {
6339             PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6340         }
6341     } else {
6342         my $vmlist = config_list();
6343         foreach my $vmid (keys %$vmlist) {
6344             if ($nolock) {
6345                 &$updatefn($vmid);
6346             } else {
6347                 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6348             }
6349         }
6350     }
6351 }
6352
6353 sub restore_vma_archive {
6354     my ($archive, $vmid, $user, $opts, $comp) = @_;
6355
6356     my $readfrom = $archive;
6357
6358     my $cfg = PVE::Storage::config();
6359     my $commands = [];
6360     my $bwlimit = $opts->{bwlimit};
6361
6362     my $dbg_cmdstring = '';
6363     my $add_pipe = sub {
6364         my ($cmd) = @_;
6365         push @$commands, $cmd;
6366         $dbg_cmdstring .= ' | ' if length($dbg_cmdstring);
6367         $dbg_cmdstring .= PVE::Tools::cmd2string($cmd);
6368         $readfrom = '-';
6369     };
6370
6371     my $input = undef;
6372     if ($archive eq '-') {
6373         $input = '<&STDIN';
6374     } else {
6375         # If we use a backup from a PVE defined storage we also consider that
6376         # storage's rate limit:
6377         my (undef, $volid) = PVE::Storage::path_to_volume_id($cfg, $archive);
6378         if (defined($volid)) {
6379             my ($sid, undef) = PVE::Storage::parse_volume_id($volid);
6380             my $readlimit = PVE::Storage::get_bandwidth_limit('restore', [$sid], $bwlimit);
6381             if ($readlimit) {
6382                 print STDERR "applying read rate limit: $readlimit\n";
6383                 my $cstream = ['cstream', '-t', $readlimit*1024, '--', $readfrom];
6384                 $add_pipe->($cstream);
6385             }
6386         }
6387     }
6388
6389     if ($comp) {
6390         my $cmd;
6391         if ($comp eq 'gzip') {
6392             $cmd = ['zcat', $readfrom];
6393         } elsif ($comp eq 'lzop') {
6394             $cmd = ['lzop', '-d', '-c', $readfrom];
6395         } else {
6396             die "unknown compression method '$comp'\n";
6397         }
6398         $add_pipe->($cmd);
6399     }
6400
6401     my $tmpdir = "/var/tmp/vzdumptmp$$";
6402     rmtree $tmpdir;
6403
6404     # disable interrupts (always do cleanups)
6405     local $SIG{INT} =
6406         local $SIG{TERM} =
6407         local $SIG{QUIT} =
6408         local $SIG{HUP} = sub { warn "got interrupt - ignored\n"; };
6409
6410     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
6411     POSIX::mkfifo($mapfifo, 0600);
6412     my $fifofh;
6413
6414     my $openfifo = sub {
6415         open($fifofh, '>', $mapfifo) || die $!;
6416     };
6417
6418     $add_pipe->(['vma', 'extract', '-v', '-r', $mapfifo, $readfrom, $tmpdir]);
6419
6420     my $oldtimeout;
6421     my $timeout = 5;
6422
6423     my $devinfo = {};
6424
6425     my $rpcenv = PVE::RPCEnvironment::get();
6426
6427     my $conffile = PVE::QemuConfig->config_file($vmid);
6428     my $tmpfn = "$conffile.$$.tmp";
6429
6430     # Note: $oldconf is undef if VM does not exists
6431     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
6432     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
6433
6434     my %storage_limits;
6435
6436     my $print_devmap = sub {
6437         my $virtdev_hash = {};
6438
6439         my $cfgfn = "$tmpdir/qemu-server.conf";
6440
6441         # we can read the config - that is already extracted
6442         my $fh = IO::File->new($cfgfn, "r") ||
6443             "unable to read qemu-server.conf - $!\n";
6444
6445         my $fwcfgfn = "$tmpdir/qemu-server.fw";
6446         if (-f $fwcfgfn) {
6447             my $pve_firewall_dir = '/etc/pve/firewall';
6448             mkdir $pve_firewall_dir; # make sure the dir exists
6449             PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
6450         }
6451
6452         while (defined(my $line = <$fh>)) {
6453             if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
6454                 my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
6455                 die "archive does not contain data for drive '$virtdev'\n"
6456                     if !$devinfo->{$devname};
6457                 if (defined($opts->{storage})) {
6458                     $storeid = $opts->{storage} || 'local';
6459                 } elsif (!$storeid) {
6460                     $storeid = 'local';
6461                 }
6462                 $format = 'raw' if !$format;
6463                 $devinfo->{$devname}->{devname} = $devname;
6464                 $devinfo->{$devname}->{virtdev} = $virtdev;
6465                 $devinfo->{$devname}->{format} = $format;
6466                 $devinfo->{$devname}->{storeid} = $storeid;
6467
6468                 # check permission on storage
6469                 my $pool = $opts->{pool}; # todo: do we need that?
6470                 if ($user ne 'root@pam') {
6471                     $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
6472                 }
6473
6474                 $storage_limits{$storeid} = $bwlimit;
6475
6476                 $virtdev_hash->{$virtdev} = $devinfo->{$devname};
6477             } elsif ($line =~ m/^((?:ide|sata|scsi)\d+):\s*(.*)\s*$/) {
6478                 my $virtdev = $1;
6479                 my $drive = parse_drive($virtdev, $2);
6480                 if (drive_is_cloudinit($drive)) {
6481                     my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6482                     my $scfg = PVE::Storage::storage_config($cfg, $storeid);
6483                     my $format = qemu_img_format($scfg, $volname); # has 'raw' fallback
6484
6485                     my $d = {
6486                         format => $format,
6487                         storeid => $opts->{storage} // $storeid,
6488                         size => PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE,
6489                         file => $drive->{file}, # to make drive_is_cloudinit check possible
6490                         name => "vm-$vmid-cloudinit",
6491                         is_cloudinit => 1,
6492                     };
6493                     $virtdev_hash->{$virtdev} = $d;
6494                 }
6495             }
6496         }
6497
6498         foreach my $key (keys %storage_limits) {
6499             my $limit = PVE::Storage::get_bandwidth_limit('restore', [$key], $bwlimit);
6500             next if !$limit;
6501             print STDERR "rate limit for storage $key: $limit KiB/s\n";
6502             $storage_limits{$key} = $limit * 1024;
6503         }
6504
6505         foreach my $devname (keys %$devinfo) {
6506             die "found no device mapping information for device '$devname'\n"
6507                 if !$devinfo->{$devname}->{virtdev};
6508         }
6509
6510         # create empty/temp config
6511         if ($oldconf) {
6512             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
6513             foreach_drive($oldconf, sub {
6514                 my ($ds, $drive) = @_;
6515
6516                 return if drive_is_cdrom($drive, 1);
6517
6518                 my $volid = $drive->{file};
6519                 return if !$volid || $volid =~ m|^/|;
6520
6521                 my ($path, $owner) = PVE::Storage::path($cfg, $volid);
6522                 return if !$path || !$owner || ($owner != $vmid);
6523
6524                 # Note: only delete disk we want to restore
6525                 # other volumes will become unused
6526                 if ($virtdev_hash->{$ds}) {
6527                     eval { PVE::Storage::vdisk_free($cfg, $volid); };
6528                     if (my $err = $@) {
6529                         warn $err;
6530                     }
6531                 }
6532             });
6533
6534             # delete vmstate files, after the restore we have no snapshots anymore
6535             foreach my $snapname (keys %{$oldconf->{snapshots}}) {
6536                 my $snap = $oldconf->{snapshots}->{$snapname};
6537                 if ($snap->{vmstate}) {
6538                     eval { PVE::Storage::vdisk_free($cfg, $snap->{vmstate}); };
6539                     if (my $err = $@) {
6540                         warn $err;
6541                     }
6542                 }
6543             }
6544         }
6545
6546         my $map = {};
6547         foreach my $virtdev (sort keys %$virtdev_hash) {
6548             my $d = $virtdev_hash->{$virtdev};
6549             my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
6550             my $storeid = $d->{storeid};
6551             my $scfg = PVE::Storage::storage_config($cfg, $storeid);
6552
6553             my $map_opts = '';
6554             if (my $limit = $storage_limits{$storeid}) {
6555                 $map_opts .= "throttling.bps=$limit:throttling.group=$storeid:";
6556             }
6557
6558             # test if requested format is supported
6559             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($cfg, $storeid);
6560             my $supported = grep { $_ eq $d->{format} } @$validFormats;
6561             $d->{format} = $defFormat if !$supported;
6562
6563             my $name;
6564             if ($d->{is_cloudinit}) {
6565                 $name = $d->{name};
6566                 $name .= ".$d->{format}" if $d->{format} ne 'raw';
6567             }
6568
6569             my $volid = PVE::Storage::vdisk_alloc($cfg, $storeid, $vmid, $d->{format}, $name, $alloc_size);
6570             print STDERR "new volume ID is '$volid'\n";
6571             $d->{volid} = $volid;
6572
6573             PVE::Storage::activate_volumes($cfg, [$volid]);
6574
6575             my $write_zeros = 1;
6576             if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
6577                 $write_zeros = 0;
6578             }
6579
6580             if (!$d->{is_cloudinit}) {
6581                 my $path = PVE::Storage::path($cfg, $volid);
6582
6583                 print $fifofh "${map_opts}format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
6584
6585                 print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
6586             }
6587             $map->{$virtdev} = $volid;
6588         }
6589
6590         $fh->seek(0, 0) || die "seek failed - $!\n";
6591
6592         my $outfd = new IO::File ($tmpfn, "w") ||
6593             die "unable to write config for VM $vmid\n";
6594
6595         my $cookie = { netcount => 0 };
6596         while (defined(my $line = <$fh>)) {
6597             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
6598         }
6599
6600         $fh->close();
6601         $outfd->close();
6602     };
6603
6604     eval {
6605         # enable interrupts
6606         local $SIG{INT} =
6607             local $SIG{TERM} =
6608             local $SIG{QUIT} =
6609             local $SIG{HUP} =
6610             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6611         local $SIG{ALRM} = sub { die "got timeout\n"; };
6612
6613         $oldtimeout = alarm($timeout);
6614
6615         my $parser = sub {
6616             my $line = shift;
6617
6618             print "$line\n";
6619
6620             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
6621                 my ($dev_id, $size, $devname) = ($1, $2, $3);
6622                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
6623             } elsif ($line =~ m/^CTIME: /) {
6624                 # we correctly received the vma config, so we can disable
6625                 # the timeout now for disk allocation (set to 10 minutes, so
6626                 # that we always timeout if something goes wrong)
6627                 alarm(600);
6628                 &$print_devmap();
6629                 print $fifofh "done\n";
6630                 my $tmp = $oldtimeout || 0;
6631                 $oldtimeout = undef;
6632                 alarm($tmp);
6633                 close($fifofh);
6634             }
6635         };
6636
6637         print "restore vma archive: $dbg_cmdstring\n";
6638         run_command($commands, input => $input, outfunc => $parser, afterfork => $openfifo);
6639     };
6640     my $err = $@;
6641
6642     alarm($oldtimeout) if $oldtimeout;
6643
6644     my $vollist = [];
6645     foreach my $devname (keys %$devinfo) {
6646         my $volid = $devinfo->{$devname}->{volid};
6647         push @$vollist, $volid if $volid;
6648     }
6649
6650     PVE::Storage::deactivate_volumes($cfg, $vollist);
6651
6652     unlink $mapfifo;
6653
6654     if ($err) {
6655         rmtree $tmpdir;
6656         unlink $tmpfn;
6657
6658         foreach my $devname (keys %$devinfo) {
6659             my $volid = $devinfo->{$devname}->{volid};
6660             next if !$volid;
6661             eval {
6662                 if ($volid =~ m|^/|) {
6663                     unlink $volid || die 'unlink failed\n';
6664                 } else {
6665                     PVE::Storage::vdisk_free($cfg, $volid);
6666                 }
6667                 print STDERR "temporary volume '$volid' sucessfuly removed\n";
6668             };
6669             print STDERR "unable to cleanup '$volid' - $@" if $@;
6670         }
6671         die $err;
6672     }
6673
6674     rmtree $tmpdir;
6675
6676     rename($tmpfn, $conffile) ||
6677         die "unable to commit configuration file '$conffile'\n";
6678
6679     PVE::Cluster::cfs_update(); # make sure we read new file
6680
6681     eval { rescan($vmid, 1); };
6682     warn $@ if $@;
6683 }
6684
6685 sub restore_tar_archive {
6686     my ($archive, $vmid, $user, $opts) = @_;
6687
6688     if ($archive ne '-') {
6689         my $firstfile = tar_archive_read_firstfile($archive);
6690         die "ERROR: file '$archive' dos not lock like a QemuServer vzdump backup\n"
6691             if $firstfile ne 'qemu-server.conf';
6692     }
6693
6694     my $storecfg = PVE::Storage::config();
6695
6696     # destroy existing data - keep empty config
6697     my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
6698     destroy_vm($storecfg, $vmid, 1) if -f $vmcfgfn;
6699
6700     my $tocmd = "/usr/lib/qemu-server/qmextract";
6701
6702     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
6703     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
6704     $tocmd .= ' --prealloc' if $opts->{prealloc};
6705     $tocmd .= ' --info' if $opts->{info};
6706
6707     # tar option "xf" does not autodetect compression when read from STDIN,
6708     # so we pipe to zcat
6709     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
6710         PVE::Tools::shellquote("--to-command=$tocmd");
6711
6712     my $tmpdir = "/var/tmp/vzdumptmp$$";
6713     mkpath $tmpdir;
6714
6715     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
6716     local $ENV{VZDUMP_VMID} = $vmid;
6717     local $ENV{VZDUMP_USER} = $user;
6718
6719     my $conffile = PVE::QemuConfig->config_file($vmid);
6720     my $tmpfn = "$conffile.$$.tmp";
6721
6722     # disable interrupts (always do cleanups)
6723     local $SIG{INT} =
6724         local $SIG{TERM} =
6725         local $SIG{QUIT} =
6726         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
6727
6728     eval {
6729         # enable interrupts
6730         local $SIG{INT} =
6731             local $SIG{TERM} =
6732             local $SIG{QUIT} =
6733             local $SIG{HUP} =
6734             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6735
6736         if ($archive eq '-') {
6737             print "extracting archive from STDIN\n";
6738             run_command($cmd, input => "<&STDIN");
6739         } else {
6740             print "extracting archive '$archive'\n";
6741             run_command($cmd);
6742         }
6743
6744         return if $opts->{info};
6745
6746         # read new mapping
6747         my $map = {};
6748         my $statfile = "$tmpdir/qmrestore.stat";
6749         if (my $fd = IO::File->new($statfile, "r")) {
6750             while (defined (my $line = <$fd>)) {
6751                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
6752                     $map->{$1} = $2 if $1;
6753                 } else {
6754                     print STDERR "unable to parse line in statfile - $line\n";
6755                 }
6756             }
6757             $fd->close();
6758         }
6759
6760         my $confsrc = "$tmpdir/qemu-server.conf";
6761
6762         my $srcfd = new IO::File($confsrc, "r") ||
6763             die "unable to open file '$confsrc'\n";
6764
6765         my $outfd = new IO::File ($tmpfn, "w") ||
6766             die "unable to write config for VM $vmid\n";
6767
6768         my $cookie = { netcount => 0 };
6769         while (defined (my $line = <$srcfd>)) {
6770             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
6771         }
6772
6773         $srcfd->close();
6774         $outfd->close();
6775     };
6776     my $err = $@;
6777
6778     if ($err) {
6779
6780         unlink $tmpfn;
6781
6782         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
6783
6784         die $err;
6785     }
6786
6787     rmtree $tmpdir;
6788
6789     rename $tmpfn, $conffile ||
6790         die "unable to commit configuration file '$conffile'\n";
6791
6792     PVE::Cluster::cfs_update(); # make sure we read new file
6793
6794     eval { rescan($vmid, 1); };
6795     warn $@ if $@;
6796 };
6797
6798 sub foreach_storage_used_by_vm {
6799     my ($conf, $func) = @_;
6800
6801     my $sidhash = {};
6802
6803     foreach_drive($conf, sub {
6804         my ($ds, $drive) = @_;
6805         return if drive_is_cdrom($drive);
6806
6807         my $volid = $drive->{file};
6808
6809         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
6810         $sidhash->{$sid} = $sid if $sid;
6811     });
6812
6813     foreach my $sid (sort keys %$sidhash) {
6814         &$func($sid);
6815     }
6816 }
6817
6818 sub do_snapshots_with_qemu {
6819     my ($storecfg, $volid) = @_;
6820
6821     my $storage_name = PVE::Storage::parse_volume_id($volid);
6822     my $scfg = $storecfg->{ids}->{$storage_name};
6823
6824     if ($qemu_snap_storage->{$scfg->{type}} && !$scfg->{krbd}){
6825         return 1;
6826     }
6827
6828     if ($volid =~ m/\.(qcow2|qed)$/){
6829         return 1;
6830     }
6831
6832     return undef;
6833 }
6834
6835 sub qga_check_running {
6836     my ($vmid, $nowarn) = @_;
6837
6838     eval { vm_mon_cmd($vmid, "guest-ping", timeout => 3); };
6839     if ($@) {
6840         warn "Qemu Guest Agent is not running - $@" if !$nowarn;
6841         return 0;
6842     }
6843     return 1;
6844 }
6845
6846 sub template_create {
6847     my ($vmid, $conf, $disk) = @_;
6848
6849     my $storecfg = PVE::Storage::config();
6850
6851     foreach_drive($conf, sub {
6852         my ($ds, $drive) = @_;
6853
6854         return if drive_is_cdrom($drive);
6855         return if $disk && $ds ne $disk;
6856
6857         my $volid = $drive->{file};
6858         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
6859
6860         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
6861         $drive->{file} = $voliddst;
6862         $conf->{$ds} = print_drive($vmid, $drive);
6863         PVE::QemuConfig->write_config($vmid, $conf);
6864     });
6865 }
6866
6867 sub convert_iscsi_path {
6868     my ($path) = @_;
6869
6870     if ($path =~ m|^iscsi://([^/]+)/([^/]+)/(.+)$|) {
6871         my $portal = $1;
6872         my $target = $2;
6873         my $lun = $3;
6874
6875         my $initiator_name = get_initiator_name();
6876
6877         return "file.driver=iscsi,file.transport=tcp,file.initiator-name=$initiator_name,".
6878                "file.portal=$portal,file.target=$target,file.lun=$lun,driver=raw";
6879     }
6880
6881     die "cannot convert iscsi path '$path', unkown format\n";
6882 }
6883
6884 sub qemu_img_convert {
6885     my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized) = @_;
6886
6887     my $storecfg = PVE::Storage::config();
6888     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
6889     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
6890
6891     die "destination '$dst_volid' is not a valid volid form qemu-img convert\n" if !$dst_storeid;
6892
6893     my $cachemode;
6894     my $src_path;
6895     my $src_is_iscsi = 0;
6896     my $src_format = 'raw';
6897
6898     if ($src_storeid) {
6899         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
6900         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
6901         $src_format = qemu_img_format($src_scfg, $src_volname);
6902         $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
6903         $src_is_iscsi = ($src_path =~ m|^iscsi://|);
6904         $cachemode = 'none' if $src_scfg->{type} eq 'zfspool';
6905     } elsif (-f $src_volid) {
6906         $src_path = $src_volid;
6907         if ($src_path =~ m/\.($QEMU_FORMAT_RE)$/) {
6908             $src_format = $1;
6909         }
6910     }
6911
6912     die "source '$src_volid' is not a valid volid nor path for qemu-img convert\n" if !$src_path;
6913
6914     my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6915     my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
6916     my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6917     my $dst_is_iscsi = ($dst_path =~ m|^iscsi://|);
6918
6919     my $cmd = [];
6920     push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
6921     push @$cmd, '-l', "snapshot.name=$snapname" if($snapname && $src_format eq "qcow2");
6922     push @$cmd, '-t', 'none' if $dst_scfg->{type} eq 'zfspool';
6923     push @$cmd, '-T', $cachemode if defined($cachemode);
6924
6925     if ($src_is_iscsi) {
6926         push @$cmd, '--image-opts';
6927         $src_path = convert_iscsi_path($src_path);
6928     } else {
6929         push @$cmd, '-f', $src_format;
6930     }
6931
6932     if ($dst_is_iscsi) {
6933         push @$cmd, '--target-image-opts';
6934         $dst_path = convert_iscsi_path($dst_path);
6935     } else {
6936         push @$cmd, '-O', $dst_format;
6937     }
6938
6939     push @$cmd, $src_path;
6940
6941     if (!$dst_is_iscsi && $is_zero_initialized) {
6942         push @$cmd, "zeroinit:$dst_path";
6943     } else {
6944         push @$cmd, $dst_path;
6945     }
6946
6947     my $parser = sub {
6948         my $line = shift;
6949         if($line =~ m/\((\S+)\/100\%\)/){
6950             my $percent = $1;
6951             my $transferred = int($size * $percent / 100);
6952             my $remaining = $size - $transferred;
6953
6954             print "transferred: $transferred bytes remaining: $remaining bytes total: $size bytes progression: $percent %\n";
6955         }
6956
6957     };
6958
6959     eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
6960     my $err = $@;
6961     die "copy failed: $err" if $err;
6962 }
6963
6964 sub qemu_img_format {
6965     my ($scfg, $volname) = @_;
6966
6967     if ($scfg->{path} && $volname =~ m/\.($QEMU_FORMAT_RE)$/) {
6968         return $1;
6969     } else {
6970         return "raw";
6971     }
6972 }
6973
6974 sub qemu_drive_mirror {
6975     my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $skipcomplete, $qga, $bwlimit) = @_;
6976
6977     $jobs = {} if !$jobs;
6978
6979     my $qemu_target;
6980     my $format;
6981     $jobs->{"drive-$drive"} = {};
6982
6983     if ($dst_volid =~ /^nbd:/) {
6984         $qemu_target = $dst_volid;
6985         $format = "nbd";
6986     } else {
6987         my $storecfg = PVE::Storage::config();
6988         my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
6989
6990         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6991
6992         $format = qemu_img_format($dst_scfg, $dst_volname);
6993
6994         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6995
6996         $qemu_target = $is_zero_initialized ? "zeroinit:$dst_path" : $dst_path;
6997     }
6998
6999     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $qemu_target };
7000     $opts->{format} = $format if $format;
7001
7002     if (defined($bwlimit)) {
7003         $opts->{speed} = $bwlimit * 1024;
7004         print "drive mirror is starting for drive-$drive with bandwidth limit: ${bwlimit} KB/s\n";
7005     } else {
7006         print "drive mirror is starting for drive-$drive\n";
7007     }
7008
7009     # if a job already runs for this device we get an error, catch it for cleanup
7010     eval { vm_mon_cmd($vmid, "drive-mirror", %$opts); };
7011     if (my $err = $@) {
7012         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
7013         warn "$@\n" if $@;
7014         die "mirroring error: $err\n";
7015     }
7016
7017     qemu_drive_mirror_monitor ($vmid, $vmiddst, $jobs, $skipcomplete, $qga);
7018 }
7019
7020 sub qemu_drive_mirror_monitor {
7021     my ($vmid, $vmiddst, $jobs, $skipcomplete, $qga) = @_;
7022
7023     eval {
7024         my $err_complete = 0;
7025
7026         while (1) {
7027             die "storage migration timed out\n" if $err_complete > 300;
7028
7029             my $stats = vm_mon_cmd($vmid, "query-block-jobs");
7030
7031             my $running_mirror_jobs = {};
7032             foreach my $stat (@$stats) {
7033                 next if $stat->{type} ne 'mirror';
7034                 $running_mirror_jobs->{$stat->{device}} = $stat;
7035             }
7036
7037             my $readycounter = 0;
7038
7039             foreach my $job (keys %$jobs) {
7040
7041                 if(defined($jobs->{$job}->{complete}) && !defined($running_mirror_jobs->{$job})) {
7042                     print "$job : finished\n";
7043                     delete $jobs->{$job};
7044                     next;
7045                 }
7046
7047                 die "$job: mirroring has been cancelled\n" if !defined($running_mirror_jobs->{$job});
7048
7049                 my $busy = $running_mirror_jobs->{$job}->{busy};
7050                 my $ready = $running_mirror_jobs->{$job}->{ready};
7051                 if (my $total = $running_mirror_jobs->{$job}->{len}) {
7052                     my $transferred = $running_mirror_jobs->{$job}->{offset} || 0;
7053                     my $remaining = $total - $transferred;
7054                     my $percent = sprintf "%.2f", ($transferred * 100 / $total);
7055
7056                     print "$job: transferred: $transferred bytes remaining: $remaining bytes total: $total bytes progression: $percent % busy: $busy ready: $ready \n";
7057                 }
7058
7059                 $readycounter++ if $running_mirror_jobs->{$job}->{ready};
7060             }
7061
7062             last if scalar(keys %$jobs) == 0;
7063
7064             if ($readycounter == scalar(keys %$jobs)) {
7065                 print "all mirroring jobs are ready \n";
7066                 last if $skipcomplete; #do the complete later
7067
7068                 if ($vmiddst && $vmiddst != $vmid) {
7069                     my $agent_running = $qga && qga_check_running($vmid);
7070                     if ($agent_running) {
7071                         print "freeze filesystem\n";
7072                         eval { PVE::QemuServer::vm_mon_cmd($vmid, "guest-fsfreeze-freeze"); };
7073                     } else {
7074                         print "suspend vm\n";
7075                         eval { PVE::QemuServer::vm_suspend($vmid, 1); };
7076                     }
7077
7078                     # if we clone a disk for a new target vm, we don't switch the disk
7079                     PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs);
7080
7081                     if ($agent_running) {
7082                         print "unfreeze filesystem\n";
7083                         eval { PVE::QemuServer::vm_mon_cmd($vmid, "guest-fsfreeze-thaw"); };
7084                     } else {
7085                         print "resume vm\n";
7086                         eval {  PVE::QemuServer::vm_resume($vmid, 1, 1); };
7087                     }
7088
7089                     last;
7090                 } else {
7091
7092                     foreach my $job (keys %$jobs) {
7093                         # try to switch the disk if source and destination are on the same guest
7094                         print "$job: Completing block job...\n";
7095
7096                         eval { vm_mon_cmd($vmid, "block-job-complete", device => $job) };
7097                         if ($@ =~ m/cannot be completed/) {
7098                             print "$job: Block job cannot be completed, try again.\n";
7099                             $err_complete++;
7100                         }else {
7101                             print "$job: Completed successfully.\n";
7102                             $jobs->{$job}->{complete} = 1;
7103                         }
7104                     }
7105                 }
7106             }
7107             sleep 1;
7108         }
7109     };
7110     my $err = $@;
7111
7112     if ($err) {
7113         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
7114         die "mirroring error: $err";
7115     }
7116
7117 }
7118
7119 sub qemu_blockjobs_cancel {
7120     my ($vmid, $jobs) = @_;
7121
7122     foreach my $job (keys %$jobs) {
7123         print "$job: Cancelling block job\n";
7124         eval { vm_mon_cmd($vmid, "block-job-cancel", device => $job); };
7125         $jobs->{$job}->{cancel} = 1;
7126     }
7127
7128     while (1) {
7129         my $stats = vm_mon_cmd($vmid, "query-block-jobs");
7130
7131         my $running_jobs = {};
7132         foreach my $stat (@$stats) {
7133             $running_jobs->{$stat->{device}} = $stat;
7134         }
7135
7136         foreach my $job (keys %$jobs) {
7137
7138             if (defined($jobs->{$job}->{cancel}) && !defined($running_jobs->{$job})) {
7139                 print "$job: Done.\n";
7140                 delete $jobs->{$job};
7141             }
7142         }
7143
7144         last if scalar(keys %$jobs) == 0;
7145
7146         sleep 1;
7147     }
7148 }
7149
7150 sub clone_disk {
7151     my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
7152         $newvmid, $storage, $format, $full, $newvollist, $jobs, $skipcomplete, $qga, $bwlimit) = @_;
7153
7154     my $newvolid;
7155
7156     if (!$full) {
7157         print "create linked clone of drive $drivename ($drive->{file})\n";
7158         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
7159         push @$newvollist, $newvolid;
7160     } else {
7161
7162         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
7163         $storeid = $storage if $storage;
7164
7165         my $dst_format = resolve_dst_disk_format($storecfg, $storeid, $volname, $format);
7166         my ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 3);
7167
7168         print "create full clone of drive $drivename ($drive->{file})\n";
7169         my $name = undef;
7170         $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $newvmid, $dst_format, $name, ($size/1024));
7171         push @$newvollist, $newvolid;
7172
7173         PVE::Storage::activate_volumes($storecfg, [$newvolid]);
7174
7175         my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
7176         if (!$running || $snapname) {
7177             # TODO: handle bwlimits
7178             qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit);
7179         } else {
7180
7181             my $kvmver = get_running_qemu_version ($vmid);
7182             if (!qemu_machine_feature_enabled (undef, $kvmver, 2, 7)) {
7183                 die "drive-mirror with iothread requires qemu version 2.7 or higher\n"
7184                     if $drive->{iothread};
7185             }
7186
7187             qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid, $sparseinit, $jobs, $skipcomplete, $qga, $bwlimit);
7188         }
7189     }
7190
7191     my ($size) = PVE::Storage::volume_size_info($storecfg, $newvolid, 3);
7192
7193     my $disk = $drive;
7194     $disk->{format} = undef;
7195     $disk->{file} = $newvolid;
7196     $disk->{size} = $size;
7197
7198     return $disk;
7199 }
7200
7201 # this only works if VM is running
7202 sub get_current_qemu_machine {
7203     my ($vmid) = @_;
7204
7205     my $cmd = { execute => 'query-machines', arguments => {} };
7206     my $res = vm_qmp_command($vmid, $cmd);
7207
7208     my ($current, $default);
7209     foreach my $e (@$res) {
7210         $default = $e->{name} if $e->{'is-default'};
7211         $current = $e->{name} if $e->{'is-current'};
7212     }
7213
7214     # fallback to the default machine if current is not supported by qemu
7215     return $current || $default || 'pc';
7216 }
7217
7218 sub get_running_qemu_version {
7219     my ($vmid) = @_;
7220     my $cmd = { execute => 'query-version', arguments => {} };
7221     my $res = vm_qmp_command($vmid, $cmd);
7222     return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
7223 }
7224
7225 sub qemu_machine_feature_enabled {
7226     my ($machine, $kvmver, $version_major, $version_minor) = @_;
7227
7228     my $current_major;
7229     my $current_minor;
7230
7231     if ($machine && $machine =~ m/^((?:pc(-i440fx|-q35)?|virt)-(\d+)\.(\d+))/) {
7232
7233         $current_major = $3;
7234         $current_minor = $4;
7235
7236     } elsif ($kvmver =~ m/^(\d+)\.(\d+)/) {
7237
7238         $current_major = $1;
7239         $current_minor = $2;
7240     }
7241
7242     return 1 if $current_major > $version_major ||
7243                 ($current_major == $version_major &&
7244                  $current_minor >= $version_minor);
7245 }
7246
7247 sub qemu_machine_pxe {
7248     my ($vmid, $conf) = @_;
7249
7250     my $machine =  PVE::QemuServer::get_current_qemu_machine($vmid);
7251
7252     if ($conf->{machine} && $conf->{machine} =~ m/\.pxe$/) {
7253         $machine .= '.pxe';
7254     }
7255
7256     return $machine;
7257 }
7258
7259 sub qemu_use_old_bios_files {
7260     my ($machine_type) = @_;
7261
7262     return if !$machine_type;
7263
7264     my $use_old_bios_files = undef;
7265
7266     if ($machine_type =~ m/^(\S+)\.pxe$/) {
7267         $machine_type = $1;
7268         $use_old_bios_files = 1;
7269     } else {
7270         my $kvmver = kvm_user_version();
7271         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
7272         # load new efi bios files on migration. So this hack is required to allow
7273         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
7274         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
7275         $use_old_bios_files = !qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 4);
7276     }
7277
7278     return ($use_old_bios_files, $machine_type);
7279 }
7280
7281 sub create_efidisk($$$$$) {
7282     my ($storecfg, $storeid, $vmid, $fmt, $arch) = @_;
7283
7284     my (undef, $ovmf_vars) = get_ovmf_files($arch);
7285     die "EFI vars default image not found\n" if ! -f $ovmf_vars;
7286
7287     my $vars_size_b = -s $ovmf_vars;
7288     my $vars_size = PVE::Tools::convert_size($vars_size_b, 'b' => 'kb');
7289     my $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $fmt, undef, $vars_size);
7290     PVE::Storage::activate_volumes($storecfg, [$volid]);
7291
7292     qemu_img_convert($ovmf_vars, $volid, $vars_size_b, undef, 0);
7293
7294     return ($volid, $vars_size);
7295 }
7296
7297 sub vm_iothreads_list {
7298     my ($vmid) = @_;
7299
7300     my $res = vm_mon_cmd($vmid, 'query-iothreads');
7301
7302     my $iothreads = {};
7303     foreach my $iothread (@$res) {
7304         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
7305     }
7306
7307     return $iothreads;
7308 }
7309
7310 sub scsihw_infos {
7311     my ($conf, $drive) = @_;
7312
7313     my $maxdev = 0;
7314
7315     if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)) {
7316         $maxdev = 7;
7317     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
7318         $maxdev = 1;
7319     } else {
7320         $maxdev = 256;
7321     }
7322
7323     my $controller = int($drive->{index} / $maxdev);
7324     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single') ? "virtioscsi" : "scsihw";
7325
7326     return ($maxdev, $controller, $controller_prefix);
7327 }
7328
7329 sub add_hyperv_enlightenments {
7330     my ($cpuFlags, $winversion, $machine_type, $kvmver, $bios, $gpu_passthrough, $hv_vendor_id) = @_;
7331
7332     return if $winversion < 6;
7333     return if $bios && $bios eq 'ovmf' && $winversion < 8;
7334
7335     if ($gpu_passthrough || defined($hv_vendor_id)) {
7336         $hv_vendor_id //= 'proxmox';
7337         push @$cpuFlags , "hv_vendor_id=$hv_vendor_id";
7338     }
7339
7340     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
7341         push @$cpuFlags , 'hv_spinlocks=0x1fff';
7342         push @$cpuFlags , 'hv_vapic';
7343         push @$cpuFlags , 'hv_time';
7344     } else {
7345         push @$cpuFlags , 'hv_spinlocks=0xffff';
7346     }
7347
7348     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 6)) {
7349         push @$cpuFlags , 'hv_reset';
7350         push @$cpuFlags , 'hv_vpindex';
7351         push @$cpuFlags , 'hv_runtime';
7352     }
7353
7354     if ($winversion >= 7) {
7355         push @$cpuFlags , 'hv_relaxed';
7356
7357         if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 12)) {
7358             push @$cpuFlags , 'hv_synic';
7359             push @$cpuFlags , 'hv_stimer';
7360         }
7361
7362         if (qemu_machine_feature_enabled ($machine_type, $kvmver, 3, 1)) {
7363             push @$cpuFlags , 'hv_ipi';
7364         }
7365     }
7366 }
7367
7368 sub windows_version {
7369     my ($ostype) = @_;
7370
7371     return 0 if !$ostype;
7372
7373     my $winversion = 0;
7374
7375     if($ostype eq 'wxp' || $ostype eq 'w2k3' || $ostype eq 'w2k') {
7376         $winversion = 5;
7377     } elsif($ostype eq 'w2k8' || $ostype eq 'wvista') {
7378         $winversion = 6;
7379     } elsif ($ostype =~ m/^win(\d+)$/) {
7380         $winversion = $1;
7381     }
7382
7383     return $winversion;
7384 }
7385
7386 sub resolve_dst_disk_format {
7387         my ($storecfg, $storeid, $src_volname, $format) = @_;
7388         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
7389
7390         if (!$format) {
7391             # if no target format is specified, use the source disk format as hint
7392             if ($src_volname) {
7393                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
7394                 $format = qemu_img_format($scfg, $src_volname);
7395             } else {
7396                 return $defFormat;
7397             }
7398         }
7399
7400         # test if requested format is supported - else use default
7401         my $supported = grep { $_ eq $format } @$validFormats;
7402         $format = $defFormat if !$supported;
7403         return $format;
7404 }
7405
7406 sub resolve_first_disk {
7407     my $conf = shift;
7408     my @disks = PVE::QemuServer::valid_drive_names();
7409     my $firstdisk;
7410     foreach my $ds (reverse @disks) {
7411         next if !$conf->{$ds};
7412         my $disk = PVE::QemuServer::parse_drive($ds, $conf->{$ds});
7413         next if PVE::QemuServer::drive_is_cdrom($disk);
7414         $firstdisk = $ds;
7415     }
7416     return $firstdisk;
7417 }
7418
7419 sub generate_uuid {
7420     my ($uuid, $uuid_str);
7421     UUID::generate($uuid);
7422     UUID::unparse($uuid, $uuid_str);
7423     return $uuid_str;
7424 }
7425
7426 sub generate_smbios1_uuid {
7427     return "uuid=".generate_uuid();
7428 }
7429
7430 sub nbd_stop {
7431     my ($vmid) = @_;
7432
7433     vm_mon_cmd($vmid, 'nbd-server-stop');
7434 }
7435
7436 sub create_reboot_request {
7437     my ($vmid) = @_;
7438     open(my $fh, '>', "/run/qemu-server/$vmid.reboot")
7439         or die "failed to create reboot trigger file: $!\n";
7440     close($fh);
7441 }
7442
7443 sub clear_reboot_request {
7444     my ($vmid) = @_;
7445     my $path = "/run/qemu-server/$vmid.reboot";
7446     my $res = 0;
7447
7448     $res = unlink($path);
7449     die "could not remove reboot request for $vmid: $!"
7450         if !$res && $! != POSIX::ENOENT;
7451
7452     return $res;
7453 }
7454
7455 # bash completion helper
7456
7457 sub complete_backup_archives {
7458     my ($cmdname, $pname, $cvalue) = @_;
7459
7460     my $cfg = PVE::Storage::config();
7461
7462     my $storeid;
7463
7464     if ($cvalue =~ m/^([^:]+):/) {
7465         $storeid = $1;
7466     }
7467
7468     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
7469
7470     my $res = [];
7471     foreach my $id (keys %$data) {
7472         foreach my $item (@{$data->{$id}}) {
7473             next if $item->{format} !~ m/^vma\.(gz|lzo)$/;
7474             push @$res, $item->{volid} if defined($item->{volid});
7475         }
7476     }
7477
7478     return $res;
7479 }
7480
7481 my $complete_vmid_full = sub {
7482     my ($running) = @_;
7483
7484     my $idlist = vmstatus();
7485
7486     my $res = [];
7487
7488     foreach my $id (keys %$idlist) {
7489         my $d = $idlist->{$id};
7490         if (defined($running)) {
7491             next if $d->{template};
7492             next if $running && $d->{status} ne 'running';
7493             next if !$running && $d->{status} eq 'running';
7494         }
7495         push @$res, $id;
7496
7497     }
7498     return $res;
7499 };
7500
7501 sub complete_vmid {
7502     return &$complete_vmid_full();
7503 }
7504
7505 sub complete_vmid_stopped {
7506     return &$complete_vmid_full(0);
7507 }
7508
7509 sub complete_vmid_running {
7510     return &$complete_vmid_full(1);
7511 }
7512
7513 sub complete_storage {
7514
7515     my $cfg = PVE::Storage::config();
7516     my $ids = $cfg->{ids};
7517
7518     my $res = [];
7519     foreach my $sid (keys %$ids) {
7520         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
7521         next if !$ids->{$sid}->{content}->{images};
7522         push @$res, $sid;
7523     }
7524
7525     return $res;
7526 }
7527
7528 1;