PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5
   6 use POSIX;
   7 use IO::Handle;
   8 use IO::Select;
   9 use IO::File;
  10 use IO::Dir;
  11 use IO::Socket::UNIX;
  12 use File::Basename;
  13 use File::Path;
  14 use File::stat;
  15 use Getopt::Long;
  16 use Digest::SHA;
  17 use Fcntl ':flock';
  18 use Cwd 'abs_path';
  19 use IPC::Open3;
  20 use JSON;
  21 use Fcntl;
  22 use PVE::SafeSyslog;
  23 use Storable qw(dclone);
  24 use MIME::Base64;
  25 use PVE::Exception qw(raise raise_param_exc);
  26 use PVE::Storage;
  27 use PVE::Tools qw(run_command lock_file lock_file_full file_read_firstline dir_glob_foreach $IPV6RE);
  28 use PVE::JSONSchema qw(get_standard_option);
  29 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file cfs_lock_file);
  30 use PVE::INotify;
  31 use PVE::ProcFSTools;
  32 use PVE::QemuConfig;
  33 use PVE::QMPClient;
  34 use PVE::RPCEnvironment;
  35 use PVE::GuestHelpers;
  36 use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr print_pcie_root_port);
  37 use PVE::QemuServer::Memory;
  38 use PVE::QemuServer::USB qw(parse_usb_device);
  39 use PVE::QemuServer::Cloudinit;
  40 use PVE::SysFSTools;
  41 use PVE::Systemd;
  42 use Time::HiRes qw(gettimeofday);
  43 use File::Copy qw(copy);
  44 use URI::Escape;
  45
  46 my $EDK2_FW_BASE = '/usr/share/pve-edk2-firmware/';
  47 my $OVMF = {
  48     x86_64 => [
  49         "$EDK2_FW_BASE/OVMF_CODE.fd",
  50         "$EDK2_FW_BASE/OVMF_VARS.fd"
  51     ],
  52     aarch64 => [
  53         "$EDK2_FW_BASE/AAVMF_CODE.fd",
  54         "$EDK2_FW_BASE/AAVMF_VARS.fd"
  55     ],
  56 };
  57
  58 my $qemu_snap_storage = { rbd => 1 };
  59
  60 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  61
  62 my $QEMU_FORMAT_RE = qr/raw|cow|qcow|qcow2|qed|vmdk|cloop/;
  63
  64 # Note about locking: we use flock on the config file protect
  65 # against concurent actions.
  66 # Aditionaly, we have a 'lock' setting in the config file. This
  67 # can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
  68 # allowed when such lock is set. But you can ignore this kind of
  69 # lock with the --skiplock flag.
  70
  71 cfs_register_file('/qemu-server/',
  72                   \&parse_vm_config,
  73                   \&write_vm_config);
  74
  75 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
  76     description => "Some command save/restore state from this location.",
  77     type => 'string',
  78     maxLength => 128,
  79     optional => 1,
  80 });
  81
  82 PVE::JSONSchema::register_standard_option('pve-qm-image-format', {
  83     type => 'string',
  84     enum => [qw(raw cow qcow qed qcow2 vmdk cloop)],
  85     description => "The drive's backing file's data format.",
  86     optional => 1,
  87 });
  88
  89 PVE::JSONSchema::register_standard_option('pve-qemu-machine', {
  90         description => "Specifies the Qemu machine type.",
  91         type => 'string',
  92         pattern => '(pc|pc(-i440fx)?-\d+\.\d+(\.pxe)?|q35|pc-q35-\d+\.\d+(\.pxe)?|virt(?:-\d+\.\d+)?)',
  93         maxLength => 40,
  94         optional => 1,
  95 });
  96
  97 #no warnings 'redefine';
  98
  99 sub cgroups_write {
 100    my ($controller, $vmid, $option, $value) = @_;
 101
 102    my $path = "/sys/fs/cgroup/$controller/qemu.slice/$vmid.scope/$option";
 103    PVE::ProcFSTools::write_proc_entry($path, $value);
 104
 105 }
 106
 107 my $nodename = PVE::INotify::nodename();
 108
 109 mkdir "/etc/pve/nodes/$nodename";
 110 my $confdir = "/etc/pve/nodes/$nodename/qemu-server";
 111 mkdir $confdir;
 112
 113 my $var_run_tmpdir = "/var/run/qemu-server";
 114 mkdir $var_run_tmpdir;
 115
 116 my $lock_dir = "/var/lock/qemu-server";
 117 mkdir $lock_dir;
 118
 119 my $cpu_vendor_list = {
 120     # Intel CPUs
 121     486 => 'GenuineIntel',
 122     pentium => 'GenuineIntel',
 123     pentium2  => 'GenuineIntel',
 124     pentium3  => 'GenuineIntel',
 125     coreduo => 'GenuineIntel',
 126     core2duo => 'GenuineIntel',
 127     Conroe  => 'GenuineIntel',
 128     Penryn  => 'GenuineIntel',
 129     Nehalem  => 'GenuineIntel',
 130     'Nehalem-IBRS'  => 'GenuineIntel',
 131     Westmere => 'GenuineIntel',
 132     'Westmere-IBRS' => 'GenuineIntel',
 133     SandyBridge => 'GenuineIntel',
 134     'SandyBridge-IBRS' => 'GenuineIntel',
 135     IvyBridge => 'GenuineIntel',
 136     'IvyBridge-IBRS' => 'GenuineIntel',
 137     Haswell => 'GenuineIntel',
 138     'Haswell-IBRS' => 'GenuineIntel',
 139     'Haswell-noTSX' => 'GenuineIntel',
 140     'Haswell-noTSX-IBRS' => 'GenuineIntel',
 141     Broadwell => 'GenuineIntel',
 142     'Broadwell-IBRS' => 'GenuineIntel',
 143     'Broadwell-noTSX' => 'GenuineIntel',
 144     'Broadwell-noTSX-IBRS' => 'GenuineIntel',
 145     'Skylake-Client' => 'GenuineIntel',
 146     'Skylake-Client-IBRS' => 'GenuineIntel',
 147     'Skylake-Server' => 'GenuineIntel',
 148     'Skylake-Server-IBRS' => 'GenuineIntel',
 149     'Cascadelake-Server' => 'GenuineIntel',
 150     KnightsMill => 'GenuineIntel',
 151
 152
 153     # AMD CPUs
 154     athlon => 'AuthenticAMD',
 155     phenom  => 'AuthenticAMD',
 156     Opteron_G1  => 'AuthenticAMD',
 157     Opteron_G2  => 'AuthenticAMD',
 158     Opteron_G3  => 'AuthenticAMD',
 159     Opteron_G4  => 'AuthenticAMD',
 160     Opteron_G5  => 'AuthenticAMD',
 161     EPYC => 'AuthenticAMD',
 162     'EPYC-IBPB' => 'AuthenticAMD',
 163
 164     # generic types, use vendor from host node
 165     host => 'default',
 166     kvm32 => 'default',
 167     kvm64 => 'default',
 168     qemu32 => 'default',
 169     qemu64 => 'default',
 170     max => 'default',
 171 };
 172
 173 my @supported_cpu_flags = (
 174     'pcid',
 175     'spec-ctrl',
 176     'ibpb',
 177     'ssbd',
 178     'virt-ssbd',
 179     'amd-ssbd',
 180     'amd-no-ssb',
 181     'pdpe1gb',
 182     'md-clear',
 183     'hv-tlbflush',
 184     'hv-evmcs',
 185     'aes'
 186 );
 187 my $cpu_flag = qr/[+-](@{[join('|', @supported_cpu_flags)]})/;
 188
 189 my $cpu_fmt = {
 190     cputype => {
 191         description => "Emulated CPU type.",
 192         type => 'string',
 193         enum => [ sort { "\L$a" cmp "\L$b" } keys %$cpu_vendor_list ],
 194         default => 'kvm64',
 195         default_key => 1,
 196     },
 197     hidden => {
 198         description => "Do not identify as a KVM virtual machine.",
 199         type => 'boolean',
 200         optional => 1,
 201         default => 0
 202     },
 203     'hv-vendor-id' => {
 204         type => 'string',
 205         pattern => qr/[a-zA-Z0-9]{1,12}/,
 206         format_description => 'vendor-id',
 207         description => 'The Hyper-V vendor ID. Some drivers or programs inside Windows guests need a specific ID.',
 208         optional => 1,
 209     },
 210     flags => {
 211         description => "List of additional CPU flags separated by ';'."
 212                      . " Use '+FLAG' to enable, '-FLAG' to disable a flag."
 213                      . " Currently supported flags: @{[join(', ', @supported_cpu_flags)]}.",
 214         format_description => '+FLAG[;-FLAG...]',
 215         type => 'string',
 216         pattern => qr/$cpu_flag(;$cpu_flag)*/,
 217         optional => 1,
 218     },
 219 };
 220
 221 my $watchdog_fmt = {
 222     model => {
 223         default_key => 1,
 224         type => 'string',
 225         enum => [qw(i6300esb ib700)],
 226         description => "Watchdog type to emulate.",
 227         default => 'i6300esb',
 228         optional => 1,
 229     },
 230     action => {
 231         type => 'string',
 232         enum => [qw(reset shutdown poweroff pause debug none)],
 233         description => "The action to perform if after activation the guest fails to poll the watchdog in time.",
 234         optional => 1,
 235     },
 236 };
 237 PVE::JSONSchema::register_format('pve-qm-watchdog', $watchdog_fmt);
 238
 239 my $agent_fmt = {
 240     enabled => {
 241         description => "Enable/disable Qemu GuestAgent.",
 242         type => 'boolean',
 243         default => 0,
 244         default_key => 1,
 245     },
 246     fstrim_cloned_disks => {
 247         description => "Run fstrim after cloning/moving a disk.",
 248         type => 'boolean',
 249         optional => 1,
 250         default => 0
 251     },
 252 };
 253
 254 my $vga_fmt = {
 255     type => {
 256         description => "Select the VGA type.",
 257         type => 'string',
 258         default => 'std',
 259         optional => 1,
 260         default_key => 1,
 261         enum => [qw(cirrus qxl qxl2 qxl3 qxl4 none serial0 serial1 serial2 serial3 std virtio vmware)],
 262     },
 263     memory => {
 264         description => "Sets the VGA memory (in MiB). Has no effect with serial display.",
 265         type => 'integer',
 266         optional => 1,
 267         minimum => 4,
 268         maximum => 512,
 269     },
 270 };
 271
 272 my $ivshmem_fmt = {
 273     size => {
 274         type => 'integer',
 275         minimum => 1,
 276         description => "The size of the file in MB.",
 277     },
 278     name => {
 279         type => 'string',
 280         pattern => '[a-zA-Z0-9\-]+',
 281         optional => 1,
 282         format_description => 'string',
 283         description => "The name of the file. Will be prefixed with 'pve-shm-'. Default is the VMID. Will be deleted when the VM is stopped.",
 284     },
 285 };
 286
 287 my $audio_fmt = {
 288     device => {
 289         type => 'string',
 290         enum => [qw(ich9-intel-hda intel-hda AC97)],
 291         description =>  "Configure an audio device."
 292     },
 293     driver =>  {
 294         type => 'string',
 295         enum => ['spice'],
 296         default => 'spice',
 297         optional => 1,
 298         description => "Driver backend for the audio device."
 299     },
 300 };
 301
 302 my $spice_enhancements_fmt = {
 303     foldersharing => {
 304         type => 'boolean',
 305         optional => 1,
 306         default => '0',
 307         description =>  "Enable folder sharing via SPICE. Needs Spice-WebDAV daemon installed in the VM."
 308     },
 309     videostreaming =>  {
 310         type => 'string',
 311         enum => ['off', 'all', 'filter'],
 312         default => 'off',
 313         optional => 1,
 314         description => "Enable video streaming. Uses compression for detected video streams."
 315     },
 316 };
 317
 318 my $confdesc = {
 319     onboot => {
 320         optional => 1,
 321         type => 'boolean',
 322         description => "Specifies whether a VM will be started during system bootup.",
 323         default => 0,
 324     },
 325     autostart => {
 326         optional => 1,
 327         type => 'boolean',
 328         description => "Automatic restart after crash (currently ignored).",
 329         default => 0,
 330     },
 331     hotplug => {
 332         optional => 1,
 333         type => 'string', format => 'pve-hotplug-features',
 334         description => "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
 335         default => 'network,disk,usb',
 336     },
 337     reboot => {
 338         optional => 1,
 339         type => 'boolean',
 340         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 341         default => 1,
 342     },
 343     lock => {
 344         optional => 1,
 345         type => 'string',
 346         description => "Lock/unlock the VM.",
 347         enum => [qw(backup clone create migrate rollback snapshot snapshot-delete suspending suspended)],
 348     },
 349     cpulimit => {
 350         optional => 1,
 351         type => 'number',
 352         description => "Limit of CPU usage.",
 353         verbose_description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
 354         minimum => 0,
 355         maximum => 128,
 356         default => 0,
 357     },
 358     cpuunits => {
 359         optional => 1,
 360         type => 'integer',
 361         description => "CPU weight for a VM.",
 362         verbose_description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.",
 363         minimum => 2,
 364         maximum => 262144,
 365         default => 1024,
 366     },
 367     memory => {
 368         optional => 1,
 369         type => 'integer',
 370         description => "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
 371         minimum => 16,
 372         default => 512,
 373     },
 374     balloon => {
 375         optional => 1,
 376         type => 'integer',
 377         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 378         minimum => 0,
 379     },
 380     shares => {
 381         optional => 1,
 382         type => 'integer',
 383         description => "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning. Auto-ballooning is done by pvestatd.",
 384         minimum => 0,
 385         maximum => 50000,
 386         default => 1000,
 387     },
 388     keyboard => {
 389         optional => 1,
 390         type => 'string',
 391         description => "Keybord layout for vnc server. Default is read from the '/etc/pve/datacenter.cfg' configuration file.".
 392                        "It should not be necessary to set it.",
 393         enum => PVE::Tools::kvmkeymaplist(),
 394         default => undef,
 395     },
 396     name => {
 397         optional => 1,
 398         type => 'string', format => 'dns-name',
 399         description => "Set a name for the VM. Only used on the configuration web interface.",
 400     },
 401     scsihw => {
 402         optional => 1,
 403         type => 'string',
 404         description => "SCSI controller model",
 405         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 406         default => 'lsi',
 407     },
 408     description => {
 409         optional => 1,
 410         type => 'string',
 411         description => "Description for the VM. Only used on the configuration web interface. This is saved as comment inside the configuration file.",
 412     },
 413     ostype => {
 414         optional => 1,
 415         type => 'string',
 416         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 l24 l26 solaris)],
 417         description => "Specify guest operating system.",
 418         verbose_description => <<EODESC,
 419 Specify guest operating system. This is used to enable special
 420 optimization/features for specific operating systems:
 421
 422 [horizontal]
 423 other;; unspecified OS
 424 wxp;; Microsoft Windows XP
 425 w2k;; Microsoft Windows 2000
 426 w2k3;; Microsoft Windows 2003
 427 w2k8;; Microsoft Windows 2008
 428 wvista;; Microsoft Windows Vista
 429 win7;; Microsoft Windows 7
 430 win8;; Microsoft Windows 8/2012/2012r2
 431 win10;; Microsoft Windows 10/2016
 432 l24;; Linux 2.4 Kernel
 433 l26;; Linux 2.6/3.X Kernel
 434 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
 435 EODESC
 436     },
 437     boot => {
 438         optional => 1,
 439         type => 'string',
 440         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n).",
 441         pattern => '[acdn]{1,4}',
 442         default => 'cdn',
 443     },
 444     bootdisk => {
 445         optional => 1,
 446         type => 'string', format => 'pve-qm-bootdisk',
 447         description => "Enable booting from specified disk.",
 448         pattern => '(ide|sata|scsi|virtio)\d+',
 449     },
 450     smp => {
 451         optional => 1,
 452         type => 'integer',
 453         description => "The number of CPUs. Please use option -sockets instead.",
 454         minimum => 1,
 455         default => 1,
 456     },
 457     sockets => {
 458         optional => 1,
 459         type => 'integer',
 460         description => "The number of CPU sockets.",
 461         minimum => 1,
 462         default => 1,
 463     },
 464     cores => {
 465         optional => 1,
 466         type => 'integer',
 467         description => "The number of cores per socket.",
 468         minimum => 1,
 469         default => 1,
 470     },
 471     numa => {
 472         optional => 1,
 473         type => 'boolean',
 474         description => "Enable/disable NUMA.",
 475         default => 0,
 476     },
 477     hugepages => {
 478         optional => 1,
 479         type => 'string',
 480         description => "Enable/disable hugepages memory.",
 481         enum => [qw(any 2 1024)],
 482     },
 483     vcpus => {
 484         optional => 1,
 485         type => 'integer',
 486         description => "Number of hotplugged vcpus.",
 487         minimum => 1,
 488         default => 0,
 489     },
 490     acpi => {
 491         optional => 1,
 492         type => 'boolean',
 493         description => "Enable/disable ACPI.",
 494         default => 1,
 495     },
 496     agent => {
 497         optional => 1,
 498         description => "Enable/disable Qemu GuestAgent and its properties.",
 499         type => 'string',
 500         format => $agent_fmt,
 501     },
 502     kvm => {
 503         optional => 1,
 504         type => 'boolean',
 505         description => "Enable/disable KVM hardware virtualization.",
 506         default => 1,
 507     },
 508     tdf => {
 509         optional => 1,
 510         type => 'boolean',
 511         description => "Enable/disable time drift fix.",
 512         default => 0,
 513     },
 514     localtime => {
 515         optional => 1,
 516         type => 'boolean',
 517         description => "Set the real time clock to local time. This is enabled by default if ostype indicates a Microsoft OS.",
 518     },
 519     freeze => {
 520         optional => 1,
 521         type => 'boolean',
 522         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 523     },
 524     vga => {
 525         optional => 1,
 526         type => 'string', format => $vga_fmt,
 527         description => "Configure the VGA hardware.",
 528         verbose_description => "Configure the VGA Hardware. If you want to use ".
 529             "high resolution modes (>= 1280x1024x16) you may need to increase " .
 530             "the vga memory option. Since QEMU 2.9 the default VGA display type " .
 531             "is 'std' for all OS types besides some Windows versions (XP and " .
 532             "older) which use 'cirrus'. The 'qxl' option enables the SPICE " .
 533             "display server. For win* OS you can select how many independent " .
 534             "displays you want, Linux guests can add displays them self.\n".
 535             "You can also run without any graphic card, using a serial device as terminal.",
 536     },
 537     watchdog => {
 538         optional => 1,
 539         type => 'string', format => 'pve-qm-watchdog',
 540         description => "Create a virtual hardware watchdog device.",
 541         verbose_description => "Create a virtual hardware watchdog device. Once enabled" .
 542             " (by a guest action), the watchdog must be periodically polled " .
 543             "by an agent inside the guest or else the watchdog will reset " .
 544             "the guest (or execute the respective action specified)",
 545     },
 546     startdate => {
 547         optional => 1,
 548         type => 'string',
 549         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 550         description => "Set the initial date of the real time clock. Valid format for date are: 'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 551         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 552         default => 'now',
 553     },
 554     startup =>  get_standard_option('pve-startup-order'),
 555     template => {
 556         optional => 1,
 557         type => 'boolean',
 558         description => "Enable/disable Template.",
 559         default => 0,
 560     },
 561     args => {
 562         optional => 1,
 563         type => 'string',
 564         description => "Arbitrary arguments passed to kvm.",
 565         verbose_description => <<EODESCR,
 566 Arbitrary arguments passed to kvm, for example:
 567
 568 args: -no-reboot -no-hpet
 569
 570 NOTE: this option is for experts only.
 571 EODESCR
 572     },
 573     tablet => {
 574         optional => 1,
 575         type => 'boolean',
 576         default => 1,
 577         description => "Enable/disable the USB tablet device.",
 578         verbose_description => "Enable/disable the USB tablet device. This device is " .
 579             "usually needed to allow absolute mouse positioning with VNC. " .
 580             "Else the mouse runs out of sync with normal VNC clients. " .
 581             "If you're running lots of console-only guests on one host, " .
 582             "you may consider disabling this to save some context switches. " .
 583             "This is turned off by default if you use spice (-vga=qxl).",
 584     },
 585     migrate_speed => {
 586         optional => 1,
 587         type => 'integer',
 588         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 589         minimum => 0,
 590         default => 0,
 591     },
 592     migrate_downtime => {
 593         optional => 1,
 594         type => 'number',
 595         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 596         minimum => 0,
 597         default => 0.1,
 598     },
 599     cdrom => {
 600         optional => 1,
 601         type => 'string', format => 'pve-qm-ide',
 602         typetext => '<volume>',
 603         description => "This is an alias for option -ide2",
 604     },
 605     cpu => {
 606         optional => 1,
 607         description => "Emulated CPU type.",
 608         type => 'string',
 609         format => $cpu_fmt,
 610     },
 611     parent => get_standard_option('pve-snapshot-name', {
 612         optional => 1,
 613         description => "Parent snapshot name. This is used internally, and should not be modified.",
 614     }),
 615     snaptime => {
 616         optional => 1,
 617         description => "Timestamp for snapshots.",
 618         type => 'integer',
 619         minimum => 0,
 620     },
 621     vmstate => {
 622         optional => 1,
 623         type => 'string', format => 'pve-volume-id',
 624         description => "Reference to a volume which stores the VM state. This is used internally for snapshots.",
 625     },
 626     vmstatestorage => get_standard_option('pve-storage-id', {
 627         description => "Default storage for VM state volumes/files.",
 628         optional => 1,
 629     }),
 630     runningmachine => get_standard_option('pve-qemu-machine', {
 631         description => "Specifies the Qemu machine type of the running vm. This is used internally for snapshots.",
 632     }),
 633     machine => get_standard_option('pve-qemu-machine'),
 634     arch => {
 635         description => "Virtual processor architecture. Defaults to the host.",
 636         optional => 1,
 637         type => 'string',
 638         enum => [qw(x86_64 aarch64)],
 639     },
 640     smbios1 => {
 641         description => "Specify SMBIOS type 1 fields.",
 642         type => 'string', format => 'pve-qm-smbios1',
 643         maxLength => 512,
 644         optional => 1,
 645     },
 646     protection => {
 647         optional => 1,
 648         type => 'boolean',
 649         description => "Sets the protection flag of the VM. This will disable the remove VM and remove disk operations.",
 650         default => 0,
 651     },
 652     bios => {
 653         optional => 1,
 654         type => 'string',
 655         enum => [ qw(seabios ovmf) ],
 656         description => "Select BIOS implementation.",
 657         default => 'seabios',
 658     },
 659     vmgenid => {
 660         type => 'string',
 661         pattern => '(?:[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}|[01])',
 662         format_description => 'UUID',
 663         description => "Set VM Generation ID. Use '1' to autogenerate on create or update, pass '0' to disable explicitly.",
 664         verbose_description => "The VM generation ID (vmgenid) device exposes a".
 665             " 128-bit integer value identifier to the guest OS. This allows to".
 666             " notify the guest operating system when the virtual machine is".
 667             " executed with a different configuration (e.g. snapshot execution".
 668             " or creation from a template). The guest operating system notices".
 669             " the change, and is then able to react as appropriate by marking".
 670             " its copies of distributed databases as dirty, re-initializing its".
 671             " random number generator, etc.\n".
 672             "Note that auto-creation only works when done throug API/CLI create".
 673             " or update methods, but not when manually editing the config file.",
 674         default => "1 (autogenerated)",
 675         optional => 1,
 676     },
 677     hookscript => {
 678         type => 'string',
 679         format => 'pve-volume-id',
 680         optional => 1,
 681         description => "Script that will be executed during various steps in the vms lifetime.",
 682     },
 683     ivshmem => {
 684         type => 'string',
 685         format => $ivshmem_fmt,
 686         description => "Inter-VM shared memory. Useful for direct communication between VMs, or to the host.",
 687         optional => 1,
 688     },
 689     audio0 => {
 690         type => 'string',
 691         format => $audio_fmt,
 692         description => "Configure a audio device, useful in combination with QXL/Spice.",
 693         optional => 1
 694     },
 695     spice_enhancements => {
 696         type => 'string',
 697         format => $spice_enhancements_fmt,
 698         description => "Configure additional enhancements for SPICE.",
 699         optional => 1
 700     },
 701 };
 702
 703 my $cicustom_fmt = {
 704     meta => {
 705         type => 'string',
 706         optional => 1,
 707         description => 'Specify a custom file containing all meta data passed to the VM via cloud-init. This is provider specific meaning configdrive2 and nocloud differ.',
 708         format => 'pve-volume-id',
 709         format_description => 'volume',
 710     },
 711     network => {
 712         type => 'string',
 713         optional => 1,
 714         description => 'Specify a custom file containing all network data passed to the VM via cloud-init.',
 715         format => 'pve-volume-id',
 716         format_description => 'volume',
 717     },
 718     user => {
 719         type => 'string',
 720         optional => 1,
 721         description => 'Specify a custom file containing all user data passed to the VM via cloud-init.',
 722         format => 'pve-volume-id',
 723         format_description => 'volume',
 724     },
 725 };
 726 PVE::JSONSchema::register_format('pve-qm-cicustom', $cicustom_fmt);
 727
 728 my $confdesc_cloudinit = {
 729     citype => {
 730         optional => 1,
 731         type => 'string',
 732         description => 'Specifies the cloud-init configuration format. The default depends on the configured operating system type (`ostype`. We use the `nocloud` format for Linux, and `configdrive2` for windows.',
 733         enum => ['configdrive2', 'nocloud'],
 734     },
 735     ciuser => {
 736         optional => 1,
 737         type => 'string',
 738         description => "cloud-init: User name to change ssh keys and password for instead of the image's configured default user.",
 739     },
 740     cipassword => {
 741         optional => 1,
 742         type => 'string',
 743         description => 'cloud-init: Password to assign the user. Using this is generally not recommended. Use ssh keys instead. Also note that older cloud-init versions do not support hashed passwords.',
 744     },
 745     cicustom => {
 746         optional => 1,
 747         type => 'string',
 748         description => 'cloud-init: Specify custom files to replace the automatically generated ones at start.',
 749         format => 'pve-qm-cicustom',
 750     },
 751     searchdomain => {
 752         optional => 1,
 753         type => 'string',
 754         description => "cloud-init: Sets DNS search domains for a container. Create will automatically use the setting from the host if neither searchdomain nor nameserver are set.",
 755     },
 756     nameserver => {
 757         optional => 1,
 758         type => 'string', format => 'address-list',
 759         description => "cloud-init: Sets DNS server IP address for a container. Create will automatically use the setting from the host if neither searchdomain nor nameserver are set.",
 760     },
 761     sshkeys => {
 762         optional => 1,
 763         type => 'string',
 764         format => 'urlencoded',
 765         description => "cloud-init: Setup public SSH keys (one key per line, OpenSSH format).",
 766     },
 767 };
 768
 769 # what about other qemu settings ?
 770 #cpu => 'string',
 771 #machine => 'string',
 772 #fda => 'file',
 773 #fdb => 'file',
 774 #mtdblock => 'file',
 775 #sd => 'file',
 776 #pflash => 'file',
 777 #snapshot => 'bool',
 778 #bootp => 'file',
 779 ##tftp => 'dir',
 780 ##smb => 'dir',
 781 #kernel => 'file',
 782 #append => 'string',
 783 #initrd => 'file',
 784 ##soundhw => 'string',
 785
 786 while (my ($k, $v) = each %$confdesc) {
 787     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 788 }
 789
 790 my $MAX_IDE_DISKS = 4;
 791 my $MAX_SCSI_DISKS = 14;
 792 my $MAX_VIRTIO_DISKS = 16;
 793 my $MAX_SATA_DISKS = 6;
 794 my $MAX_USB_DEVICES = 5;
 795 my $MAX_NETS = 32;
 796 my $MAX_UNUSED_DISKS = 256;
 797 my $MAX_HOSTPCI_DEVICES = 16;
 798 my $MAX_SERIAL_PORTS = 4;
 799 my $MAX_PARALLEL_PORTS = 3;
 800 my $MAX_NUMA = 8;
 801
 802 my $numa_fmt = {
 803     cpus => {
 804         type => "string",
 805         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 806         description => "CPUs accessing this NUMA node.",
 807         format_description => "id[-id];...",
 808     },
 809     memory => {
 810         type => "number",
 811         description => "Amount of memory this NUMA node provides.",
 812         optional => 1,
 813     },
 814     hostnodes => {
 815         type => "string",
 816         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 817         description => "Host NUMA nodes to use.",
 818         format_description => "id[-id];...",
 819         optional => 1,
 820     },
 821     policy => {
 822         type => 'string',
 823         enum => [qw(preferred bind interleave)],
 824         description => "NUMA allocation policy.",
 825         optional => 1,
 826     },
 827 };
 828 PVE::JSONSchema::register_format('pve-qm-numanode', $numa_fmt);
 829 my $numadesc = {
 830     optional => 1,
 831     type => 'string', format => $numa_fmt,
 832     description => "NUMA topology.",
 833 };
 834 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 835
 836 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 837     $confdesc->{"numa$i"} = $numadesc;
 838 }
 839
 840 my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000',  'pcnet',  'virtio',
 841                       'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3',
 842                       'e1000-82540em', 'e1000-82544gc', 'e1000-82545em'];
 843 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 844
 845 my $net_fmt_bridge_descr = <<__EOD__;
 846 Bridge to attach the network device to. The Proxmox VE standard bridge
 847 is called 'vmbr0'.
 848
 849 If you do not specify a bridge, we create a kvm user (NATed) network
 850 device, which provides DHCP and DNS services. The following addresses
 851 are used:
 852
 853  10.0.2.2   Gateway
 854  10.0.2.3   DNS Server
 855  10.0.2.4   SMB Server
 856
 857 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 858 __EOD__
 859
 860 my $net_fmt = {
 861     macaddr  => get_standard_option('mac-addr', {
 862         description => "MAC address. That address must be unique withing your network. This is automatically generated if not specified.",
 863     }),
 864     model => {
 865         type => 'string',
 866         description => "Network Card Model. The 'virtio' model provides the best performance with very low CPU overhead. If your guest does not support this driver, it is usually best to use 'e1000'.",
 867         enum => $nic_model_list,
 868         default_key => 1,
 869     },
 870     (map { $_ => { keyAlias => 'model', alias => 'macaddr' }} @$nic_model_list),
 871     bridge => {
 872         type => 'string',
 873         description => $net_fmt_bridge_descr,
 874         format_description => 'bridge',
 875         optional => 1,
 876     },
 877     queues => {
 878         type => 'integer',
 879         minimum => 0, maximum => 16,
 880         description => 'Number of packet queues to be used on the device.',
 881         optional => 1,
 882     },
 883     rate => {
 884         type => 'number',
 885         minimum => 0,
 886         description => "Rate limit in mbps (megabytes per second) as floating point number.",
 887         optional => 1,
 888     },
 889     tag => {
 890         type => 'integer',
 891         minimum => 1, maximum => 4094,
 892         description => 'VLAN tag to apply to packets on this interface.',
 893         optional => 1,
 894     },
 895     trunks => {
 896         type => 'string',
 897         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 898         description => 'VLAN trunks to pass through this interface.',
 899         format_description => 'vlanid[;vlanid...]',
 900         optional => 1,
 901     },
 902     firewall => {
 903         type => 'boolean',
 904         description => 'Whether this interface should be protected by the firewall.',
 905         optional => 1,
 906     },
 907     link_down => {
 908         type => 'boolean',
 909         description => 'Whether this interface should be disconnected (like pulling the plug).',
 910         optional => 1,
 911     },
 912 };
 913
 914 my $netdesc = {
 915     optional => 1,
 916     type => 'string', format => $net_fmt,
 917     description => "Specify network devices.",
 918 };
 919
 920 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 921
 922 my $ipconfig_fmt = {
 923     ip => {
 924         type => 'string',
 925         format => 'pve-ipv4-config',
 926         format_description => 'IPv4Format/CIDR',
 927         description => 'IPv4 address in CIDR format.',
 928         optional => 1,
 929         default => 'dhcp',
 930     },
 931     gw => {
 932         type => 'string',
 933         format => 'ipv4',
 934         format_description => 'GatewayIPv4',
 935         description => 'Default gateway for IPv4 traffic.',
 936         optional => 1,
 937         requires => 'ip',
 938     },
 939     ip6 => {
 940         type => 'string',
 941         format => 'pve-ipv6-config',
 942         format_description => 'IPv6Format/CIDR',
 943         description => 'IPv6 address in CIDR format.',
 944         optional => 1,
 945         default => 'dhcp',
 946     },
 947     gw6 => {
 948         type => 'string',
 949         format => 'ipv6',
 950         format_description => 'GatewayIPv6',
 951         description => 'Default gateway for IPv6 traffic.',
 952         optional => 1,
 953         requires => 'ip6',
 954     },
 955 };
 956 PVE::JSONSchema::register_format('pve-qm-ipconfig', $ipconfig_fmt);
 957 my $ipconfigdesc = {
 958     optional => 1,
 959     type => 'string', format => 'pve-qm-ipconfig',
 960     description => <<'EODESCR',
 961 cloud-init: Specify IP addresses and gateways for the corresponding interface.
 962
 963 IP addresses use CIDR notation, gateways are optional but need an IP of the same type specified.
 964
 965 The special string 'dhcp' can be used for IP addresses to use DHCP, in which case no explicit gateway should be provided.
 966 For IPv6 the special string 'auto' can be used to use stateless autoconfiguration.
 967
 968 If cloud-init is enabled and neither an IPv4 nor an IPv6 address is specified, it defaults to using dhcp on IPv4.
 969 EODESCR
 970 };
 971 PVE::JSONSchema::register_standard_option("pve-qm-ipconfig", $netdesc);
 972
 973 for (my $i = 0; $i < $MAX_NETS; $i++)  {
 974     $confdesc->{"net$i"} = $netdesc;
 975     $confdesc_cloudinit->{"ipconfig$i"} = $ipconfigdesc;
 976 }
 977
 978 foreach my $key (keys %$confdesc_cloudinit) {
 979     $confdesc->{$key} = $confdesc_cloudinit->{$key};
 980 }
 981
 982 PVE::JSONSchema::register_format('pve-volume-id-or-qm-path', \&verify_volume_id_or_qm_path);
 983 sub verify_volume_id_or_qm_path {
 984     my ($volid, $noerr) = @_;
 985
 986     if ($volid eq 'none' || $volid eq 'cdrom' || $volid =~ m|^/|) {
 987         return $volid;
 988     }
 989
 990     # if its neither 'none' nor 'cdrom' nor a path, check if its a volume-id
 991     $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') };
 992     if ($@) {
 993         return undef if $noerr;
 994         die $@;
 995     }
 996     return $volid;
 997 }
 998
 999 my $drivename_hash;
1000
1001 my %drivedesc_base = (
1002     volume => { alias => 'file' },
1003     file => {
1004         type => 'string',
1005         format => 'pve-volume-id-or-qm-path',
1006         default_key => 1,
1007         format_description => 'volume',
1008         description => "The drive's backing volume.",
1009     },
1010     media => {
1011         type => 'string',
1012         enum => [qw(cdrom disk)],
1013         description => "The drive's media type.",
1014         default => 'disk',
1015         optional => 1
1016     },
1017     cyls => {
1018         type => 'integer',
1019         description => "Force the drive's physical geometry to have a specific cylinder count.",
1020         optional => 1
1021     },
1022     heads => {
1023         type => 'integer',
1024         description => "Force the drive's physical geometry to have a specific head count.",
1025         optional => 1
1026     },
1027     secs => {
1028         type => 'integer',
1029         description => "Force the drive's physical geometry to have a specific sector count.",
1030         optional => 1
1031     },
1032     trans => {
1033         type => 'string',
1034         enum => [qw(none lba auto)],
1035         description => "Force disk geometry bios translation mode.",
1036         optional => 1,
1037     },
1038     snapshot => {
1039         type => 'boolean',
1040         description => "Controls qemu's snapshot mode feature."
1041             . " If activated, changes made to the disk are temporary and will"
1042             . " be discarded when the VM is shutdown.",
1043         optional => 1,
1044     },
1045     cache => {
1046         type => 'string',
1047         enum => [qw(none writethrough writeback unsafe directsync)],
1048         description => "The drive's cache mode",
1049         optional => 1,
1050     },
1051     format => get_standard_option('pve-qm-image-format'),
1052     size => {
1053         type => 'string',
1054         format => 'disk-size',
1055         format_description => 'DiskSize',
1056         description => "Disk size. This is purely informational and has no effect.",
1057         optional => 1,
1058     },
1059     backup => {
1060         type => 'boolean',
1061         description => "Whether the drive should be included when making backups.",
1062         optional => 1,
1063     },
1064     replicate => {
1065         type => 'boolean',
1066         description => 'Whether the drive should considered for replication jobs.',
1067         optional => 1,
1068         default => 1,
1069     },
1070     rerror => {
1071         type => 'string',
1072         enum => [qw(ignore report stop)],
1073         description => 'Read error action.',
1074         optional => 1,
1075     },
1076     werror => {
1077         type => 'string',
1078         enum => [qw(enospc ignore report stop)],
1079         description => 'Write error action.',
1080         optional => 1,
1081     },
1082     aio => {
1083         type => 'string',
1084         enum => [qw(native threads)],
1085         description => 'AIO type to use.',
1086         optional => 1,
1087     },
1088     discard => {
1089         type => 'string',
1090         enum => [qw(ignore on)],
1091         description => 'Controls whether to pass discard/trim requests to the underlying storage.',
1092         optional => 1,
1093     },
1094     detect_zeroes => {
1095         type => 'boolean',
1096         description => 'Controls whether to detect and try to optimize writes of zeroes.',
1097         optional => 1,
1098     },
1099     serial => {
1100         type => 'string',
1101         format => 'urlencoded',
1102         format_description => 'serial',
1103         maxLength => 20*3, # *3 since it's %xx url enoded
1104         description => "The drive's reported serial number, url-encoded, up to 20 bytes long.",
1105         optional => 1,
1106     },
1107     shared => {
1108         type => 'boolean',
1109         description => 'Mark this locally-managed volume as available on all nodes',
1110         verbose_description => "Mark this locally-managed volume as available on all nodes.\n\nWARNING: This option does not share the volume automatically, it assumes it is shared already!",
1111         optional => 1,
1112         default => 0,
1113     }
1114 );
1115
1116 my %iothread_fmt = ( iothread => {
1117         type => 'boolean',
1118         description => "Whether to use iothreads for this drive",
1119         optional => 1,
1120 });
1121
1122 my %model_fmt = (
1123     model => {
1124         type => 'string',
1125         format => 'urlencoded',
1126         format_description => 'model',
1127         maxLength => 40*3, # *3 since it's %xx url enoded
1128         description => "The drive's reported model name, url-encoded, up to 40 bytes long.",
1129         optional => 1,
1130     },
1131 );
1132
1133 my %queues_fmt = (
1134     queues => {
1135         type => 'integer',
1136         description => "Number of queues.",
1137         minimum => 2,
1138         optional => 1
1139     }
1140 );
1141
1142 my %scsiblock_fmt = (
1143     scsiblock => {
1144         type => 'boolean',
1145         description => "whether to use scsi-block for full passthrough of host block device\n\nWARNING: can lead to I/O errors in combination with low memory or high memory fragmentation on host",
1146         optional => 1,
1147         default => 0,
1148     },
1149 );
1150
1151 my %ssd_fmt = (
1152     ssd => {
1153         type => 'boolean',
1154         description => "Whether to expose this drive as an SSD, rather than a rotational hard disk.",
1155         optional => 1,
1156     },
1157 );
1158
1159 my %wwn_fmt = (
1160     wwn => {
1161         type => 'string',
1162         pattern => qr/^(0x)[0-9a-fA-F]{16}/,
1163         format_description => 'wwn',
1164         description => "The drive's worldwide name, encoded as 16 bytes hex string, prefixed by '0x'.",
1165         optional => 1,
1166     },
1167 );
1168
1169 my $add_throttle_desc = sub {
1170     my ($key, $type, $what, $unit, $longunit, $minimum) = @_;
1171     my $d = {
1172         type => $type,
1173         format_description => $unit,
1174         description => "Maximum $what in $longunit.",
1175         optional => 1,
1176     };
1177     $d->{minimum} = $minimum if defined($minimum);
1178     $drivedesc_base{$key} = $d;
1179 };
1180 # throughput: (leaky bucket)
1181 $add_throttle_desc->('bps',     'integer', 'r/w speed',   'bps',  'bytes per second');
1182 $add_throttle_desc->('bps_rd',  'integer', 'read speed',  'bps',  'bytes per second');
1183 $add_throttle_desc->('bps_wr',  'integer', 'write speed', 'bps',  'bytes per second');
1184 $add_throttle_desc->('mbps',    'number',  'r/w speed',   'mbps', 'megabytes per second');
1185 $add_throttle_desc->('mbps_rd', 'number',  'read speed',  'mbps', 'megabytes per second');
1186 $add_throttle_desc->('mbps_wr', 'number',  'write speed', 'mbps', 'megabytes per second');
1187 $add_throttle_desc->('iops',    'integer', 'r/w I/O',     'iops', 'operations per second');
1188 $add_throttle_desc->('iops_rd', 'integer', 'read I/O',    'iops', 'operations per second');
1189 $add_throttle_desc->('iops_wr', 'integer', 'write I/O',   'iops', 'operations per second');
1190
1191 # pools: (pool of IO before throttling starts taking effect)
1192 $add_throttle_desc->('mbps_max',    'number',  'unthrottled r/w pool',       'mbps', 'megabytes per second');
1193 $add_throttle_desc->('mbps_rd_max', 'number',  'unthrottled read pool',      'mbps', 'megabytes per second');
1194 $add_throttle_desc->('mbps_wr_max', 'number',  'unthrottled write pool',     'mbps', 'megabytes per second');
1195 $add_throttle_desc->('iops_max',    'integer', 'unthrottled r/w I/O pool',   'iops', 'operations per second');
1196 $add_throttle_desc->('iops_rd_max', 'integer', 'unthrottled read I/O pool',  'iops', 'operations per second');
1197 $add_throttle_desc->('iops_wr_max', 'integer', 'unthrottled write I/O pool', 'iops', 'operations per second');
1198
1199 # burst lengths
1200 $add_throttle_desc->('bps_max_length',     'integer', 'length of I/O bursts',       'seconds', 'seconds', 1);
1201 $add_throttle_desc->('bps_rd_max_length',  'integer', 'length of read I/O bursts',  'seconds', 'seconds', 1);
1202 $add_throttle_desc->('bps_wr_max_length',  'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
1203 $add_throttle_desc->('iops_max_length',    'integer', 'length of I/O bursts',       'seconds', 'seconds', 1);
1204 $add_throttle_desc->('iops_rd_max_length', 'integer', 'length of read I/O bursts',  'seconds', 'seconds', 1);
1205 $add_throttle_desc->('iops_wr_max_length', 'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
1206
1207 # legacy support
1208 $drivedesc_base{'bps_rd_length'} = { alias => 'bps_rd_max_length' };
1209 $drivedesc_base{'bps_wr_length'} = { alias => 'bps_wr_max_length' };
1210 $drivedesc_base{'iops_rd_length'} = { alias => 'iops_rd_max_length' };
1211 $drivedesc_base{'iops_wr_length'} = { alias => 'iops_wr_max_length' };
1212
1213 my $ide_fmt = {
1214     %drivedesc_base,
1215     %model_fmt,
1216     %ssd_fmt,
1217     %wwn_fmt,
1218 };
1219 PVE::JSONSchema::register_format("pve-qm-ide", $ide_fmt);
1220
1221 my $idedesc = {
1222     optional => 1,
1223     type => 'string', format => $ide_fmt,
1224     description => "Use volume as IDE hard disk or CD-ROM (n is 0 to " .($MAX_IDE_DISKS -1) . ").",
1225 };
1226 PVE::JSONSchema::register_standard_option("pve-qm-ide", $idedesc);
1227
1228 my $scsi_fmt = {
1229     %drivedesc_base,
1230     %iothread_fmt,
1231     %queues_fmt,
1232     %scsiblock_fmt,
1233     %ssd_fmt,
1234     %wwn_fmt,
1235 };
1236 my $scsidesc = {
1237     optional => 1,
1238     type => 'string', format => $scsi_fmt,
1239     description => "Use volume as SCSI hard disk or CD-ROM (n is 0 to " . ($MAX_SCSI_DISKS - 1) . ").",
1240 };
1241 PVE::JSONSchema::register_standard_option("pve-qm-scsi", $scsidesc);
1242
1243 my $sata_fmt = {
1244     %drivedesc_base,
1245     %ssd_fmt,
1246     %wwn_fmt,
1247 };
1248 my $satadesc = {
1249     optional => 1,
1250     type => 'string', format => $sata_fmt,
1251     description => "Use volume as SATA hard disk or CD-ROM (n is 0 to " . ($MAX_SATA_DISKS - 1). ").",
1252 };
1253 PVE::JSONSchema::register_standard_option("pve-qm-sata", $satadesc);
1254
1255 my $virtio_fmt = {
1256     %drivedesc_base,
1257     %iothread_fmt,
1258 };
1259 my $virtiodesc = {
1260     optional => 1,
1261     type => 'string', format => $virtio_fmt,
1262     description => "Use volume as VIRTIO hard disk (n is 0 to " . ($MAX_VIRTIO_DISKS - 1) . ").",
1263 };
1264 PVE::JSONSchema::register_standard_option("pve-qm-virtio", $virtiodesc);
1265
1266 my $alldrive_fmt = {
1267     %drivedesc_base,
1268     %iothread_fmt,
1269     %model_fmt,
1270     %queues_fmt,
1271     %scsiblock_fmt,
1272     %ssd_fmt,
1273     %wwn_fmt,
1274 };
1275
1276 my $efidisk_fmt = {
1277     volume => { alias => 'file' },
1278     file => {
1279         type => 'string',
1280         format => 'pve-volume-id-or-qm-path',
1281         default_key => 1,
1282         format_description => 'volume',
1283         description => "The drive's backing volume.",
1284     },
1285     format => get_standard_option('pve-qm-image-format'),
1286     size => {
1287         type => 'string',
1288         format => 'disk-size',
1289         format_description => 'DiskSize',
1290         description => "Disk size. This is purely informational and has no effect.",
1291         optional => 1,
1292     },
1293 };
1294
1295 my $efidisk_desc = {
1296     optional => 1,
1297     type => 'string', format => $efidisk_fmt,
1298     description => "Configure a Disk for storing EFI vars",
1299 };
1300
1301 PVE::JSONSchema::register_standard_option("pve-qm-efidisk", $efidisk_desc);
1302
1303 my $usb_fmt = {
1304     host => {
1305         default_key => 1,
1306         type => 'string', format => 'pve-qm-usb-device',
1307         format_description => 'HOSTUSBDEVICE|spice',
1308         description => <<EODESCR,
1309 The Host USB device or port or the value 'spice'. HOSTUSBDEVICE syntax is:
1310
1311  'bus-port(.port)*' (decimal numbers) or
1312  'vendor_id:product_id' (hexadeciaml numbers) or
1313  'spice'
1314
1315 You can use the 'lsusb -t' command to list existing usb devices.
1316
1317 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1318
1319 The value 'spice' can be used to add a usb redirection devices for spice.
1320 EODESCR
1321     },
1322     usb3 => {
1323         optional => 1,
1324         type => 'boolean',
1325         description => "Specifies whether if given host option is a USB3 device or port.",
1326         default => 0,
1327     },
1328 };
1329
1330 my $usbdesc = {
1331     optional => 1,
1332     type => 'string', format => $usb_fmt,
1333     description => "Configure an USB device (n is 0 to 4).",
1334 };
1335 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
1336
1337 my $PCIRE = qr/[a-f0-9]{2}:[a-f0-9]{2}(?:\.[a-f0-9])?/;
1338 my $hostpci_fmt = {
1339     host => {
1340         default_key => 1,
1341         type => 'string',
1342         pattern => qr/$PCIRE(;$PCIRE)*/,
1343         format_description => 'HOSTPCIID[;HOSTPCIID2...]',
1344         description => <<EODESCR,
1345 Host PCI device pass through. The PCI ID of a host's PCI device or a list
1346 of PCI virtual functions of the host. HOSTPCIID syntax is:
1347
1348 'bus:dev.func' (hexadecimal numbers)
1349
1350 You can us the 'lspci' command to list existing PCI devices.
1351 EODESCR
1352     },
1353     rombar => {
1354         type => 'boolean',
1355         description =>  "Specify whether or not the device's ROM will be visible in the guest's memory map.",
1356         optional => 1,
1357         default => 1,
1358     },
1359     romfile => {
1360         type => 'string',
1361         pattern => '[^,;]+',
1362         format_description => 'string',
1363         description => "Custom pci device rom filename (must be located in /usr/share/kvm/).",
1364         optional => 1,
1365     },
1366     pcie => {
1367         type => 'boolean',
1368         description =>  "Choose the PCI-express bus (needs the 'q35' machine model).",
1369         optional => 1,
1370         default => 0,
1371     },
1372     'x-vga' => {
1373         type => 'boolean',
1374         description =>  "Enable vfio-vga device support.",
1375         optional => 1,
1376         default => 0,
1377     },
1378     'mdev' => {
1379         type => 'string',
1380         format_description => 'string',
1381         pattern => '[^/\.:]+',
1382         optional => 1,
1383         description => <<EODESCR
1384 The type of mediated device to use.
1385 An instance of this type will be created on startup of the VM and
1386 will be cleaned up when the VM stops.
1387 EODESCR
1388     }
1389 };
1390 PVE::JSONSchema::register_format('pve-qm-hostpci', $hostpci_fmt);
1391
1392 my $hostpcidesc = {
1393         optional => 1,
1394         type => 'string', format => 'pve-qm-hostpci',
1395         description => "Map host PCI devices into guest.",
1396         verbose_description =>  <<EODESCR,
1397 Map host PCI devices into guest.
1398
1399 NOTE: This option allows direct access to host hardware. So it is no longer
1400 possible to migrate such machines - use with special care.
1401
1402 CAUTION: Experimental! User reported problems with this option.
1403 EODESCR
1404 };
1405 PVE::JSONSchema::register_standard_option("pve-qm-hostpci", $hostpcidesc);
1406
1407 my $serialdesc = {
1408         optional => 1,
1409         type => 'string',
1410         pattern => '(/dev/.+|socket)',
1411         description =>  "Create a serial device inside the VM (n is 0 to 3)",
1412         verbose_description =>  <<EODESCR,
1413 Create a serial device inside the VM (n is 0 to 3), and pass through a
1414 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1415 host side (use 'qm terminal' to open a terminal connection).
1416
1417 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines - use with special care.
1418
1419 CAUTION: Experimental! User reported problems with this option.
1420 EODESCR
1421 };
1422
1423 my $paralleldesc= {
1424         optional => 1,
1425         type => 'string',
1426         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
1427         description =>  "Map host parallel devices (n is 0 to 2).",
1428         verbose_description =>  <<EODESCR,
1429 Map host parallel devices (n is 0 to 2).
1430
1431 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1432
1433 CAUTION: Experimental! User reported problems with this option.
1434 EODESCR
1435 };
1436
1437 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
1438     $confdesc->{"parallel$i"} = $paralleldesc;
1439 }
1440
1441 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
1442     $confdesc->{"serial$i"} = $serialdesc;
1443 }
1444
1445 for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
1446     $confdesc->{"hostpci$i"} = $hostpcidesc;
1447 }
1448
1449 for (my $i = 0; $i < $MAX_IDE_DISKS; $i++)  {
1450     $drivename_hash->{"ide$i"} = 1;
1451     $confdesc->{"ide$i"} = $idedesc;
1452 }
1453
1454 for (my $i = 0; $i < $MAX_SATA_DISKS; $i++)  {
1455     $drivename_hash->{"sata$i"} = 1;
1456     $confdesc->{"sata$i"} = $satadesc;
1457 }
1458
1459 for (my $i = 0; $i < $MAX_SCSI_DISKS; $i++)  {
1460     $drivename_hash->{"scsi$i"} = 1;
1461     $confdesc->{"scsi$i"} = $scsidesc ;
1462 }
1463
1464 for (my $i = 0; $i < $MAX_VIRTIO_DISKS; $i++)  {
1465     $drivename_hash->{"virtio$i"} = 1;
1466     $confdesc->{"virtio$i"} = $virtiodesc;
1467 }
1468
1469 $drivename_hash->{efidisk0} = 1;
1470 $confdesc->{efidisk0} = $efidisk_desc;
1471
1472 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
1473     $confdesc->{"usb$i"} = $usbdesc;
1474 }
1475
1476 my $unuseddesc = {
1477     optional => 1,
1478     type => 'string', format => 'pve-volume-id',
1479     description => "Reference to unused volumes. This is used internally, and should not be modified manually.",
1480 };
1481
1482 for (my $i = 0; $i < $MAX_UNUSED_DISKS; $i++)  {
1483     $confdesc->{"unused$i"} = $unuseddesc;
1484 }
1485
1486 my $kvm_api_version = 0;
1487
1488 sub kvm_version {
1489     return $kvm_api_version if $kvm_api_version;
1490
1491     open my $fh, '<', '/dev/kvm'
1492         or return undef;
1493
1494     # 0xae00 => KVM_GET_API_VERSION
1495     $kvm_api_version = ioctl($fh, 0xae00, 0);
1496
1497     return $kvm_api_version;
1498 }
1499
1500 my $kvm_user_version = {};
1501 my $kvm_mtime = {};
1502
1503 sub kvm_user_version {
1504     my ($binary) = @_;
1505
1506     $binary //= get_command_for_arch(get_host_arch()); # get the native arch by default
1507     my $st = stat($binary);
1508
1509     my $cachedmtime = $kvm_mtime->{$binary} // -1;
1510     return $kvm_user_version->{$binary} if $kvm_user_version->{$binary} &&
1511         $cachedmtime == $st->mtime;
1512
1513     $kvm_user_version->{$binary} = 'unknown';
1514     $kvm_mtime->{$binary} = $st->mtime;
1515
1516     my $code = sub {
1517         my $line = shift;
1518         if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1519             $kvm_user_version->{$binary} = $2;
1520         }
1521     };
1522
1523     eval { run_command([$binary, '--version'], outfunc => $code); };
1524     warn $@ if $@;
1525
1526     return $kvm_user_version->{$binary};
1527
1528 }
1529
1530 sub kernel_has_vhost_net {
1531     return -c '/dev/vhost-net';
1532 }
1533
1534 sub valid_drive_names {
1535     # order is important - used to autoselect boot disk
1536     return ((map { "ide$_" } (0 .. ($MAX_IDE_DISKS - 1))),
1537             (map { "scsi$_" } (0 .. ($MAX_SCSI_DISKS - 1))),
1538             (map { "virtio$_" } (0 .. ($MAX_VIRTIO_DISKS - 1))),
1539             (map { "sata$_" } (0 .. ($MAX_SATA_DISKS - 1))),
1540             'efidisk0');
1541 }
1542
1543 sub is_valid_drivename {
1544     my $dev = shift;
1545
1546     return defined($drivename_hash->{$dev});
1547 }
1548
1549 sub option_exists {
1550     my $key = shift;
1551     return defined($confdesc->{$key});
1552 }
1553
1554 sub nic_models {
1555     return $nic_model_list;
1556 }
1557
1558 sub os_list_description {
1559
1560     return {
1561         other => 'Other',
1562         wxp => 'Windows XP',
1563         w2k => 'Windows 2000',
1564         w2k3 =>, 'Windows 2003',
1565         w2k8 => 'Windows 2008',
1566         wvista => 'Windows Vista',
1567         win7 => 'Windows 7',
1568         win8 => 'Windows 8/2012',
1569         win10 => 'Windows 10/2016',
1570         l24 => 'Linux 2.4',
1571         l26 => 'Linux 2.6',
1572     };
1573 }
1574
1575 my $cdrom_path;
1576
1577 sub get_cdrom_path {
1578
1579     return  $cdrom_path if $cdrom_path;
1580
1581     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
1582     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
1583     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
1584 }
1585
1586 sub get_iso_path {
1587     my ($storecfg, $vmid, $cdrom) = @_;
1588
1589     if ($cdrom eq 'cdrom') {
1590         return get_cdrom_path();
1591     } elsif ($cdrom eq 'none') {
1592         return '';
1593     } elsif ($cdrom =~ m|^/|) {
1594         return $cdrom;
1595     } else {
1596         return PVE::Storage::path($storecfg, $cdrom);
1597     }
1598 }
1599
1600 # try to convert old style file names to volume IDs
1601 sub filename_to_volume_id {
1602     my ($vmid, $file, $media) = @_;
1603
1604      if (!($file eq 'none' || $file eq 'cdrom' ||
1605           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1606
1607         return undef if $file =~ m|/|;
1608
1609         if ($media && $media eq 'cdrom') {
1610             $file = "local:iso/$file";
1611         } else {
1612             $file = "local:$vmid/$file";
1613         }
1614     }
1615
1616     return $file;
1617 }
1618
1619 sub verify_media_type {
1620     my ($opt, $vtype, $media) = @_;
1621
1622     return if !$media;
1623
1624     my $etype;
1625     if ($media eq 'disk') {
1626         $etype = 'images';
1627     } elsif ($media eq 'cdrom') {
1628         $etype = 'iso';
1629     } else {
1630         die "internal error";
1631     }
1632
1633     return if ($vtype eq $etype);
1634
1635     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1636 }
1637
1638 sub cleanup_drive_path {
1639     my ($opt, $storecfg, $drive) = @_;
1640
1641     # try to convert filesystem paths to volume IDs
1642
1643     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1644         ($drive->{file} !~ m|^/dev/.+|) &&
1645         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1646         ($drive->{file} !~ m/^\d+$/)) {
1647         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1648         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"}) if !$vtype;
1649         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1650         verify_media_type($opt, $vtype, $drive->{media});
1651         $drive->{file} = $volid;
1652     }
1653
1654     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1655 }
1656
1657 sub parse_hotplug_features {
1658     my ($data) = @_;
1659
1660     my $res = {};
1661
1662     return $res if $data eq '0';
1663
1664     $data = $confdesc->{hotplug}->{default} if $data eq '1';
1665
1666     foreach my $feature (PVE::Tools::split_list($data)) {
1667         if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
1668             $res->{$1} = 1;
1669         } else {
1670             die "invalid hotplug feature '$feature'\n";
1671         }
1672     }
1673     return $res;
1674 }
1675
1676 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1677 sub pve_verify_hotplug_features {
1678     my ($value, $noerr) = @_;
1679
1680     return $value if parse_hotplug_features($value);
1681
1682     return undef if $noerr;
1683
1684     die "unable to parse hotplug option\n";
1685 }
1686
1687 # ideX = [volume=]volume-id[,media=d][,cyls=c,heads=h,secs=s[,trans=t]]
1688 #        [,snapshot=on|off][,cache=on|off][,format=f][,backup=yes|no]
1689 #        [,rerror=ignore|report|stop][,werror=enospc|ignore|report|stop]
1690 #        [,aio=native|threads][,discard=ignore|on][,detect_zeroes=on|off]
1691 #        [,iothread=on][,serial=serial][,model=model]
1692
1693 sub parse_drive {
1694     my ($key, $data) = @_;
1695
1696     my ($interface, $index);
1697
1698     if ($key =~ m/^([^\d]+)(\d+)$/) {
1699         $interface = $1;
1700         $index = $2;
1701     } else {
1702         return undef;
1703     }
1704
1705     my $desc = $key =~ /^unused\d+$/ ? $alldrive_fmt
1706                                      : $confdesc->{$key}->{format};
1707     if (!$desc) {
1708         warn "invalid drive key: $key\n";
1709         return undef;
1710     }
1711     my $res = eval { PVE::JSONSchema::parse_property_string($desc, $data) };
1712     return undef if !$res;
1713     $res->{interface} = $interface;
1714     $res->{index} = $index;
1715
1716     my $error = 0;
1717     foreach my $opt (qw(bps bps_rd bps_wr)) {
1718         if (my $bps = defined(delete $res->{$opt})) {
1719             if (defined($res->{"m$opt"})) {
1720                 warn "both $opt and m$opt specified\n";
1721                 ++$error;
1722                 next;
1723             }
1724             $res->{"m$opt"} = sprintf("%.3f", $bps / (1024*1024.0));
1725         }
1726     }
1727
1728     # can't use the schema's 'requires' because of the mbps* => bps* "transforming aliases"
1729     for my $requirement (
1730         [mbps_max => 'mbps'],
1731         [mbps_rd_max => 'mbps_rd'],
1732         [mbps_wr_max => 'mbps_wr'],
1733         [miops_max => 'miops'],
1734         [miops_rd_max => 'miops_rd'],
1735         [miops_wr_max => 'miops_wr'],
1736         [bps_max_length => 'mbps_max'],
1737         [bps_rd_max_length => 'mbps_rd_max'],
1738         [bps_wr_max_length => 'mbps_wr_max'],
1739         [iops_max_length => 'iops_max'],
1740         [iops_rd_max_length => 'iops_rd_max'],
1741         [iops_wr_max_length => 'iops_wr_max']) {
1742         my ($option, $requires) = @$requirement;
1743         if ($res->{$option} && !$res->{$requires}) {
1744             warn "$option requires $requires\n";
1745             ++$error;
1746         }
1747     }
1748
1749     return undef if $error;
1750
1751     return undef if $res->{mbps_rd} && $res->{mbps};
1752     return undef if $res->{mbps_wr} && $res->{mbps};
1753     return undef if $res->{iops_rd} && $res->{iops};
1754     return undef if $res->{iops_wr} && $res->{iops};
1755
1756     if ($res->{media} && ($res->{media} eq 'cdrom')) {
1757         return undef if $res->{snapshot} || $res->{trans} || $res->{format};
1758         return undef if $res->{heads} || $res->{secs} || $res->{cyls};
1759         return undef if $res->{interface} eq 'virtio';
1760     }
1761
1762     if (my $size = $res->{size}) {
1763         return undef if !defined($res->{size} = PVE::JSONSchema::parse_size($size));
1764     }
1765
1766     return $res;
1767 }
1768
1769 sub print_drive {
1770     my ($vmid, $drive) = @_;
1771     my $data = { %$drive };
1772     delete $data->{$_} for qw(index interface);
1773     return PVE::JSONSchema::print_property_string($data, $alldrive_fmt);
1774 }
1775
1776 sub scsi_inquiry {
1777     my($fh, $noerr) = @_;
1778
1779     my $SG_IO = 0x2285;
1780     my $SG_GET_VERSION_NUM = 0x2282;
1781
1782     my $versionbuf = "\x00" x 8;
1783     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1784     if (!$ret) {
1785         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1786         return undef;
1787     }
1788     my $version = unpack("I", $versionbuf);
1789     if ($version < 30000) {
1790         die "scsi generic interface too old\n"  if !$noerr;
1791         return undef;
1792     }
1793
1794     my $buf = "\x00" x 36;
1795     my $sensebuf = "\x00" x 8;
1796     my $cmd = pack("C x3 C x1", 0x12, 36);
1797
1798     # see /usr/include/scsi/sg.h
1799     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1800
1801     my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1802                       length($sensebuf), 0, length($buf), $buf,
1803                       $cmd, $sensebuf, 6000);
1804
1805     $ret = ioctl($fh, $SG_IO, $packet);
1806     if (!$ret) {
1807         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1808         return undef;
1809     }
1810
1811     my @res = unpack($sg_io_hdr_t, $packet);
1812     if ($res[17] || $res[18]) {
1813         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1814         return undef;
1815     }
1816
1817     my $res = {};
1818     (my $byte0, my $byte1, $res->{vendor},
1819      $res->{product}, $res->{revision}) = unpack("C C x6 A8 A16 A4", $buf);
1820
1821     $res->{removable} = $byte1 & 128 ? 1 : 0;
1822     $res->{type} = $byte0 & 31;
1823
1824     return $res;
1825 }
1826
1827 sub path_is_scsi {
1828     my ($path) = @_;
1829
1830     my $fh = IO::File->new("+<$path") || return undef;
1831     my $res = scsi_inquiry($fh, 1);
1832     close($fh);
1833
1834     return $res;
1835 }
1836
1837 sub machine_type_is_q35 {
1838     my ($conf) = @_;
1839
1840     return $conf->{machine} && ($conf->{machine} =~ m/q35/) ? 1 : 0;
1841 }
1842
1843 sub print_tabletdevice_full {
1844     my ($conf, $arch) = @_;
1845
1846     my $q35 = machine_type_is_q35($conf);
1847
1848     # we use uhci for old VMs because tablet driver was buggy in older qemu
1849     my $usbbus;
1850     if (machine_type_is_q35($conf) || $arch eq 'aarch64') {
1851         $usbbus = 'ehci';
1852     } else {
1853         $usbbus = 'uhci';
1854     }
1855
1856     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1857 }
1858
1859 sub print_keyboarddevice_full {
1860     my ($conf, $arch, $machine) = @_;
1861
1862     return undef if $arch ne 'aarch64';
1863
1864     return "usb-kbd,id=keyboard,bus=ehci.0,port=2";
1865 }
1866
1867 sub print_drivedevice_full {
1868     my ($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type) = @_;
1869
1870     my $device = '';
1871     my $maxdev = 0;
1872
1873     if ($drive->{interface} eq 'virtio') {
1874         my $pciaddr = print_pci_addr("$drive->{interface}$drive->{index}", $bridges, $arch, $machine_type);
1875         $device = "virtio-blk-pci,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}$pciaddr";
1876         $device .= ",iothread=iothread-$drive->{interface}$drive->{index}" if $drive->{iothread};
1877     } elsif ($drive->{interface} eq 'scsi') {
1878
1879         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1880         my $unit = $drive->{index} % $maxdev;
1881         my $devicetype = 'hd';
1882         my $path = '';
1883         if (drive_is_cdrom($drive)) {
1884             $devicetype = 'cd';
1885         } else {
1886             if ($drive->{file} =~ m|^/|) {
1887                 $path = $drive->{file};
1888                 if (my $info = path_is_scsi($path)) {
1889                     if ($info->{type} == 0 && $drive->{scsiblock}) {
1890                         $devicetype = 'block';
1891                     } elsif ($info->{type} == 1) { # tape
1892                         $devicetype = 'generic';
1893                     }
1894                 }
1895             } else {
1896                  $path = PVE::Storage::path($storecfg, $drive->{file});
1897             }
1898
1899             if($path =~ m/^iscsi\:\/\//){
1900                 $devicetype = 'generic';
1901             }
1902         }
1903
1904         if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)){
1905             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1906         } else {
1907             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0,lun=$drive->{index},drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1908         }
1909
1910         if ($drive->{ssd} && ($devicetype eq 'block' || $devicetype eq 'hd')) {
1911             $device .= ",rotation_rate=1";
1912         }
1913         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1914
1915     } elsif ($drive->{interface} eq 'ide' || $drive->{interface} eq 'sata') {
1916         my $maxdev = ($drive->{interface} eq 'sata') ? $MAX_SATA_DISKS : 2;
1917         my $controller = int($drive->{index} / $maxdev);
1918         my $unit = $drive->{index} % $maxdev;
1919         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1920
1921         $device = "ide-$devicetype";
1922         if ($drive->{interface} eq 'ide') {
1923             $device .= ",bus=ide.$controller,unit=$unit";
1924         } else {
1925             $device .= ",bus=ahci$controller.$unit";
1926         }
1927         $device .= ",drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1928
1929         if ($devicetype eq 'hd') {
1930             if (my $model = $drive->{model}) {
1931                 $model = URI::Escape::uri_unescape($model);
1932                 $device .= ",model=$model";
1933             }
1934             if ($drive->{ssd}) {
1935                 $device .= ",rotation_rate=1";
1936             }
1937         }
1938         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1939     } elsif ($drive->{interface} eq 'usb') {
1940         die "implement me";
1941         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1942     } else {
1943         die "unsupported interface type";
1944     }
1945
1946     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1947
1948     if (my $serial = $drive->{serial}) {
1949         $serial = URI::Escape::uri_unescape($serial);
1950         $device .= ",serial=$serial";
1951     }
1952
1953
1954     return $device;
1955 }
1956
1957 sub get_initiator_name {
1958     my $initiator;
1959
1960     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return undef;
1961     while (defined(my $line = <$fh>)) {
1962         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1963         $initiator = $1;
1964         last;
1965     }
1966     $fh->close();
1967
1968     return $initiator;
1969 }
1970
1971 sub print_drive_full {
1972     my ($storecfg, $vmid, $drive) = @_;
1973
1974     my $path;
1975     my $volid = $drive->{file};
1976     my $format;
1977
1978     if (drive_is_cdrom($drive)) {
1979         $path = get_iso_path($storecfg, $vmid, $volid);
1980     } else {
1981         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1982         if ($storeid) {
1983             $path = PVE::Storage::path($storecfg, $volid);
1984             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
1985             $format = qemu_img_format($scfg, $volname);
1986         } else {
1987             $path = $volid;
1988             $format = "raw";
1989         }
1990    }
1991
1992     my $opts = '';
1993     my @qemu_drive_options = qw(heads secs cyls trans media format cache rerror werror aio discard);
1994     foreach my $o (@qemu_drive_options) {
1995         $opts .= ",$o=$drive->{$o}" if defined($drive->{$o});
1996     }
1997
1998     # snapshot only accepts on|off
1999     if (defined($drive->{snapshot})) {
2000         my $v = $drive->{snapshot} ? 'on' : 'off';
2001         $opts .= ",snapshot=$v";
2002     }
2003
2004     foreach my $type (['', '-total'], [_rd => '-read'], [_wr => '-write']) {
2005         my ($dir, $qmpname) = @$type;
2006         if (my $v = $drive->{"mbps$dir"}) {
2007             $opts .= ",throttling.bps$qmpname=".int($v*1024*1024);
2008         }
2009         if (my $v = $drive->{"mbps${dir}_max"}) {
2010             $opts .= ",throttling.bps$qmpname-max=".int($v*1024*1024);
2011         }
2012         if (my $v = $drive->{"bps${dir}_max_length"}) {
2013             $opts .= ",throttling.bps$qmpname-max-length=$v";
2014         }
2015         if (my $v = $drive->{"iops${dir}"}) {
2016             $opts .= ",throttling.iops$qmpname=$v";
2017         }
2018         if (my $v = $drive->{"iops${dir}_max"}) {
2019             $opts .= ",throttling.iops$qmpname-max=$v";
2020         }
2021         if (my $v = $drive->{"iops${dir}_max_length"}) {
2022             $opts .= ",throttling.iops$qmpname-max-length=$v";
2023         }
2024     }
2025
2026     $opts .= ",format=$format" if $format && !$drive->{format};
2027
2028     my $cache_direct = 0;
2029
2030     if (my $cache = $drive->{cache}) {
2031         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
2032     } elsif (!drive_is_cdrom($drive)) {
2033         $opts .= ",cache=none";
2034         $cache_direct = 1;
2035     }
2036
2037     # aio native works only with O_DIRECT
2038     if (!$drive->{aio}) {
2039         if($cache_direct) {
2040             $opts .= ",aio=native";
2041         } else {
2042             $opts .= ",aio=threads";
2043         }
2044     }
2045
2046     if (!drive_is_cdrom($drive)) {
2047         my $detectzeroes;
2048         if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
2049             $detectzeroes = 'off';
2050         } elsif ($drive->{discard}) {
2051             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
2052         } else {
2053             # This used to be our default with discard not being specified:
2054             $detectzeroes = 'on';
2055         }
2056         $opts .= ",detect-zeroes=$detectzeroes" if $detectzeroes;
2057     }
2058
2059     my $pathinfo = $path ? "file=$path," : '';
2060
2061     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
2062 }
2063
2064 sub print_netdevice_full {
2065     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files, $arch, $machine_type) = @_;
2066
2067     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
2068
2069     my $device = $net->{model};
2070     if ($net->{model} eq 'virtio') {
2071          $device = 'virtio-net-pci';
2072      };
2073
2074     my $pciaddr = print_pci_addr("$netid", $bridges, $arch, $machine_type);
2075     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
2076     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
2077         #Consider we have N queues, the number of vectors needed is 2*N + 2 (plus one config interrupt and control vq)
2078         my $vectors = $net->{queues} * 2 + 2;
2079         $tmpstr .= ",vectors=$vectors,mq=on";
2080     }
2081     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
2082
2083     if ($use_old_bios_files) {
2084         my $romfile;
2085         if ($device eq 'virtio-net-pci') {
2086             $romfile = 'pxe-virtio.rom';
2087         } elsif ($device eq 'e1000') {
2088             $romfile = 'pxe-e1000.rom';
2089         } elsif ($device eq 'ne2k') {
2090             $romfile = 'pxe-ne2k_pci.rom';
2091         } elsif ($device eq 'pcnet') {
2092             $romfile = 'pxe-pcnet.rom';
2093         } elsif ($device eq 'rtl8139') {
2094             $romfile = 'pxe-rtl8139.rom';
2095         }
2096         $tmpstr .= ",romfile=$romfile" if $romfile;
2097     }
2098
2099     return $tmpstr;
2100 }
2101
2102 sub print_netdev_full {
2103     my ($vmid, $conf, $arch, $net, $netid, $hotplug) = @_;
2104
2105     my $i = '';
2106     if ($netid =~ m/^net(\d+)$/) {
2107         $i = int($1);
2108     }
2109
2110     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
2111
2112     my $ifname = "tap${vmid}i$i";
2113
2114     # kvm uses TUNSETIFF ioctl, and that limits ifname length
2115     die "interface name '$ifname' is too long (max 15 character)\n"
2116         if length($ifname) >= 16;
2117
2118     my $vhostparam = '';
2119     if (is_native($arch)) {
2120         $vhostparam = ',vhost=on' if kernel_has_vhost_net() && $net->{model} eq 'virtio';
2121     }
2122
2123     my $vmname = $conf->{name} || "vm$vmid";
2124
2125     my $netdev = "";
2126     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
2127
2128     if ($net->{bridge}) {
2129         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script,downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
2130     } else {
2131         $netdev = "type=user,id=$netid,hostname=$vmname";
2132     }
2133
2134     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
2135
2136     return $netdev;
2137 }
2138
2139
2140 sub print_cpu_device {
2141     my ($conf, $id) = @_;
2142
2143     my $kvm = $conf->{kvm} // 1;
2144     my $cpu = $kvm ? "kvm64" : "qemu64";
2145     if (my $cputype = $conf->{cpu}) {
2146         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
2147             or die "Cannot parse cpu description: $cputype\n";
2148         $cpu = $cpuconf->{cputype};
2149     }
2150
2151     my $cores = $conf->{cores} || 1;
2152
2153     my $current_core = ($id - 1) % $cores;
2154     my $current_socket = int(($id - 1 - $current_core)/$cores);
2155
2156     return "$cpu-x86_64-cpu,id=cpu$id,socket-id=$current_socket,core-id=$current_core,thread-id=0";
2157 }
2158
2159 my $vga_map = {
2160     'cirrus' => 'cirrus-vga',
2161     'std' => 'VGA',
2162     'vmware' => 'vmware-svga',
2163     'virtio' => 'virtio-vga',
2164 };
2165
2166 sub print_vga_device {
2167     my ($conf, $vga, $arch, $machine, $id, $qxlnum, $bridges) = @_;
2168
2169     my $type = $vga_map->{$vga->{type}};
2170     if ($arch eq 'aarch64' && defined($type) && $type eq 'virtio-vga') {
2171         $type = 'virtio-gpu';
2172     }
2173     my $vgamem_mb = $vga->{memory};
2174     if ($qxlnum) {
2175         $type = $id ? 'qxl' : 'qxl-vga';
2176     }
2177     die "no devicetype for $vga->{type}\n" if !$type;
2178
2179     my $memory = "";
2180     if ($vgamem_mb) {
2181         if ($vga->{type} eq 'virtio') {
2182             my $bytes = PVE::Tools::convert_size($vgamem_mb, "mb" => "b");
2183             $memory = ",max_hostmem=$bytes";
2184         } elsif ($qxlnum) {
2185             # from https://www.spice-space.org/multiple-monitors.html
2186             $memory = ",vgamem_mb=$vga->{memory}";
2187             my $ram = $vgamem_mb * 4;
2188             my $vram = $vgamem_mb * 2;
2189             $memory .= ",ram_size_mb=$ram,vram_size_mb=$vram";
2190         } else {
2191             $memory = ",vgamem_mb=$vga->{memory}";
2192         }
2193     } elsif ($qxlnum && $id) {
2194         $memory = ",ram_size=67108864,vram_size=33554432";
2195     }
2196
2197     my $q35 = machine_type_is_q35($conf);
2198     my $vgaid = "vga" . ($id // '');
2199     my $pciaddr;
2200
2201     if ($q35 && $vgaid eq 'vga') {
2202         # the first display uses pcie.0 bus on q35 machines
2203         $pciaddr = print_pcie_addr($vgaid, $bridges, $arch, $machine);
2204     } else {
2205         $pciaddr = print_pci_addr($vgaid, $bridges, $arch, $machine);
2206     }
2207
2208     return "$type,id=${vgaid}${memory}${pciaddr}";
2209 }
2210
2211 sub drive_is_cloudinit {
2212     my ($drive) = @_;
2213     return $drive->{file} =~ m@[:/]vm-\d+-cloudinit(?:\.$QEMU_FORMAT_RE)?$@;
2214 }
2215
2216 sub drive_is_cdrom {
2217     my ($drive, $exclude_cloudinit) = @_;
2218
2219     return 0 if $exclude_cloudinit && drive_is_cloudinit($drive);
2220
2221     return $drive && $drive->{media} && ($drive->{media} eq 'cdrom');
2222
2223 }
2224
2225 sub parse_number_sets {
2226     my ($set) = @_;
2227     my $res = [];
2228     foreach my $part (split(/;/, $set)) {
2229         if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
2230             die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
2231             push @$res, [ $1, $2 ];
2232         } else {
2233             die "invalid range: $part\n";
2234         }
2235     }
2236     return $res;
2237 }
2238
2239 sub parse_numa {
2240     my ($data) = @_;
2241
2242     my $res = PVE::JSONSchema::parse_property_string($numa_fmt, $data);
2243     $res->{cpus} = parse_number_sets($res->{cpus}) if defined($res->{cpus});
2244     $res->{hostnodes} = parse_number_sets($res->{hostnodes}) if defined($res->{hostnodes});
2245     return $res;
2246 }
2247
2248 sub parse_hostpci {
2249     my ($value) = @_;
2250
2251     return undef if !$value;
2252
2253     my $res = PVE::JSONSchema::parse_property_string($hostpci_fmt, $value);
2254
2255     my @idlist = split(/;/, $res->{host});
2256     delete $res->{host};
2257     foreach my $id (@idlist) {
2258         if ($id =~ m/\./) { # full id 00:00.1
2259             push @{$res->{pciid}}, {
2260                 id => $id,
2261             };
2262         } else { # partial id 00:00
2263             $res->{pciid} = PVE::SysFSTools::lspci($id);
2264         }
2265     }
2266     return $res;
2267 }
2268
2269 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
2270 sub parse_net {
2271     my ($data) = @_;
2272
2273     my $res = eval { PVE::JSONSchema::parse_property_string($net_fmt, $data) };
2274     if ($@) {
2275         warn $@;
2276         return undef;
2277     }
2278     if (!defined($res->{macaddr})) {
2279         my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
2280         $res->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix});
2281     }
2282     return $res;
2283 }
2284
2285 # ipconfigX ip=cidr,gw=ip,ip6=cidr,gw6=ip
2286 sub parse_ipconfig {
2287     my ($data) = @_;
2288
2289     my $res = eval { PVE::JSONSchema::parse_property_string($ipconfig_fmt, $data) };
2290     if ($@) {
2291         warn $@;
2292         return undef;
2293     }
2294
2295     if ($res->{gw} && !$res->{ip}) {
2296         warn 'gateway specified without specifying an IP address';
2297         return undef;
2298     }
2299     if ($res->{gw6} && !$res->{ip6}) {
2300         warn 'IPv6 gateway specified without specifying an IPv6 address';
2301         return undef;
2302     }
2303     if ($res->{gw} && $res->{ip} eq 'dhcp') {
2304         warn 'gateway specified together with DHCP';
2305         return undef;
2306     }
2307     if ($res->{gw6} && $res->{ip6} !~ /^$IPV6RE/) {
2308         # gw6 + auto/dhcp
2309         warn "IPv6 gateway specified together with $res->{ip6} address";
2310         return undef;
2311     }
2312
2313     if (!$res->{ip} && !$res->{ip6}) {
2314         return { ip => 'dhcp', ip6 => 'dhcp' };
2315     }
2316
2317     return $res;
2318 }
2319
2320 sub print_net {
2321     my $net = shift;
2322
2323     return PVE::JSONSchema::print_property_string($net, $net_fmt);
2324 }
2325
2326 sub add_random_macs {
2327     my ($settings) = @_;
2328
2329     foreach my $opt (keys %$settings) {
2330         next if $opt !~ m/^net(\d+)$/;
2331         my $net = parse_net($settings->{$opt});
2332         next if !$net;
2333         $settings->{$opt} = print_net($net);
2334     }
2335 }
2336
2337 sub vm_is_volid_owner {
2338     my ($storecfg, $vmid, $volid) = @_;
2339
2340     if ($volid !~  m|^/|) {
2341         my ($path, $owner);
2342         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
2343         if ($owner && ($owner == $vmid)) {
2344             return 1;
2345         }
2346     }
2347
2348     return undef;
2349 }
2350
2351 sub split_flagged_list {
2352     my $text = shift || '';
2353     $text =~ s/[,;]/ /g;
2354     $text =~ s/^\s+//;
2355     return { map { /^(!?)(.*)$/ && ($2, $1) } ($text =~ /\S+/g) };
2356 }
2357
2358 sub join_flagged_list {
2359     my ($how, $lst) = @_;
2360     join $how, map { $lst->{$_} . $_ } keys %$lst;
2361 }
2362
2363 sub vmconfig_delete_pending_option {
2364     my ($conf, $key, $force) = @_;
2365
2366     delete $conf->{pending}->{$key};
2367     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
2368     $pending_delete_hash->{$key} = $force ? '!' : '';
2369     $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
2370 }
2371
2372 sub vmconfig_undelete_pending_option {
2373     my ($conf, $key) = @_;
2374
2375     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
2376     delete $pending_delete_hash->{$key};
2377
2378     if (%$pending_delete_hash) {
2379         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
2380     } else {
2381         delete $conf->{pending}->{delete};
2382     }
2383 }
2384
2385 sub vmconfig_register_unused_drive {
2386     my ($storecfg, $vmid, $conf, $drive) = @_;
2387
2388     if (drive_is_cloudinit($drive)) {
2389         eval { PVE::Storage::vdisk_free($storecfg, $drive->{file}) };
2390         warn $@ if $@;
2391     } elsif (!drive_is_cdrom($drive)) {
2392         my $volid = $drive->{file};
2393         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
2394             PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
2395         }
2396     }
2397 }
2398
2399 sub vmconfig_cleanup_pending {
2400     my ($conf) = @_;
2401
2402     # remove pending changes when nothing changed
2403     my $changes;
2404     foreach my $opt (keys %{$conf->{pending}}) {
2405         if (defined($conf->{$opt}) && ($conf->{pending}->{$opt} eq  $conf->{$opt})) {
2406             $changes = 1;
2407             delete $conf->{pending}->{$opt};
2408         }
2409     }
2410
2411     my $current_delete_hash = split_flagged_list($conf->{pending}->{delete});
2412     my $pending_delete_hash = {};
2413     while (my ($opt, $force) = each %$current_delete_hash) {
2414         if (defined($conf->{$opt})) {
2415             $pending_delete_hash->{$opt} = $force;
2416         } else {
2417             $changes = 1;
2418         }
2419     }
2420
2421     if (%$pending_delete_hash) {
2422         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
2423     } else {
2424         delete $conf->{pending}->{delete};
2425     }
2426
2427     return $changes;
2428 }
2429
2430 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str][,base64=bool]
2431 my $smbios1_fmt = {
2432     uuid => {
2433         type => 'string',
2434         pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
2435         format_description => 'UUID',
2436         description => "Set SMBIOS1 UUID.",
2437         optional => 1,
2438     },
2439     version => {
2440         type => 'string',
2441         pattern => '[A-Za-z0-9+\/]+={0,2}',
2442         format_description => 'Base64 encoded string',
2443         description => "Set SMBIOS1 version.",
2444         optional => 1,
2445     },
2446     serial => {
2447         type => 'string',
2448         pattern => '[A-Za-z0-9+\/]+={0,2}',
2449         format_description => 'Base64 encoded string',
2450         description => "Set SMBIOS1 serial number.",
2451         optional => 1,
2452     },
2453     manufacturer => {
2454         type => 'string',
2455         pattern => '[A-Za-z0-9+\/]+={0,2}',
2456         format_description => 'Base64 encoded string',
2457         description => "Set SMBIOS1 manufacturer.",
2458         optional => 1,
2459     },
2460     product => {
2461         type => 'string',
2462         pattern => '[A-Za-z0-9+\/]+={0,2}',
2463         format_description => 'Base64 encoded string',
2464         description => "Set SMBIOS1 product ID.",
2465         optional => 1,
2466     },
2467     sku => {
2468         type => 'string',
2469         pattern => '[A-Za-z0-9+\/]+={0,2}',
2470         format_description => 'Base64 encoded string',
2471         description => "Set SMBIOS1 SKU string.",
2472         optional => 1,
2473     },
2474     family => {
2475         type => 'string',
2476         pattern => '[A-Za-z0-9+\/]+={0,2}',
2477         format_description => 'Base64 encoded string',
2478         description => "Set SMBIOS1 family string.",
2479         optional => 1,
2480     },
2481     base64 => {
2482         type => 'boolean',
2483         description => 'Flag to indicate that the SMBIOS values are base64 encoded',
2484         optional => 1,
2485     },
2486 };
2487
2488 sub parse_smbios1 {
2489     my ($data) = @_;
2490
2491     my $res = eval { PVE::JSONSchema::parse_property_string($smbios1_fmt, $data) };
2492     warn $@ if $@;
2493     return $res;
2494 }
2495
2496 sub print_smbios1 {
2497     my ($smbios1) = @_;
2498     return PVE::JSONSchema::print_property_string($smbios1, $smbios1_fmt);
2499 }
2500
2501 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_fmt);
2502
2503 PVE::JSONSchema::register_format('pve-qm-bootdisk', \&verify_bootdisk);
2504 sub verify_bootdisk {
2505     my ($value, $noerr) = @_;
2506
2507     return $value if is_valid_drivename($value);
2508
2509     return undef if $noerr;
2510
2511     die "invalid boot disk '$value'\n";
2512 }
2513
2514 sub parse_watchdog {
2515     my ($value) = @_;
2516
2517     return undef if !$value;
2518
2519     my $res = eval { PVE::JSONSchema::parse_property_string($watchdog_fmt, $value) };
2520     warn $@ if $@;
2521     return $res;
2522 }
2523
2524 sub parse_guest_agent {
2525     my ($value) = @_;
2526
2527     return {} if !defined($value->{agent});
2528
2529     my $res = eval { PVE::JSONSchema::parse_property_string($agent_fmt, $value->{agent}) };
2530     warn $@ if $@;
2531
2532     # if the agent is disabled ignore the other potentially set properties
2533     return {} if !$res->{enabled};
2534     return $res;
2535 }
2536
2537 sub parse_vga {
2538     my ($value) = @_;
2539
2540     return {} if !$value;
2541     my $res = eval { PVE::JSONSchema::parse_property_string($vga_fmt, $value) };
2542     warn $@ if $@;
2543     return $res;
2544 }
2545
2546 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
2547 sub verify_usb_device {
2548     my ($value, $noerr) = @_;
2549
2550     return $value if parse_usb_device($value);
2551
2552     return undef if $noerr;
2553
2554     die "unable to parse usb device\n";
2555 }
2556
2557 # add JSON properties for create and set function
2558 sub json_config_properties {
2559     my $prop = shift;
2560
2561     foreach my $opt (keys %$confdesc) {
2562         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate' || $opt eq 'runningmachine';
2563         $prop->{$opt} = $confdesc->{$opt};
2564     }
2565
2566     return $prop;
2567 }
2568
2569 # return copy of $confdesc_cloudinit to generate documentation
2570 sub cloudinit_config_properties {
2571
2572     return dclone($confdesc_cloudinit);
2573 }
2574
2575 sub check_type {
2576     my ($key, $value) = @_;
2577
2578     die "unknown setting '$key'\n" if !$confdesc->{$key};
2579
2580     my $type = $confdesc->{$key}->{type};
2581
2582     if (!defined($value)) {
2583         die "got undefined value\n";
2584     }
2585
2586     if ($value =~ m/[\n\r]/) {
2587         die "property contains a line feed\n";
2588     }
2589
2590     if ($type eq 'boolean') {
2591         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
2592         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
2593         die "type check ('boolean') failed - got '$value'\n";
2594     } elsif ($type eq 'integer') {
2595         return int($1) if $value =~ m/^(\d+)$/;
2596         die "type check ('integer') failed - got '$value'\n";
2597     } elsif ($type eq 'number') {
2598         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
2599         die "type check ('number') failed - got '$value'\n";
2600     } elsif ($type eq 'string') {
2601         if (my $fmt = $confdesc->{$key}->{format}) {
2602             PVE::JSONSchema::check_format($fmt, $value);
2603             return $value;
2604         }
2605         $value =~ s/^\"(.*)\"$/$1/;
2606         return $value;
2607     } else {
2608         die "internal error"
2609     }
2610 }
2611
2612 sub touch_config {
2613     my ($vmid) = @_;
2614
2615     my $conf = PVE::QemuConfig->config_file($vmid);
2616     utime undef, undef, $conf;
2617 }
2618
2619 sub destroy_vm {
2620     my ($storecfg, $vmid, $keep_empty_config, $skiplock) = @_;
2621
2622     my $conffile = PVE::QemuConfig->config_file($vmid);
2623
2624     my $conf = PVE::QemuConfig->load_config($vmid);
2625
2626     PVE::QemuConfig->check_lock($conf) if !$skiplock;
2627
2628     if ($conf->{template}) {
2629         # check if any base image is still used by a linked clone
2630         foreach_drive($conf, sub {
2631                 my ($ds, $drive) = @_;
2632
2633                 return if drive_is_cdrom($drive);
2634
2635                 my $volid = $drive->{file};
2636
2637                 return if !$volid || $volid =~ m|^/|;
2638
2639                 die "base volume '$volid' is still in use by linked cloned\n"
2640                     if PVE::Storage::volume_is_base_and_used($storecfg, $volid);
2641
2642         });
2643     }
2644
2645     # only remove disks owned by this VM
2646     foreach_drive($conf, sub {
2647         my ($ds, $drive) = @_;
2648
2649         return if drive_is_cdrom($drive, 1);
2650
2651         my $volid = $drive->{file};
2652
2653         return if !$volid || $volid =~ m|^/|;
2654
2655         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
2656         return if !$path || !$owner || ($owner != $vmid);
2657
2658         eval {
2659             PVE::Storage::vdisk_free($storecfg, $volid);
2660         };
2661         warn "Could not remove disk '$volid', check manually: $@" if $@;
2662
2663     });
2664
2665     if ($keep_empty_config) {
2666         PVE::Tools::file_set_contents($conffile, "memory: 128\n");
2667     } else {
2668         unlink $conffile;
2669     }
2670
2671     # also remove unused disk
2672     eval {
2673         my $dl = PVE::Storage::vdisk_list($storecfg, undef, $vmid);
2674
2675         eval {
2676             PVE::Storage::foreach_volid($dl, sub {
2677                 my ($volid, $sid, $volname, $d) = @_;
2678                 PVE::Storage::vdisk_free($storecfg, $volid);
2679             });
2680         };
2681         warn $@ if $@;
2682
2683     };
2684     warn $@ if $@;
2685 }
2686
2687 sub parse_vm_config {
2688     my ($filename, $raw) = @_;
2689
2690     return undef if !defined($raw);
2691
2692     my $res = {
2693         digest => Digest::SHA::sha1_hex($raw),
2694         snapshots => {},
2695         pending => {},
2696     };
2697
2698     $filename =~ m|/qemu-server/(\d+)\.conf$|
2699         || die "got strange filename '$filename'";
2700
2701     my $vmid = $1;
2702
2703     my $conf = $res;
2704     my $descr;
2705     my $section = '';
2706
2707     my @lines = split(/\n/, $raw);
2708     foreach my $line (@lines) {
2709         next if $line =~ m/^\s*$/;
2710
2711         if ($line =~ m/^\[PENDING\]\s*$/i) {
2712             $section = 'pending';
2713             if (defined($descr)) {
2714                 $descr =~ s/\s+$//;
2715                 $conf->{description} = $descr;
2716             }
2717             $descr = undef;
2718             $conf = $res->{$section} = {};
2719             next;
2720
2721         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2722             $section = $1;
2723             if (defined($descr)) {
2724                 $descr =~ s/\s+$//;
2725                 $conf->{description} = $descr;
2726             }
2727             $descr = undef;
2728             $conf = $res->{snapshots}->{$section} = {};
2729             next;
2730         }
2731
2732         if ($line =~ m/^\#(.*)\s*$/) {
2733             $descr = '' if !defined($descr);
2734             $descr .= PVE::Tools::decode_text($1) . "\n";
2735             next;
2736         }
2737
2738         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2739             $descr = '' if !defined($descr);
2740             $descr .= PVE::Tools::decode_text($2);
2741         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2742             $conf->{snapstate} = $1;
2743         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2744             my $key = $1;
2745             my $value = $2;
2746             $conf->{$key} = $value;
2747         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2748             my $value = $1;
2749             if ($section eq 'pending') {
2750                 $conf->{delete} = $value; # we parse this later
2751             } else {
2752                 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
2753             }
2754         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(.+?)\s*$/) {
2755             my $key = $1;
2756             my $value = $2;
2757             eval { $value = check_type($key, $value); };
2758             if ($@) {
2759                 warn "vm $vmid - unable to parse value of '$key' - $@";
2760             } else {
2761                 $key = 'ide2' if $key eq 'cdrom';
2762                 my $fmt = $confdesc->{$key}->{format};
2763                 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2764                     my $v = parse_drive($key, $value);
2765                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2766                         $v->{file} = $volid;
2767                         $value = print_drive($vmid, $v);
2768                     } else {
2769                         warn "vm $vmid - unable to parse value of '$key'\n";
2770                         next;
2771                     }
2772                 }
2773
2774                 $conf->{$key} = $value;
2775             }
2776         }
2777     }
2778
2779     if (defined($descr)) {
2780         $descr =~ s/\s+$//;
2781         $conf->{description} = $descr;
2782     }
2783     delete $res->{snapstate}; # just to be sure
2784
2785     return $res;
2786 }
2787
2788 sub write_vm_config {
2789     my ($filename, $conf) = @_;
2790
2791     delete $conf->{snapstate}; # just to be sure
2792
2793     if ($conf->{cdrom}) {
2794         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2795         $conf->{ide2} = $conf->{cdrom};
2796         delete $conf->{cdrom};
2797     }
2798
2799     # we do not use 'smp' any longer
2800     if ($conf->{sockets}) {
2801         delete $conf->{smp};
2802     } elsif ($conf->{smp}) {
2803         $conf->{sockets} = $conf->{smp};
2804         delete $conf->{cores};
2805         delete $conf->{smp};
2806     }
2807
2808     my $used_volids = {};
2809
2810     my $cleanup_config = sub {
2811         my ($cref, $pending, $snapname) = @_;
2812
2813         foreach my $key (keys %$cref) {
2814             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2815                 $key eq 'snapstate' || $key eq 'pending';
2816             my $value = $cref->{$key};
2817             if ($key eq 'delete') {
2818                 die "propertry 'delete' is only allowed in [PENDING]\n"
2819                     if !$pending;
2820                 # fixme: check syntax?
2821                 next;
2822             }
2823             eval { $value = check_type($key, $value); };
2824             die "unable to parse value of '$key' - $@" if $@;
2825
2826             $cref->{$key} = $value;
2827
2828             if (!$snapname && is_valid_drivename($key)) {
2829                 my $drive = parse_drive($key, $value);
2830                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2831             }
2832         }
2833     };
2834
2835     &$cleanup_config($conf);
2836
2837     &$cleanup_config($conf->{pending}, 1);
2838
2839     foreach my $snapname (keys %{$conf->{snapshots}}) {
2840         die "internal error" if $snapname eq 'pending';
2841         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2842     }
2843
2844     # remove 'unusedX' settings if we re-add a volume
2845     foreach my $key (keys %$conf) {
2846         my $value = $conf->{$key};
2847         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2848             delete $conf->{$key};
2849         }
2850     }
2851
2852     my $generate_raw_config = sub {
2853         my ($conf, $pending) = @_;
2854
2855         my $raw = '';
2856
2857         # add description as comment to top of file
2858         if (defined(my $descr = $conf->{description})) {
2859             if ($descr) {
2860                 foreach my $cl (split(/\n/, $descr)) {
2861                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2862                 }
2863             } else {
2864                 $raw .= "#\n" if $pending;
2865             }
2866         }
2867
2868         foreach my $key (sort keys %$conf) {
2869             next if $key eq 'digest' || $key eq 'description' || $key eq 'pending' || $key eq 'snapshots';
2870             $raw .= "$key: $conf->{$key}\n";
2871         }
2872         return $raw;
2873     };
2874
2875     my $raw = &$generate_raw_config($conf);
2876
2877     if (scalar(keys %{$conf->{pending}})){
2878         $raw .= "\n[PENDING]\n";
2879         $raw .= &$generate_raw_config($conf->{pending}, 1);
2880     }
2881
2882     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2883         $raw .= "\n[$snapname]\n";
2884         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2885     }
2886
2887     return $raw;
2888 }
2889
2890 sub load_defaults {
2891
2892     my $res = {};
2893
2894     # we use static defaults from our JSON schema configuration
2895     foreach my $key (keys %$confdesc) {
2896         if (defined(my $default = $confdesc->{$key}->{default})) {
2897             $res->{$key} = $default;
2898         }
2899     }
2900
2901     return $res;
2902 }
2903
2904 sub config_list {
2905     my $vmlist = PVE::Cluster::get_vmlist();
2906     my $res = {};
2907     return $res if !$vmlist || !$vmlist->{ids};
2908     my $ids = $vmlist->{ids};
2909
2910     foreach my $vmid (keys %$ids) {
2911         my $d = $ids->{$vmid};
2912         next if !$d->{node} || $d->{node} ne $nodename;
2913         next if !$d->{type} || $d->{type} ne 'qemu';
2914         $res->{$vmid}->{exists} = 1;
2915     }
2916     return $res;
2917 }
2918
2919 # test if VM uses local resources (to prevent migration)
2920 sub check_local_resources {
2921     my ($conf, $noerr) = @_;
2922
2923     my @loc_res = ();
2924
2925     push @loc_res, "hostusb" if $conf->{hostusb}; # old syntax
2926     push @loc_res, "hostpci" if $conf->{hostpci}; # old syntax
2927
2928     push @loc_res, "ivshmem" if $conf->{ivshmem};
2929
2930     foreach my $k (keys %$conf) {
2931         next if $k =~ m/^usb/ && ($conf->{$k} =~ m/^spice(?![^,])/);
2932         # sockets are safe: they will recreated be on the target side post-migrate
2933         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2934         push @loc_res, $k if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2935     }
2936
2937     die "VM uses local resources\n" if scalar @loc_res && !$noerr;
2938
2939     return \@loc_res;
2940 }
2941
2942 # check if used storages are available on all nodes (use by migrate)
2943 sub check_storage_availability {
2944     my ($storecfg, $conf, $node) = @_;
2945
2946     foreach_drive($conf, sub {
2947         my ($ds, $drive) = @_;
2948
2949         my $volid = $drive->{file};
2950         return if !$volid;
2951
2952         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2953         return if !$sid;
2954
2955         # check if storage is available on both nodes
2956         my $scfg = PVE::Storage::storage_check_node($storecfg, $sid);
2957         PVE::Storage::storage_check_node($storecfg, $sid, $node);
2958    });
2959 }
2960
2961 # list nodes where all VM images are available (used by has_feature API)
2962 sub shared_nodes {
2963     my ($conf, $storecfg) = @_;
2964
2965     my $nodelist = PVE::Cluster::get_nodelist();
2966     my $nodehash = { map { $_ => 1 } @$nodelist };
2967     my $nodename = PVE::INotify::nodename();
2968
2969     foreach_drive($conf, sub {
2970         my ($ds, $drive) = @_;
2971
2972         my $volid = $drive->{file};
2973         return if !$volid;
2974
2975         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2976         if ($storeid) {
2977             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2978             if ($scfg->{disable}) {
2979                 $nodehash = {};
2980             } elsif (my $avail = $scfg->{nodes}) {
2981                 foreach my $node (keys %$nodehash) {
2982                     delete $nodehash->{$node} if !$avail->{$node};
2983                 }
2984             } elsif (!$scfg->{shared}) {
2985                 foreach my $node (keys %$nodehash) {
2986                     delete $nodehash->{$node} if $node ne $nodename
2987                 }
2988             }
2989         }
2990     });
2991
2992     return $nodehash
2993 }
2994
2995 sub check_local_storage_availability {
2996     my ($conf, $storecfg) = @_;
2997
2998     my $nodelist = PVE::Cluster::get_nodelist();
2999     my $nodehash = { map { $_ => {} } @$nodelist };
3000
3001     foreach_drive($conf, sub {
3002         my ($ds, $drive) = @_;
3003
3004         my $volid = $drive->{file};
3005         return if !$volid;
3006
3007         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
3008         if ($storeid) {
3009             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
3010
3011             if ($scfg->{disable}) {
3012                 foreach my $node (keys %$nodehash) {
3013                     $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
3014                 }
3015             } elsif (my $avail = $scfg->{nodes}) {
3016                 foreach my $node (keys %$nodehash) {
3017                     if (!$avail->{$node}) {
3018                         $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
3019                     }
3020                 }
3021             }
3022         }
3023     });
3024
3025     foreach my $node (values %$nodehash) {
3026         if (my $unavail = $node->{unavailable_storages}) {
3027             $node->{unavailable_storages} = [ sort keys %$unavail ];
3028         }
3029     }
3030
3031     return $nodehash
3032 }
3033
3034 sub check_cmdline {
3035     my ($pidfile, $pid) = @_;
3036
3037     my $fh = IO::File->new("/proc/$pid/cmdline", "r");
3038     if (defined($fh)) {
3039         my $line = <$fh>;
3040         $fh->close;
3041         return undef if !$line;
3042         my @param = split(/\0/, $line);
3043
3044         my $cmd = $param[0];
3045         return if !$cmd || ($cmd !~ m|kvm$| && $cmd !~ m@(?:^|/)qemu-system-[^/]+$@);
3046
3047         for (my $i = 0; $i < scalar (@param); $i++) {
3048             my $p = $param[$i];
3049             next if !$p;
3050             if (($p eq '-pidfile') || ($p eq '--pidfile')) {
3051                 my $p = $param[$i+1];
3052                 return 1 if $p && ($p eq $pidfile);
3053                 return undef;
3054             }
3055         }
3056     }
3057     return undef;
3058 }
3059
3060 sub check_running {
3061     my ($vmid, $nocheck, $node) = @_;
3062
3063     my $filename = PVE::QemuConfig->config_file($vmid, $node);
3064
3065     die "unable to find configuration file for VM $vmid - no such machine\n"
3066         if !$nocheck && ! -f $filename;
3067
3068     my $pidfile = pidfile_name($vmid);
3069
3070     if (my $fd = IO::File->new("<$pidfile")) {
3071         my $st = stat($fd);
3072         my $line = <$fd>;
3073         close($fd);
3074
3075         my $mtime = $st->mtime;
3076         if ($mtime > time()) {
3077             warn "file '$filename' modified in future\n";
3078         }
3079
3080         if ($line =~ m/^(\d+)$/) {
3081             my $pid = $1;
3082             if (check_cmdline($pidfile, $pid)) {
3083                 if (my $pinfo = PVE::ProcFSTools::check_process_running($pid)) {
3084                     return $pid;
3085                 }
3086             }
3087         }
3088     }
3089
3090     return undef;
3091 }
3092
3093 sub vzlist {
3094
3095     my $vzlist = config_list();
3096
3097     my $fd = IO::Dir->new($var_run_tmpdir) || return $vzlist;
3098
3099     while (defined(my $de = $fd->read)) {
3100         next if $de !~ m/^(\d+)\.pid$/;
3101         my $vmid = $1;
3102         next if !defined($vzlist->{$vmid});
3103         if (my $pid = check_running($vmid)) {
3104             $vzlist->{$vmid}->{pid} = $pid;
3105         }
3106     }
3107
3108     return $vzlist;
3109 }
3110
3111 sub disksize {
3112     my ($storecfg, $conf) = @_;
3113
3114     my $bootdisk = $conf->{bootdisk};
3115     return undef if !$bootdisk;
3116     return undef if !is_valid_drivename($bootdisk);
3117
3118     return undef if !$conf->{$bootdisk};
3119
3120     my $drive = parse_drive($bootdisk, $conf->{$bootdisk});
3121     return undef if !defined($drive);
3122
3123     return undef if drive_is_cdrom($drive);
3124
3125     my $volid = $drive->{file};
3126     return undef if !$volid;
3127
3128     return $drive->{size};
3129 }
3130
3131 our $vmstatus_return_properties = {
3132     vmid => get_standard_option('pve-vmid'),
3133     status => {
3134         description => "Qemu process status.",
3135         type => 'string',
3136         enum => ['stopped', 'running'],
3137     },
3138     maxmem => {
3139         description => "Maximum memory in bytes.",
3140         type => 'integer',
3141         optional => 1,
3142         renderer => 'bytes',
3143     },
3144     maxdisk => {
3145         description => "Root disk size in bytes.",
3146         type => 'integer',
3147         optional => 1,
3148         renderer => 'bytes',
3149     },
3150     name => {
3151         description => "VM name.",
3152         type => 'string',
3153         optional => 1,
3154     },
3155     qmpstatus => {
3156         description => "Qemu QMP agent status.",
3157         type => 'string',
3158         optional => 1,
3159     },
3160     pid => {
3161         description => "PID of running qemu process.",
3162         type => 'integer',
3163         optional => 1,
3164     },
3165     uptime => {
3166         description => "Uptime.",
3167         type => 'integer',
3168         optional => 1,
3169         renderer => 'duration',
3170     },
3171     cpus => {
3172         description => "Maximum usable CPUs.",
3173         type => 'number',
3174         optional => 1,
3175     },
3176     lock => {
3177         description => "The current config lock, if any.",
3178         type => 'string',
3179         optional => 1,
3180     }
3181 };
3182
3183 my $last_proc_pid_stat;
3184
3185 # get VM status information
3186 # This must be fast and should not block ($full == false)
3187 # We only query KVM using QMP if $full == true (this can be slow)
3188 sub vmstatus {
3189     my ($opt_vmid, $full) = @_;
3190
3191     my $res = {};
3192
3193     my $storecfg = PVE::Storage::config();
3194
3195     my $list = vzlist();
3196     my $defaults = load_defaults();
3197
3198     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
3199
3200     my $cpucount = $cpuinfo->{cpus} || 1;
3201
3202     foreach my $vmid (keys %$list) {
3203         next if $opt_vmid && ($vmid ne $opt_vmid);
3204
3205         my $cfspath = PVE::QemuConfig->cfs_config_path($vmid);
3206         my $conf = PVE::Cluster::cfs_read_file($cfspath) || {};
3207
3208         my $d = { vmid => $vmid };
3209         $d->{pid} = $list->{$vmid}->{pid};
3210
3211         # fixme: better status?
3212         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
3213
3214         my $size = disksize($storecfg, $conf);
3215         if (defined($size)) {
3216             $d->{disk} = 0; # no info available
3217             $d->{maxdisk} = $size;
3218         } else {
3219             $d->{disk} = 0;
3220             $d->{maxdisk} = 0;
3221         }
3222
3223         $d->{cpus} = ($conf->{sockets} || $defaults->{sockets})
3224             * ($conf->{cores} || $defaults->{cores});
3225         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
3226         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
3227
3228         $d->{name} = $conf->{name} || "VM $vmid";
3229         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024)
3230             : $defaults->{memory}*(1024*1024);
3231
3232         if ($conf->{balloon}) {
3233             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
3234             $d->{shares} = defined($conf->{shares}) ? $conf->{shares}
3235                 : $defaults->{shares};
3236         }
3237
3238         $d->{uptime} = 0;
3239         $d->{cpu} = 0;
3240         $d->{mem} = 0;
3241
3242         $d->{netout} = 0;
3243         $d->{netin} = 0;
3244
3245         $d->{diskread} = 0;
3246         $d->{diskwrite} = 0;
3247
3248         $d->{template} = PVE::QemuConfig->is_template($conf);
3249
3250         $d->{serial} = 1 if conf_has_serial($conf);
3251         $d->{lock} = $conf->{lock} if $conf->{lock};
3252
3253         $res->{$vmid} = $d;
3254     }
3255
3256     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
3257     foreach my $dev (keys %$netdev) {
3258         next if $dev !~ m/^tap([1-9]\d*)i/;
3259         my $vmid = $1;
3260         my $d = $res->{$vmid};
3261         next if !$d;
3262
3263         $d->{netout} += $netdev->{$dev}->{receive};
3264         $d->{netin} += $netdev->{$dev}->{transmit};
3265
3266         if ($full) {
3267             $d->{nics}->{$dev}->{netout} = $netdev->{$dev}->{receive};
3268             $d->{nics}->{$dev}->{netin} = $netdev->{$dev}->{transmit};
3269         }
3270
3271     }
3272
3273     my $ctime = gettimeofday;
3274
3275     foreach my $vmid (keys %$list) {
3276
3277         my $d = $res->{$vmid};
3278         my $pid = $d->{pid};
3279         next if !$pid;
3280
3281         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
3282         next if !$pstat; # not running
3283
3284         my $used = $pstat->{utime} + $pstat->{stime};
3285
3286         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
3287
3288         if ($pstat->{vsize}) {
3289             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
3290         }
3291
3292         my $old = $last_proc_pid_stat->{$pid};
3293         if (!$old) {
3294             $last_proc_pid_stat->{$pid} = {
3295                 time => $ctime,
3296                 used => $used,
3297                 cpu => 0,
3298             };
3299             next;
3300         }
3301
3302         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
3303
3304         if ($dtime > 1000) {
3305             my $dutime = $used -  $old->{used};
3306
3307             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
3308             $last_proc_pid_stat->{$pid} = {
3309                 time => $ctime,
3310                 used => $used,
3311                 cpu => $d->{cpu},
3312             };
3313         } else {
3314             $d->{cpu} = $old->{cpu};
3315         }
3316     }
3317
3318     return $res if !$full;
3319
3320     my $qmpclient = PVE::QMPClient->new();
3321
3322     my $ballooncb = sub {
3323         my ($vmid, $resp) = @_;
3324
3325         my $info = $resp->{'return'};
3326         return if !$info->{max_mem};
3327
3328         my $d = $res->{$vmid};
3329
3330         # use memory assigned to VM
3331         $d->{maxmem} = $info->{max_mem};
3332         $d->{balloon} = $info->{actual};
3333
3334         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
3335             $d->{mem} = $info->{total_mem} - $info->{free_mem};
3336             $d->{freemem} = $info->{free_mem};
3337         }
3338
3339         $d->{ballooninfo} = $info;
3340     };
3341
3342     my $blockstatscb = sub {
3343         my ($vmid, $resp) = @_;
3344         my $data = $resp->{'return'} || [];
3345         my $totalrdbytes = 0;
3346         my $totalwrbytes = 0;
3347
3348         for my $blockstat (@$data) {
3349             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
3350             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
3351
3352             $blockstat->{device} =~ s/drive-//;
3353             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
3354         }
3355         $res->{$vmid}->{diskread} = $totalrdbytes;
3356         $res->{$vmid}->{diskwrite} = $totalwrbytes;
3357     };
3358
3359     my $statuscb = sub {
3360         my ($vmid, $resp) = @_;
3361
3362         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
3363         # this fails if ballon driver is not loaded, so this must be
3364         # the last commnand (following command are aborted if this fails).
3365         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
3366
3367         my $status = 'unknown';
3368         if (!defined($status = $resp->{'return'}->{status})) {
3369             warn "unable to get VM status\n";
3370             return;
3371         }
3372
3373         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
3374     };
3375
3376     foreach my $vmid (keys %$list) {
3377         next if $opt_vmid && ($vmid ne $opt_vmid);
3378         next if !$res->{$vmid}->{pid}; # not running
3379         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
3380     }
3381
3382     $qmpclient->queue_execute(undef, 2);
3383
3384     foreach my $vmid (keys %$list) {
3385         next if $opt_vmid && ($vmid ne $opt_vmid);
3386         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
3387     }
3388
3389     return $res;
3390 }
3391
3392 sub foreach_drive {
3393     my ($conf, $func, @param) = @_;
3394
3395     foreach my $ds (valid_drive_names()) {
3396         next if !defined($conf->{$ds});
3397
3398         my $drive = parse_drive($ds, $conf->{$ds});
3399         next if !$drive;
3400
3401         &$func($ds, $drive, @param);
3402     }
3403 }
3404
3405 sub foreach_volid {
3406     my ($conf, $func, @param) = @_;
3407
3408     my $volhash = {};
3409
3410     my $test_volid = sub {
3411         my ($volid, $is_cdrom, $replicate, $shared, $snapname, $size) = @_;
3412
3413         return if !$volid;
3414
3415         $volhash->{$volid}->{cdrom} //= 1;
3416         $volhash->{$volid}->{cdrom} = 0 if !$is_cdrom;
3417
3418         $volhash->{$volid}->{replicate} //= 0;
3419         $volhash->{$volid}->{replicate} = 1 if $replicate;
3420
3421         $volhash->{$volid}->{shared} //= 0;
3422         $volhash->{$volid}->{shared} = 1 if $shared;
3423
3424         $volhash->{$volid}->{referenced_in_config} //= 0;
3425         $volhash->{$volid}->{referenced_in_config} = 1 if !defined($snapname);
3426
3427         $volhash->{$volid}->{referenced_in_snapshot}->{$snapname} = 1
3428             if defined($snapname);
3429         $volhash->{$volid}->{size} = $size if $size;
3430     };
3431
3432     foreach_drive($conf, sub {
3433         my ($ds, $drive) = @_;
3434         $test_volid->($drive->{file}, drive_is_cdrom($drive), $drive->{replicate} // 1, $drive->{shared}, undef, $drive->{size});
3435     });
3436
3437     foreach my $snapname (keys %{$conf->{snapshots}}) {
3438         my $snap = $conf->{snapshots}->{$snapname};
3439         $test_volid->($snap->{vmstate}, 0, 1, $snapname);
3440         foreach_drive($snap, sub {
3441             my ($ds, $drive) = @_;
3442             $test_volid->($drive->{file}, drive_is_cdrom($drive), $drive->{replicate} // 1, $drive->{shared}, $snapname);
3443         });
3444     }
3445
3446     foreach my $volid (keys %$volhash) {
3447         &$func($volid, $volhash->{$volid}, @param);
3448     }
3449 }
3450
3451 sub conf_has_serial {
3452     my ($conf) = @_;
3453
3454     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3455         if ($conf->{"serial$i"}) {
3456             return 1;
3457         }
3458     }
3459
3460     return 0;
3461 }
3462
3463 sub conf_has_audio {
3464     my ($conf, $id) = @_;
3465
3466     $id //= 0;
3467     my $audio = $conf->{"audio$id"};
3468     return undef if !defined($audio);
3469
3470     my $audioproperties = PVE::JSONSchema::parse_property_string($audio_fmt, $audio);
3471     my $audiodriver = $audioproperties->{driver} // 'spice';
3472
3473     return {
3474         dev => $audioproperties->{device},
3475         dev_id => "audiodev$id",
3476         backend => $audiodriver,
3477         backend_id => "$audiodriver-backend${id}",
3478     };
3479 }
3480
3481 sub vga_conf_has_spice {
3482     my ($vga) = @_;
3483
3484     my $vgaconf = parse_vga($vga);
3485     my $vgatype = $vgaconf->{type};
3486     return 0 if !$vgatype || $vgatype !~ m/^qxl([234])?$/;
3487
3488     return $1 || 1;
3489 }
3490
3491 my $host_arch; # FIXME: fix PVE::Tools::get_host_arch
3492 sub get_host_arch() {
3493     $host_arch = (POSIX::uname())[4] if !$host_arch;
3494     return $host_arch;
3495 }
3496
3497 sub is_native($) {
3498     my ($arch) = @_;
3499     return get_host_arch() eq $arch;
3500 }
3501
3502 my $default_machines = {
3503     x86_64 => 'pc',
3504     aarch64 => 'virt',
3505 };
3506
3507 sub get_basic_machine_info {
3508     my ($conf, $forcemachine) = @_;
3509
3510     my $arch = $conf->{arch} // get_host_arch();
3511     my $machine = $forcemachine || $conf->{machine} || $default_machines->{$arch};
3512     return ($arch, $machine);
3513 }
3514
3515 sub get_ovmf_files($) {
3516     my ($arch) = @_;
3517
3518     my $ovmf = $OVMF->{$arch}
3519         or die "no OVMF images known for architecture '$arch'\n";
3520
3521     return @$ovmf;
3522 }
3523
3524 my $Arch2Qemu = {
3525     aarch64 => '/usr/bin/qemu-system-aarch64',
3526     x86_64 => '/usr/bin/qemu-system-x86_64',
3527 };
3528 sub get_command_for_arch($) {
3529     my ($arch) = @_;
3530     return '/usr/bin/kvm' if is_native($arch);
3531
3532     my $cmd = $Arch2Qemu->{$arch}
3533         or die "don't know how to emulate architecture '$arch'\n";
3534     return $cmd;
3535 }
3536
3537 sub get_cpu_options {
3538     my ($conf, $arch, $kvm, $machine_type, $kvm_off, $kvmver, $winversion, $gpu_passthrough) = @_;
3539
3540     my $cpuFlags = [];
3541     my $ostype = $conf->{ostype};
3542
3543     my $cpu = $kvm ? "kvm64" : "qemu64";
3544     if ($arch eq 'aarch64') {
3545         $cpu = 'cortex-a57';
3546     }
3547     my $hv_vendor_id;
3548     if (my $cputype = $conf->{cpu}) {
3549         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
3550             or die "Cannot parse cpu description: $cputype\n";
3551         $cpu = $cpuconf->{cputype};
3552         $kvm_off = 1 if $cpuconf->{hidden};
3553         $hv_vendor_id = $cpuconf->{'hv-vendor-id'};
3554
3555         if (defined(my $flags = $cpuconf->{flags})) {
3556             push @$cpuFlags, split(";", $flags);
3557         }
3558     }
3559
3560     push @$cpuFlags , '+lahf_lm' if $cpu eq 'kvm64' && $arch eq 'x86_64';
3561
3562     push @$cpuFlags , '-x2apic'
3563         if $conf->{ostype} && $conf->{ostype} eq 'solaris';
3564
3565     push @$cpuFlags, '+sep' if $cpu eq 'kvm64' || $cpu eq 'kvm32';
3566
3567     push @$cpuFlags, '-rdtscp' if $cpu =~ m/^Opteron/;
3568
3569     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3) && $arch eq 'x86_64') {
3570
3571         push @$cpuFlags , '+kvm_pv_unhalt' if $kvm;
3572         push @$cpuFlags , '+kvm_pv_eoi' if $kvm;
3573     }
3574
3575     add_hyperv_enlightenments($cpuFlags, $winversion, $machine_type, $kvmver, $conf->{bios}, $gpu_passthrough, $hv_vendor_id) if $kvm;
3576
3577     push @$cpuFlags, 'enforce' if $cpu ne 'host' && $kvm && $arch eq 'x86_64';
3578
3579     push @$cpuFlags, 'kvm=off' if $kvm_off;
3580
3581     if (my $cpu_vendor = $cpu_vendor_list->{$cpu}) {
3582         push @$cpuFlags, "vendor=${cpu_vendor}"
3583             if $cpu_vendor ne 'default';
3584     } elsif ($arch ne 'aarch64') {
3585         die "internal error"; # should not happen
3586     }
3587
3588     $cpu .= "," . join(',', @$cpuFlags) if scalar(@$cpuFlags);
3589
3590     return ('-cpu', $cpu);
3591 }
3592
3593 sub config_to_command {
3594     my ($storecfg, $vmid, $conf, $defaults, $forcemachine) = @_;
3595
3596     my $cmd = [];
3597     my $globalFlags = [];
3598     my $machineFlags = [];
3599     my $rtcFlags = [];
3600     my $devices = [];
3601     my $pciaddr = '';
3602     my $bridges = {};
3603     my $vernum = 0; # unknown
3604     my $ostype = $conf->{ostype};
3605     my $winversion = windows_version($ostype);
3606     my $kvm = $conf->{kvm};
3607
3608     my ($arch, $machine_type) = get_basic_machine_info($conf, $forcemachine);
3609     my $kvm_binary = get_command_for_arch($arch);
3610     my $kvmver = kvm_user_version($kvm_binary);
3611     $kvm //= 1 if is_native($arch);
3612
3613     if ($kvm) {
3614         die "KVM virtualisation configured, but not available. Either disable in VM configuration or enable in BIOS.\n"
3615             if !defined kvm_version();
3616     }
3617
3618     if ($kvmver =~ m/^(\d+)\.(\d+)$/) {
3619         $vernum = $1*1000000+$2*1000;
3620     } elsif ($kvmver =~ m/^(\d+)\.(\d+)\.(\d+)$/) {
3621         $vernum = $1*1000000+$2*1000+$3;
3622     }
3623
3624     die "detected old qemu-kvm binary ($kvmver)\n" if $vernum < 15000;
3625
3626     my $q35 = machine_type_is_q35($conf);
3627     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
3628     my $use_old_bios_files = undef;
3629     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3630
3631     my $cpuunits = defined($conf->{cpuunits}) ?
3632             $conf->{cpuunits} : $defaults->{cpuunits};
3633
3634     push @$cmd, $kvm_binary;
3635
3636     push @$cmd, '-id', $vmid;
3637
3638     my $vmname = $conf->{name} || "vm$vmid";
3639
3640     push @$cmd, '-name', $vmname;
3641
3642     my $use_virtio = 0;
3643
3644     my $qmpsocket = qmp_socket($vmid);
3645     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server,nowait";
3646     push @$cmd, '-mon', "chardev=qmp,mode=control";
3647
3648     if (qemu_machine_feature_enabled($machine_type, $kvmver, 2, 12)) {
3649         push @$cmd, '-chardev', "socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5";
3650         push @$cmd, '-mon', "chardev=qmp-event,mode=control";
3651     }
3652
3653     push @$cmd, '-pidfile' , pidfile_name($vmid);
3654
3655     push @$cmd, '-daemonize';
3656
3657     if ($conf->{smbios1}) {
3658         my $smbios_conf = parse_smbios1($conf->{smbios1});
3659         if ($smbios_conf->{base64}) {
3660             # Do not pass base64 flag to qemu
3661             delete $smbios_conf->{base64};
3662             my $smbios_string = "";
3663             foreach my $key (keys %$smbios_conf) {
3664                 my $value;
3665                 if ($key eq "uuid") {
3666                     $value = $smbios_conf->{uuid}
3667                 } else {
3668                     $value = decode_base64($smbios_conf->{$key});
3669                 }
3670                 # qemu accepts any binary data, only commas need escaping by double comma
3671                 $value =~ s/,/,,/g;
3672                 $smbios_string .= "," . $key . "=" . $value if $value;
3673             }
3674             push @$cmd, '-smbios', "type=1" . $smbios_string;
3675         } else {
3676             push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
3677         }
3678     }
3679
3680     if ($conf->{vmgenid}) {
3681         push @$devices, '-device', 'vmgenid,guid='.$conf->{vmgenid};
3682     }
3683
3684     my ($ovmf_code, $ovmf_vars) = get_ovmf_files($arch);
3685     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3686         die "uefi base image not found\n" if ! -f $ovmf_code;
3687
3688         my $path;
3689         my $format;
3690         if (my $efidisk = $conf->{efidisk0}) {
3691             my $d = PVE::JSONSchema::parse_property_string($efidisk_fmt, $efidisk);
3692             my ($storeid, $volname) = PVE::Storage::parse_volume_id($d->{file}, 1);
3693             $format = $d->{format};
3694             if ($storeid) {
3695                 $path = PVE::Storage::path($storecfg, $d->{file});
3696                 if (!defined($format)) {
3697                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
3698                     $format = qemu_img_format($scfg, $volname);
3699                 }
3700             } else {
3701                 $path = $d->{file};
3702                 die "efidisk format must be specified\n"
3703                     if !defined($format);
3704             }
3705         } else {
3706             warn "no efidisk configured! Using temporary efivars disk.\n";
3707             $path = "/tmp/$vmid-ovmf.fd";
3708             PVE::Tools::file_copy($ovmf_vars, $path, -s $ovmf_vars);
3709             $format = 'raw';
3710         }
3711
3712         push @$cmd, '-drive', "if=pflash,unit=0,format=raw,readonly,file=$ovmf_code";
3713         push @$cmd, '-drive', "if=pflash,unit=1,format=$format,id=drive-efidisk0,file=$path";
3714     }
3715
3716     # load q35 config
3717     if ($q35) {
3718         # we use different pcie-port hardware for qemu >= 4.0 for passthrough
3719         if (qemu_machine_feature_enabled($machine_type, $kvmver, 4, 0)) {
3720             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35-4.0.cfg';
3721         } else {
3722             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35.cfg';
3723         }
3724     }
3725
3726     # add usb controllers
3727     my @usbcontrollers = PVE::QemuServer::USB::get_usb_controllers($conf, $bridges, $arch, $machine_type, $usbdesc->{format}, $MAX_USB_DEVICES);
3728     push @$devices, @usbcontrollers if @usbcontrollers;
3729     my $vga = parse_vga($conf->{vga});
3730
3731     my $qxlnum = vga_conf_has_spice($conf->{vga});
3732     $vga->{type} = 'qxl' if $qxlnum;
3733
3734     if (!$vga->{type}) {
3735         if ($arch eq 'aarch64') {
3736             $vga->{type} = 'virtio';
3737         } elsif (qemu_machine_feature_enabled($machine_type, $kvmver, 2, 9)) {
3738             $vga->{type} = (!$winversion || $winversion >= 6) ? 'std' : 'cirrus';
3739         } else {
3740             $vga->{type} = ($winversion >= 6) ? 'std' : 'cirrus';
3741         }
3742     }
3743
3744     # enable absolute mouse coordinates (needed by vnc)
3745     my $tablet;
3746     if (defined($conf->{tablet})) {
3747         $tablet = $conf->{tablet};
3748     } else {
3749         $tablet = $defaults->{tablet};
3750         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
3751         $tablet = 0 if $vga->{type} =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
3752     }
3753
3754     if ($tablet) {
3755         push @$devices, '-device', print_tabletdevice_full($conf, $arch) if $tablet;
3756         my $kbd = print_keyboarddevice_full($conf, $arch);
3757         push @$devices, '-device', $kbd if defined($kbd);
3758     }
3759
3760     my $kvm_off = 0;
3761     my $gpu_passthrough;
3762
3763     # host pci devices
3764     for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
3765         my $id = "hostpci$i";
3766         my $d = parse_hostpci($conf->{$id});
3767         next if !$d;
3768
3769         if (my $pcie = $d->{pcie}) {
3770             die "q35 machine model is not enabled" if !$q35;
3771             # win7 wants to have the pcie devices directly on the pcie bus
3772             # instead of in the root port
3773             if ($winversion == 7) {
3774                 $pciaddr = print_pcie_addr("${id}bus0");
3775             } else {
3776                 # add more root ports if needed, 4 are present by default
3777                 # by pve-q35 cfgs, rest added here on demand.
3778                 if ($i > 3) {
3779                     push @$devices, '-device', print_pcie_root_port($i);
3780                 }
3781                 $pciaddr = print_pcie_addr($id);
3782             }
3783         } else {
3784             $pciaddr = print_pci_addr($id, $bridges, $arch, $machine_type);
3785         }
3786
3787         my $xvga = '';
3788         if ($d->{'x-vga'}) {
3789             $xvga = ',x-vga=on' if !($conf->{bios} && $conf->{bios} eq 'ovmf');
3790             $kvm_off = 1;
3791             $vga->{type} = 'none' if !defined($conf->{vga});
3792             $gpu_passthrough = 1;
3793         }
3794
3795         my $pcidevices = $d->{pciid};
3796         my $multifunction = 1 if @$pcidevices > 1;
3797
3798         my $sysfspath;
3799         if ($d->{mdev} && scalar(@$pcidevices) == 1) {
3800             my $pci_id = $pcidevices->[0]->{id};
3801             my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $i);
3802             $sysfspath = "/sys/bus/pci/devices/0000:$pci_id/$uuid";
3803         } elsif ($d->{mdev}) {
3804             warn "ignoring mediated device '$id' with multifunction device\n";
3805         }
3806
3807         my $j=0;
3808         foreach my $pcidevice (@$pcidevices) {
3809             my $devicestr = "vfio-pci";
3810
3811             if ($sysfspath) {
3812                 $devicestr .= ",sysfsdev=$sysfspath";
3813             } else {
3814                 $devicestr .= ",host=$pcidevice->{id}";
3815             }
3816
3817             my $mf_addr = $multifunction ? ".$j" : '';
3818             $devicestr .= ",id=${id}${mf_addr}${pciaddr}${mf_addr}";
3819
3820             if ($j == 0) {
3821                 $devicestr .= ',rombar=0' if defined($d->{rombar}) && !$d->{rombar};
3822                 $devicestr .= "$xvga";
3823                 $devicestr .= ",multifunction=on" if $multifunction;
3824                 $devicestr .= ",romfile=/usr/share/kvm/$d->{romfile}" if $d->{romfile};
3825             }
3826
3827             push @$devices, '-device', $devicestr;
3828             $j++;
3829         }
3830     }
3831
3832     # usb devices
3833     my $usb_dev_features = {};
3834     $usb_dev_features->{spice_usb3} = 1 if qemu_machine_feature_enabled($machine_type, $kvmver, 4, 0);
3835
3836     my @usbdevices = PVE::QemuServer::USB::get_usb_devices($conf, $usbdesc->{format}, $MAX_USB_DEVICES, $usb_dev_features);
3837     push @$devices, @usbdevices if @usbdevices;
3838     # serial devices
3839     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3840         if (my $path = $conf->{"serial$i"}) {
3841             if ($path eq 'socket') {
3842                 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
3843                 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server,nowait";
3844                 # On aarch64, serial0 is the UART device. Qemu only allows
3845                 # connecting UART devices via the '-serial' command line, as
3846                 # the device has a fixed slot on the hardware...
3847                 if ($arch eq 'aarch64' && $i == 0) {
3848                     push @$devices, '-serial', "chardev:serial$i";
3849                 } else {
3850                     push @$devices, '-device', "isa-serial,chardev=serial$i";
3851                 }
3852             } else {
3853                 die "no such serial device\n" if ! -c $path;
3854                 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
3855                 push @$devices, '-device', "isa-serial,chardev=serial$i";
3856             }
3857         }
3858     }
3859
3860     # parallel devices
3861     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
3862         if (my $path = $conf->{"parallel$i"}) {
3863             die "no such parallel device\n" if ! -c $path;
3864             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
3865             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
3866             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
3867         }
3868     }
3869
3870     if (my $audio = conf_has_audio($conf)) {
3871
3872         my $audiopciaddr = print_pci_addr("audio0", $bridges, $arch, $machine_type);
3873
3874         my $id = $audio->{dev_id};
3875         if ($audio->{dev} eq 'AC97') {
3876             push @$devices, '-device', "AC97,id=${id}${audiopciaddr}";
3877         } elsif ($audio->{dev} =~ /intel\-hda$/) {
3878             push @$devices, '-device', "$audio->{dev},id=${id}${audiopciaddr}";
3879             push @$devices, '-device', "hda-micro,id=${id}-codec0,bus=${id}.0,cad=0";
3880             push @$devices, '-device', "hda-duplex,id=${id}-codec1,bus=${id}.0,cad=1";
3881         } else {
3882             die "unkown audio device '$audio->{dev}', implement me!";
3883         }
3884
3885         push @$devices, '-audiodev', "$audio->{backend},id=$audio->{backend_id}";
3886     }
3887
3888     my $sockets = 1;
3889     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3890     $sockets = $conf->{sockets} if  $conf->{sockets};
3891
3892     my $cores = $conf->{cores} || 1;
3893
3894     my $maxcpus = $sockets * $cores;
3895
3896     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
3897
3898     my $allowed_vcpus = $cpuinfo->{cpus};
3899
3900     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n"
3901         if ($allowed_vcpus < $maxcpus);
3902
3903     if($hotplug_features->{cpu} && qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 7)) {
3904
3905         push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3906         for (my $i = 2; $i <= $vcpus; $i++)  {
3907             my $cpustr = print_cpu_device($conf,$i);
3908             push @$cmd, '-device', $cpustr;
3909         }
3910
3911     } else {
3912
3913         push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3914     }
3915     push @$cmd, '-nodefaults';
3916
3917     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
3918
3919     my $bootindex_hash = {};
3920     my $i = 1;
3921     foreach my $o (split(//, $bootorder)) {
3922         $bootindex_hash->{$o} = $i*100;
3923         $i++;
3924     }
3925
3926     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
3927
3928     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
3929
3930     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
3931
3932     if ($vga->{type} && $vga->{type} !~ m/^serial\d+$/ && $vga->{type} ne 'none'){
3933         push @$devices, '-device', print_vga_device($conf, $vga, $arch, $machine_type, undef, $qxlnum, $bridges);
3934         my $socket = vnc_socket($vmid);
3935         push @$cmd,  '-vnc', "unix:$socket,password";
3936     } else {
3937         push @$cmd, '-vga', 'none' if $vga->{type} eq 'none';
3938         push @$cmd, '-nographic';
3939     }
3940
3941     # time drift fix
3942     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
3943
3944     my $useLocaltime = $conf->{localtime};
3945
3946     if ($winversion >= 5) { # windows
3947         $useLocaltime = 1 if !defined($conf->{localtime});
3948
3949         # use time drift fix when acpi is enabled
3950         if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
3951             $tdf = 1 if !defined($conf->{tdf});
3952         }
3953     }
3954
3955     if ($winversion >= 6) {
3956         push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3957         push @$cmd, '-no-hpet';
3958     }
3959
3960     push @$rtcFlags, 'driftfix=slew' if $tdf;
3961
3962     if (!$kvm) {
3963         push @$machineFlags, 'accel=tcg';
3964     }
3965
3966     if ($machine_type) {
3967         push @$machineFlags, "type=${machine_type}";
3968     }
3969
3970     if (($conf->{startdate}) && ($conf->{startdate} ne 'now')) {
3971         push @$rtcFlags, "base=$conf->{startdate}";
3972     } elsif ($useLocaltime) {
3973         push @$rtcFlags, 'base=localtime';
3974     }
3975
3976     push @$cmd, get_cpu_options($conf, $arch, $kvm, $machine_type, $kvm_off, $kvmver, $winversion, $gpu_passthrough);
3977
3978     PVE::QemuServer::Memory::config($conf, $vmid, $sockets, $cores, $defaults, $hotplug_features, $cmd);
3979
3980     push @$cmd, '-S' if $conf->{freeze};
3981
3982     push @$cmd, '-k', $conf->{keyboard} if defined($conf->{keyboard});
3983
3984     if (parse_guest_agent($conf)->{enabled}) {
3985         my $qgasocket = qmp_socket($vmid, 1);
3986         my $pciaddr = print_pci_addr("qga0", $bridges, $arch, $machine_type);
3987         push @$devices, '-chardev', "socket,path=$qgasocket,server,nowait,id=qga0";
3988         push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3989         push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3990     }
3991
3992     my $spice_port;
3993
3994     if ($qxlnum) {
3995         if ($qxlnum > 1) {
3996             if ($winversion){
3997                 for(my $i = 1; $i < $qxlnum; $i++){
3998                     push @$devices, '-device', print_vga_device($conf, $vga, $arch, $machine_type, $i, $qxlnum, $bridges);
3999                 }
4000             } else {
4001                 # assume other OS works like Linux
4002                 my ($ram, $vram) = ("134217728", "67108864");
4003                 if ($vga->{memory}) {
4004                     $ram = PVE::Tools::convert_size($qxlnum*4*$vga->{memory}, 'mb' => 'b');
4005                     $vram = PVE::Tools::convert_size($qxlnum*2*$vga->{memory}, 'mb' => 'b');
4006                 }
4007                 push @$cmd, '-global', "qxl-vga.ram_size=$ram";
4008                 push @$cmd, '-global', "qxl-vga.vram_size=$vram";
4009             }
4010         }
4011
4012         my $pciaddr = print_pci_addr("spice", $bridges, $arch, $machine_type);
4013
4014         my $nodename = PVE::INotify::nodename();
4015         my $pfamily = PVE::Tools::get_host_address_family($nodename);
4016         my @nodeaddrs = PVE::Tools::getaddrinfo_all('localhost', family => $pfamily);
4017         die "failed to get an ip address of type $pfamily for 'localhost'\n" if !@nodeaddrs;
4018
4019         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
4020         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
4021         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
4022
4023         my $localhost = PVE::Network::addr_to_ip($nodeaddrs[0]->{addr});
4024         $spice_port = PVE::Tools::next_spice_port($pfamily, $localhost);
4025
4026         my $spice_enhancement = PVE::JSONSchema::parse_property_string($spice_enhancements_fmt, $conf->{spice_enhancements} // '');
4027         if ($spice_enhancement->{foldersharing}) {
4028             push @$devices, '-chardev', "spiceport,id=foldershare,name=org.spice-space.webdav.0";
4029             push @$devices, '-device', "virtserialport,chardev=foldershare,name=org.spice-space.webdav.0";
4030         }
4031
4032         my $spice_opts = "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";
4033         $spice_opts .= ",streaming-video=$spice_enhancement->{videostreaming}" if $spice_enhancement->{videostreaming};
4034         push @$devices, '-spice', "$spice_opts";
4035     }
4036
4037     # enable balloon by default, unless explicitly disabled
4038     if (!defined($conf->{balloon}) || $conf->{balloon}) {
4039         $pciaddr = print_pci_addr("balloon0", $bridges, $arch, $machine_type);
4040         push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
4041     }
4042
4043     if ($conf->{watchdog}) {
4044         my $wdopts = parse_watchdog($conf->{watchdog});
4045         $pciaddr = print_pci_addr("watchdog", $bridges, $arch, $machine_type);
4046         my $watchdog = $wdopts->{model} || 'i6300esb';
4047         push @$devices, '-device', "$watchdog$pciaddr";
4048         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
4049     }
4050
4051     my $vollist = [];
4052     my $scsicontroller = {};
4053     my $ahcicontroller = {};
4054     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
4055
4056     # Add iscsi initiator name if available
4057     if (my $initiator = get_initiator_name()) {
4058         push @$devices, '-iscsi', "initiator-name=$initiator";
4059     }
4060
4061     foreach_drive($conf, sub {
4062         my ($ds, $drive) = @_;
4063
4064         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
4065             push @$vollist, $drive->{file};
4066         }
4067
4068         # ignore efidisk here, already added in bios/fw handling code above
4069         return if $drive->{interface} eq 'efidisk';
4070
4071         $use_virtio = 1 if $ds =~ m/^virtio/;
4072
4073         if (drive_is_cdrom ($drive)) {
4074             if ($bootindex_hash->{d}) {
4075                 $drive->{bootindex} = $bootindex_hash->{d};
4076                 $bootindex_hash->{d} += 1;
4077             }
4078         } else {
4079             if ($bootindex_hash->{c}) {
4080                 $drive->{bootindex} = $bootindex_hash->{c} if $conf->{bootdisk} && ($conf->{bootdisk} eq $ds);
4081                 $bootindex_hash->{c} += 1;
4082             }
4083         }
4084
4085         if($drive->{interface} eq 'virtio'){
4086            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
4087         }
4088
4089         if ($drive->{interface} eq 'scsi') {
4090
4091             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
4092
4093             $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges, $arch, $machine_type);
4094             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
4095
4096             my $iothread = '';
4097             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
4098                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
4099                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
4100             } elsif ($drive->{iothread}) {
4101                 warn "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n";
4102             }
4103
4104             my $queues = '';
4105             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
4106                 $queues = ",num_queues=$drive->{queues}";
4107             }
4108
4109             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues" if !$scsicontroller->{$controller};
4110             $scsicontroller->{$controller}=1;
4111         }
4112
4113         if ($drive->{interface} eq 'sata') {
4114            my $controller = int($drive->{index} / $MAX_SATA_DISKS);
4115            $pciaddr = print_pci_addr("ahci$controller", $bridges, $arch, $machine_type);
4116            push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr" if !$ahcicontroller->{$controller};
4117            $ahcicontroller->{$controller}=1;
4118         }
4119
4120         my $drive_cmd = print_drive_full($storecfg, $vmid, $drive);
4121         push @$devices, '-drive',$drive_cmd;
4122         push @$devices, '-device', print_drivedevice_full($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type);
4123     });
4124
4125     for (my $i = 0; $i < $MAX_NETS; $i++) {
4126          next if !$conf->{"net$i"};
4127          my $d = parse_net($conf->{"net$i"});
4128          next if !$d;
4129
4130          $use_virtio = 1 if $d->{model} eq 'virtio';
4131
4132          if ($bootindex_hash->{n}) {
4133             $d->{bootindex} = $bootindex_hash->{n};
4134             $bootindex_hash->{n} += 1;
4135          }
4136
4137          my $netdevfull = print_netdev_full($vmid, $conf, $arch, $d, "net$i");
4138          push @$devices, '-netdev', $netdevfull;
4139
4140          my $netdevicefull = print_netdevice_full($vmid, $conf, $d, "net$i", $bridges, $use_old_bios_files, $arch, $machine_type);
4141          push @$devices, '-device', $netdevicefull;
4142     }
4143
4144     if ($conf->{ivshmem}) {
4145         my $ivshmem = PVE::JSONSchema::parse_property_string($ivshmem_fmt, $conf->{ivshmem});
4146
4147         my $bus;
4148         if ($q35) {
4149             $bus = print_pcie_addr("ivshmem");
4150         } else {
4151             $bus = print_pci_addr("ivshmem", $bridges, $arch, $machine_type);
4152         }
4153
4154         my $ivshmem_name = $ivshmem->{name} // $vmid;
4155         my $path = '/dev/shm/pve-shm-' . $ivshmem_name;
4156
4157         push @$devices, '-device', "ivshmem-plain,memdev=ivshmem$bus,";
4158         push @$devices, '-object', "memory-backend-file,id=ivshmem,share=on,mem-path=$path,size=$ivshmem->{size}M";
4159     }
4160
4161     if (!$q35) {
4162         # add pci bridges
4163         if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
4164            $bridges->{1} = 1;
4165            $bridges->{2} = 1;
4166         }
4167
4168         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
4169
4170         for my $k (sort {$b cmp $a} keys %$bridges) {
4171             $pciaddr = print_pci_addr("pci.$k", undef, $arch, $machine_type);
4172             unshift @$devices, '-device', "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr" if $k > 0;
4173         }
4174     }
4175
4176     push @$cmd, @$devices;
4177     push @$cmd, '-rtc', join(',', @$rtcFlags)
4178         if scalar(@$rtcFlags);
4179     push @$cmd, '-machine', join(',', @$machineFlags)
4180         if scalar(@$machineFlags);
4181     push @$cmd, '-global', join(',', @$globalFlags)
4182         if scalar(@$globalFlags);
4183
4184     if (my $vmstate = $conf->{vmstate}) {
4185         my $statepath = PVE::Storage::path($storecfg, $vmstate);
4186         push @$vollist, $statepath;
4187         push @$cmd, '-loadstate', $statepath;
4188     }
4189
4190     # add custom args
4191     if ($conf->{args}) {
4192         my $aa = PVE::Tools::split_args($conf->{args});
4193         push @$cmd, @$aa;
4194     }
4195
4196     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
4197 }
4198
4199 sub vnc_socket {
4200     my ($vmid) = @_;
4201     return "${var_run_tmpdir}/$vmid.vnc";
4202 }
4203
4204 sub spice_port {
4205     my ($vmid) = @_;
4206
4207     my $res = vm_mon_cmd($vmid, 'query-spice');
4208
4209     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
4210 }
4211
4212 sub qmp_socket {
4213     my ($vmid, $qga, $name) = @_;
4214     my $sockettype = $qga ? 'qga' : 'qmp';
4215     my $ext = $name ? '-'.$name : '';
4216     return "${var_run_tmpdir}/$vmid$ext.$sockettype";
4217 }
4218
4219 sub pidfile_name {
4220     my ($vmid) = @_;
4221     return "${var_run_tmpdir}/$vmid.pid";
4222 }
4223
4224 sub vm_devices_list {
4225     my ($vmid) = @_;
4226
4227     my $res = vm_mon_cmd($vmid, 'query-pci');
4228     my $devices_to_check = [];
4229     my $devices = {};
4230     foreach my $pcibus (@$res) {
4231         push @$devices_to_check, @{$pcibus->{devices}},
4232     }
4233
4234     while (@$devices_to_check) {
4235         my $to_check = [];
4236         for my $d (@$devices_to_check) {
4237             $devices->{$d->{'qdev_id'}} = 1 if $d->{'qdev_id'};
4238             next if !$d->{'pci_bridge'};
4239
4240             $devices->{$d->{'qdev_id'}} += scalar(@{$d->{'pci_bridge'}->{devices}});
4241             push @$to_check, @{$d->{'pci_bridge'}->{devices}};
4242         }
4243         $devices_to_check = $to_check;
4244     }
4245
4246     my $resblock = vm_mon_cmd($vmid, 'query-block');
4247     foreach my $block (@$resblock) {
4248         if($block->{device} =~ m/^drive-(\S+)/){
4249                 $devices->{$1} = 1;
4250         }
4251     }
4252
4253     my $resmice = vm_mon_cmd($vmid, 'query-mice');
4254     foreach my $mice (@$resmice) {
4255         if ($mice->{name} eq 'QEMU HID Tablet') {
4256             $devices->{tablet} = 1;
4257             last;
4258         }
4259     }
4260
4261     # for usb devices there is no query-usb
4262     # but we can iterate over the entries in
4263     # qom-list path=/machine/peripheral
4264     my $resperipheral = vm_mon_cmd($vmid, 'qom-list', path => '/machine/peripheral');
4265     foreach my $per (@$resperipheral) {
4266         if ($per->{name} =~ m/^usb\d+$/) {
4267             $devices->{$per->{name}} = 1;
4268         }
4269     }
4270
4271     return $devices;
4272 }
4273
4274 sub vm_deviceplug {
4275     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4276
4277     my $q35 = machine_type_is_q35($conf);
4278
4279     my $devices_list = vm_devices_list($vmid);
4280     return 1 if defined($devices_list->{$deviceid});
4281
4282     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid, $arch, $machine_type); # add PCI bridge if we need it for the device
4283
4284     if ($deviceid eq 'tablet') {
4285
4286         qemu_deviceadd($vmid, print_tabletdevice_full($conf, $arch));
4287
4288     } elsif ($deviceid eq 'keyboard') {
4289
4290         qemu_deviceadd($vmid, print_keyboarddevice_full($conf, $arch));
4291
4292     } elsif ($deviceid =~ m/^usb(\d+)$/) {
4293
4294         die "usb hotplug currently not reliable\n";
4295         # since we can't reliably hot unplug all added usb devices
4296         # and usb passthrough disables live migration
4297         # we disable usb hotplugging for now
4298         qemu_deviceadd($vmid, PVE::QemuServer::USB::print_usbdevice_full($conf, $deviceid, $device));
4299
4300     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4301
4302         qemu_iothread_add($vmid, $deviceid, $device);
4303
4304         qemu_driveadd($storecfg, $vmid, $device);
4305         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, $arch, $machine_type);
4306
4307         qemu_deviceadd($vmid, $devicefull);
4308         eval { qemu_deviceaddverify($vmid, $deviceid); };
4309         if (my $err = $@) {
4310             eval { qemu_drivedel($vmid, $deviceid); };
4311             warn $@ if $@;
4312             die $err;
4313         }
4314
4315     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4316
4317
4318         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
4319         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
4320         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
4321
4322         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
4323
4324         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
4325             qemu_iothread_add($vmid, $deviceid, $device);
4326             $devicefull .= ",iothread=iothread-$deviceid";
4327         }
4328
4329         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
4330             $devicefull .= ",num_queues=$device->{queues}";
4331         }
4332
4333         qemu_deviceadd($vmid, $devicefull);
4334         qemu_deviceaddverify($vmid, $deviceid);
4335
4336     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4337
4338         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device, $arch, $machine_type);
4339         qemu_driveadd($storecfg, $vmid, $device);
4340
4341         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, $arch, $machine_type);
4342         eval { qemu_deviceadd($vmid, $devicefull); };
4343         if (my $err = $@) {
4344             eval { qemu_drivedel($vmid, $deviceid); };
4345             warn $@ if $@;
4346             die $err;
4347         }
4348
4349     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4350
4351         return undef if !qemu_netdevadd($vmid, $conf, $arch, $device, $deviceid);
4352
4353         my $machine_type = PVE::QemuServer::qemu_machine_pxe($vmid, $conf);
4354         my $use_old_bios_files = undef;
4355         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
4356
4357         my $netdevicefull = print_netdevice_full($vmid, $conf, $device, $deviceid, undef, $use_old_bios_files, $arch, $machine_type);
4358         qemu_deviceadd($vmid, $netdevicefull);
4359         eval {
4360             qemu_deviceaddverify($vmid, $deviceid);
4361             qemu_set_link_status($vmid, $deviceid, !$device->{link_down});
4362         };
4363         if (my $err = $@) {
4364             eval { qemu_netdevdel($vmid, $deviceid); };
4365             warn $@ if $@;
4366             die $err;
4367         }
4368
4369     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
4370
4371         my $bridgeid = $2;
4372         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
4373         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
4374
4375         qemu_deviceadd($vmid, $devicefull);
4376         qemu_deviceaddverify($vmid, $deviceid);
4377
4378     } else {
4379         die "can't hotplug device '$deviceid'\n";
4380     }
4381
4382     return 1;
4383 }
4384
4385 # fixme: this should raise exceptions on error!
4386 sub vm_deviceunplug {
4387     my ($vmid, $conf, $deviceid) = @_;
4388
4389     my $devices_list = vm_devices_list($vmid);
4390     return 1 if !defined($devices_list->{$deviceid});
4391
4392     die "can't unplug bootdisk" if $conf->{bootdisk} && $conf->{bootdisk} eq $deviceid;
4393
4394     if ($deviceid eq 'tablet' || $deviceid eq 'keyboard') {
4395
4396         qemu_devicedel($vmid, $deviceid);
4397
4398     } elsif ($deviceid =~ m/^usb\d+$/) {
4399
4400         die "usb hotplug currently not reliable\n";
4401         # when unplugging usb devices this way,
4402         # there may be remaining usb controllers/hubs
4403         # so we disable it for now
4404         qemu_devicedel($vmid, $deviceid);
4405         qemu_devicedelverify($vmid, $deviceid);
4406
4407     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4408
4409         qemu_devicedel($vmid, $deviceid);
4410         qemu_devicedelverify($vmid, $deviceid);
4411         qemu_drivedel($vmid, $deviceid);
4412         qemu_iothread_del($conf, $vmid, $deviceid);
4413
4414     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4415
4416         qemu_devicedel($vmid, $deviceid);
4417         qemu_devicedelverify($vmid, $deviceid);
4418         qemu_iothread_del($conf, $vmid, $deviceid);
4419
4420     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4421
4422         qemu_devicedel($vmid, $deviceid);
4423         qemu_drivedel($vmid, $deviceid);
4424         qemu_deletescsihw($conf, $vmid, $deviceid);
4425
4426     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4427
4428         qemu_devicedel($vmid, $deviceid);
4429         qemu_devicedelverify($vmid, $deviceid);
4430         qemu_netdevdel($vmid, $deviceid);
4431
4432     } else {
4433         die "can't unplug device '$deviceid'\n";
4434     }
4435
4436     return 1;
4437 }
4438
4439 sub qemu_deviceadd {
4440     my ($vmid, $devicefull) = @_;
4441
4442     $devicefull = "driver=".$devicefull;
4443     my %options =  split(/[=,]/, $devicefull);
4444
4445     vm_mon_cmd($vmid, "device_add" , %options);
4446 }
4447
4448 sub qemu_devicedel {
4449     my ($vmid, $deviceid) = @_;
4450
4451     my $ret = vm_mon_cmd($vmid, "device_del", id => $deviceid);
4452 }
4453
4454 sub qemu_iothread_add {
4455     my($vmid, $deviceid, $device) = @_;
4456
4457     if ($device->{iothread}) {
4458         my $iothreads = vm_iothreads_list($vmid);
4459         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
4460     }
4461 }
4462
4463 sub qemu_iothread_del {
4464     my($conf, $vmid, $deviceid) = @_;
4465
4466     my $confid = $deviceid;
4467     if ($deviceid =~ m/^(?:virtioscsi|scsihw)(\d+)$/) {
4468         $confid = 'scsi' . $1;
4469     }
4470     my $device = parse_drive($confid, $conf->{$confid});
4471     if ($device->{iothread}) {
4472         my $iothreads = vm_iothreads_list($vmid);
4473         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
4474     }
4475 }
4476
4477 sub qemu_objectadd {
4478     my($vmid, $objectid, $qomtype) = @_;
4479
4480     vm_mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
4481
4482     return 1;
4483 }
4484
4485 sub qemu_objectdel {
4486     my($vmid, $objectid) = @_;
4487
4488     vm_mon_cmd($vmid, "object-del", id => $objectid);
4489
4490     return 1;
4491 }
4492
4493 sub qemu_driveadd {
4494     my ($storecfg, $vmid, $device) = @_;
4495
4496     my $drive = print_drive_full($storecfg, $vmid, $device);
4497     $drive =~ s/\\/\\\\/g;
4498     my $ret = vm_human_monitor_command($vmid, "drive_add auto \"$drive\"");
4499
4500     # If the command succeeds qemu prints: "OK"
4501     return 1 if $ret =~ m/OK/s;
4502
4503     die "adding drive failed: $ret\n";
4504 }
4505
4506 sub qemu_drivedel {
4507     my($vmid, $deviceid) = @_;
4508
4509     my $ret = vm_human_monitor_command($vmid, "drive_del drive-$deviceid");
4510     $ret =~ s/^\s+//;
4511
4512     return 1 if $ret eq "";
4513
4514     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
4515     return 1 if $ret =~ m/Device \'.*?\' not found/s;
4516
4517     die "deleting drive $deviceid failed : $ret\n";
4518 }
4519
4520 sub qemu_deviceaddverify {
4521     my ($vmid, $deviceid) = @_;
4522
4523     for (my $i = 0; $i <= 5; $i++) {
4524          my $devices_list = vm_devices_list($vmid);
4525          return 1 if defined($devices_list->{$deviceid});
4526          sleep 1;
4527     }
4528
4529     die "error on hotplug device '$deviceid'\n";
4530 }
4531
4532
4533 sub qemu_devicedelverify {
4534     my ($vmid, $deviceid) = @_;
4535
4536     # need to verify that the device is correctly removed as device_del
4537     # is async and empty return is not reliable
4538
4539     for (my $i = 0; $i <= 5; $i++) {
4540          my $devices_list = vm_devices_list($vmid);
4541          return 1 if !defined($devices_list->{$deviceid});
4542          sleep 1;
4543     }
4544
4545     die "error on hot-unplugging device '$deviceid'\n";
4546 }
4547
4548 sub qemu_findorcreatescsihw {
4549     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4550
4551     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4552
4553     my $scsihwid="$controller_prefix$controller";
4554     my $devices_list = vm_devices_list($vmid);
4555
4556     if(!defined($devices_list->{$scsihwid})) {
4557         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device, $arch, $machine_type);
4558     }
4559
4560     return 1;
4561 }
4562
4563 sub qemu_deletescsihw {
4564     my ($conf, $vmid, $opt) = @_;
4565
4566     my $device = parse_drive($opt, $conf->{$opt});
4567
4568     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
4569         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
4570         return 1;
4571     }
4572
4573     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4574
4575     my $devices_list = vm_devices_list($vmid);
4576     foreach my $opt (keys %{$devices_list}) {
4577         if (PVE::QemuServer::is_valid_drivename($opt)) {
4578             my $drive = PVE::QemuServer::parse_drive($opt, $conf->{$opt});
4579             if($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
4580                 return 1;
4581             }
4582         }
4583     }
4584
4585     my $scsihwid="scsihw$controller";
4586
4587     vm_deviceunplug($vmid, $conf, $scsihwid);
4588
4589     return 1;
4590 }
4591
4592 sub qemu_add_pci_bridge {
4593     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4594
4595     my $bridges = {};
4596
4597     my $bridgeid;
4598
4599     print_pci_addr($device, $bridges, $arch, $machine_type);
4600
4601     while (my ($k, $v) = each %$bridges) {
4602         $bridgeid = $k;
4603     }
4604     return 1 if !defined($bridgeid) || $bridgeid < 1;
4605
4606     my $bridge = "pci.$bridgeid";
4607     my $devices_list = vm_devices_list($vmid);
4608
4609     if (!defined($devices_list->{$bridge})) {
4610         vm_deviceplug($storecfg, $conf, $vmid, $bridge, $arch, $machine_type);
4611     }
4612
4613     return 1;
4614 }
4615
4616 sub qemu_set_link_status {
4617     my ($vmid, $device, $up) = @_;
4618
4619     vm_mon_cmd($vmid, "set_link", name => $device,
4620                up => $up ? JSON::true : JSON::false);
4621 }
4622
4623 sub qemu_netdevadd {
4624     my ($vmid, $conf, $arch, $device, $deviceid) = @_;
4625
4626     my $netdev = print_netdev_full($vmid, $conf, $arch, $device, $deviceid, 1);
4627     my %options =  split(/[=,]/, $netdev);
4628
4629     vm_mon_cmd($vmid, "netdev_add",  %options);
4630     return 1;
4631 }
4632
4633 sub qemu_netdevdel {
4634     my ($vmid, $deviceid) = @_;
4635
4636     vm_mon_cmd($vmid, "netdev_del", id => $deviceid);
4637 }
4638
4639 sub qemu_usb_hotplug {
4640     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4641
4642     return if !$device;
4643
4644     # remove the old one first
4645     vm_deviceunplug($vmid, $conf, $deviceid);
4646
4647     # check if xhci controller is necessary and available
4648     if ($device->{usb3}) {
4649
4650         my $devicelist = vm_devices_list($vmid);
4651
4652         if (!$devicelist->{xhci}) {
4653             my $pciaddr = print_pci_addr("xhci", undef, $arch, $machine_type);
4654             qemu_deviceadd($vmid, "nec-usb-xhci,id=xhci$pciaddr");
4655         }
4656     }
4657     my $d = parse_usb_device($device->{host});
4658     $d->{usb3} = $device->{usb3};
4659
4660     # add the new one
4661     vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $d, $arch, $machine_type);
4662 }
4663
4664 sub qemu_cpu_hotplug {
4665     my ($vmid, $conf, $vcpus) = @_;
4666
4667     my $machine_type = PVE::QemuServer::get_current_qemu_machine($vmid);
4668
4669     my $sockets = 1;
4670     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
4671     $sockets = $conf->{sockets} if  $conf->{sockets};
4672     my $cores = $conf->{cores} || 1;
4673     my $maxcpus = $sockets * $cores;
4674
4675     $vcpus = $maxcpus if !$vcpus;
4676
4677     die "you can't add more vcpus than maxcpus\n"
4678         if $vcpus > $maxcpus;
4679
4680     my $currentvcpus = $conf->{vcpus} || $maxcpus;
4681
4682     if ($vcpus < $currentvcpus) {
4683
4684         if (qemu_machine_feature_enabled ($machine_type, undef, 2, 7)) {
4685
4686             for (my $i = $currentvcpus; $i > $vcpus; $i--) {
4687                 qemu_devicedel($vmid, "cpu$i");
4688                 my $retry = 0;
4689                 my $currentrunningvcpus = undef;
4690                 while (1) {
4691                     $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
4692                     last if scalar(@{$currentrunningvcpus}) == $i-1;
4693                     raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
4694                     $retry++;
4695                     sleep 1;
4696                 }
4697                 #update conf after each succesfull cpu unplug
4698                 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4699                 PVE::QemuConfig->write_config($vmid, $conf);
4700             }
4701         } else {
4702             die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
4703         }
4704
4705         return;
4706     }
4707
4708     my $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
4709     die "vcpus in running vm does not match its configuration\n"
4710         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
4711
4712     if (qemu_machine_feature_enabled ($machine_type, undef, 2, 7)) {
4713
4714         for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
4715             my $cpustr = print_cpu_device($conf, $i);
4716             qemu_deviceadd($vmid, $cpustr);
4717
4718             my $retry = 0;
4719             my $currentrunningvcpus = undef;
4720             while (1) {
4721                 $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
4722                 last if scalar(@{$currentrunningvcpus}) == $i;
4723                 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
4724                 sleep 1;
4725                 $retry++;
4726             }
4727             #update conf after each succesfull cpu hotplug
4728             $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4729             PVE::QemuConfig->write_config($vmid, $conf);
4730         }
4731     } else {
4732
4733         for (my $i = $currentvcpus; $i < $vcpus; $i++) {
4734             vm_mon_cmd($vmid, "cpu-add", id => int($i));
4735         }
4736     }
4737 }
4738
4739 sub qemu_block_set_io_throttle {
4740     my ($vmid, $deviceid,
4741         $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
4742         $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max,
4743         $bps_max_length, $bps_rd_max_length, $bps_wr_max_length,
4744         $iops_max_length, $iops_rd_max_length, $iops_wr_max_length) = @_;
4745
4746     return if !check_running($vmid) ;
4747
4748     vm_mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
4749         bps => int($bps),
4750         bps_rd => int($bps_rd),
4751         bps_wr => int($bps_wr),
4752         iops => int($iops),
4753         iops_rd => int($iops_rd),
4754         iops_wr => int($iops_wr),
4755         bps_max => int($bps_max),
4756         bps_rd_max => int($bps_rd_max),
4757         bps_wr_max => int($bps_wr_max),
4758         iops_max => int($iops_max),
4759         iops_rd_max => int($iops_rd_max),
4760         iops_wr_max => int($iops_wr_max),
4761         bps_max_length => int($bps_max_length),
4762         bps_rd_max_length => int($bps_rd_max_length),
4763         bps_wr_max_length => int($bps_wr_max_length),
4764         iops_max_length => int($iops_max_length),
4765         iops_rd_max_length => int($iops_rd_max_length),
4766         iops_wr_max_length => int($iops_wr_max_length),
4767     );
4768
4769 }
4770
4771 # old code, only used to shutdown old VM after update
4772 sub __read_avail {
4773     my ($fh, $timeout) = @_;
4774
4775     my $sel = new IO::Select;
4776     $sel->add($fh);
4777
4778     my $res = '';
4779     my $buf;
4780
4781     my @ready;
4782     while (scalar (@ready = $sel->can_read($timeout))) {
4783         my $count;
4784         if ($count = $fh->sysread($buf, 8192)) {
4785             if ($buf =~ /^(.*)\(qemu\) $/s) {
4786                 $res .= $1;
4787                 last;
4788             } else {
4789                 $res .= $buf;
4790             }
4791         } else {
4792             if (!defined($count)) {
4793                 die "$!\n";
4794             }
4795             last;
4796         }
4797     }
4798
4799     die "monitor read timeout\n" if !scalar(@ready);
4800
4801     return $res;
4802 }
4803
4804 sub qemu_block_resize {
4805     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4806
4807     my $running = check_running($vmid);
4808
4809     $size = 0 if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4810
4811     return if !$running;
4812
4813     vm_mon_cmd($vmid, "block_resize", device => $deviceid, size => int($size));
4814
4815 }
4816
4817 sub qemu_volume_snapshot {
4818     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4819
4820     my $running = check_running($vmid);
4821
4822     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4823         vm_mon_cmd($vmid, 'blockdev-snapshot-internal-sync', device => $deviceid, name => $snap);
4824     } else {
4825         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4826     }
4827 }
4828
4829 sub qemu_volume_snapshot_delete {
4830     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4831
4832     my $running = check_running($vmid);
4833
4834     if($running) {
4835
4836         $running = undef;
4837         my $conf = PVE::QemuConfig->load_config($vmid);
4838         foreach_drive($conf, sub {
4839             my ($ds, $drive) = @_;
4840             $running = 1 if $drive->{file} eq $volid;
4841         });
4842     }
4843
4844     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4845         vm_mon_cmd($vmid, 'blockdev-snapshot-delete-internal-sync', device => $deviceid, name => $snap);
4846     } else {
4847         PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4848     }
4849 }
4850
4851 sub set_migration_caps {
4852     my ($vmid) = @_;
4853
4854     my $cap_ref = [];
4855
4856     my $enabled_cap = {
4857         "auto-converge" => 1,
4858         "xbzrle" => 1,
4859         "x-rdma-pin-all" => 0,
4860         "zero-blocks" => 0,
4861         "compress" => 0
4862     };
4863
4864     my $supported_capabilities = vm_mon_cmd_nocheck($vmid, "query-migrate-capabilities");
4865
4866     for my $supported_capability (@$supported_capabilities) {
4867         push @$cap_ref, {
4868             capability => $supported_capability->{capability},
4869             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4870         };
4871     }
4872
4873     vm_mon_cmd_nocheck($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4874 }
4875
4876 my $fast_plug_option = {
4877     'lock' => 1,
4878     'name' => 1,
4879     'onboot' => 1,
4880     'shares' => 1,
4881     'startup' => 1,
4882     'description' => 1,
4883     'protection' => 1,
4884     'vmstatestorage' => 1,
4885     'hookscript' => 1,
4886 };
4887
4888 # hotplug changes in [PENDING]
4889 # $selection hash can be used to only apply specified options, for
4890 # example: { cores => 1 } (only apply changed 'cores')
4891 # $errors ref is used to return error messages
4892 sub vmconfig_hotplug_pending {
4893     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4894
4895     my $defaults = load_defaults();
4896     my ($arch, $machine_type) = get_basic_machine_info($conf, undef);
4897
4898     # commit values which do not have any impact on running VM first
4899     # Note: those option cannot raise errors, we we do not care about
4900     # $selection and always apply them.
4901
4902     my $add_error = sub {
4903         my ($opt, $msg) = @_;
4904         $errors->{$opt} = "hotplug problem - $msg";
4905     };
4906
4907     my $changes = 0;
4908     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4909         if ($fast_plug_option->{$opt}) {
4910             $conf->{$opt} = $conf->{pending}->{$opt};
4911             delete $conf->{pending}->{$opt};
4912             $changes = 1;
4913         }
4914     }
4915
4916     if ($changes) {
4917         PVE::QemuConfig->write_config($vmid, $conf);
4918         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4919     }
4920
4921     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4922
4923     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4924     while (my ($opt, $force) = each %$pending_delete_hash) {
4925         next if $selection && !$selection->{$opt};
4926         eval {
4927             if ($opt eq 'hotplug') {
4928                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
4929             } elsif ($opt eq 'tablet') {
4930                 die "skip\n" if !$hotplug_features->{usb};
4931                 if ($defaults->{tablet}) {
4932                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4933                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4934                         if $arch eq 'aarch64';
4935                 } else {
4936                     vm_deviceunplug($vmid, $conf, 'tablet');
4937                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4938                 }
4939             } elsif ($opt =~ m/^usb\d+/) {
4940                 die "skip\n";
4941                 # since we cannot reliably hot unplug usb devices
4942                 # we are disabling it
4943                 die "skip\n" if !$hotplug_features->{usb} || $conf->{$opt} =~ m/spice/i;
4944                 vm_deviceunplug($vmid, $conf, $opt);
4945             } elsif ($opt eq 'vcpus') {
4946                 die "skip\n" if !$hotplug_features->{cpu};
4947                 qemu_cpu_hotplug($vmid, $conf, undef);
4948             } elsif ($opt eq 'balloon') {
4949                 # enable balloon device is not hotpluggable
4950                 die "skip\n" if defined($conf->{balloon}) && $conf->{balloon} == 0;
4951                 # here we reset the ballooning value to memory
4952                 my $balloon = $conf->{memory} || $defaults->{memory};
4953                 vm_mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4954             } elsif ($fast_plug_option->{$opt}) {
4955                 # do nothing
4956             } elsif ($opt =~ m/^net(\d+)$/) {
4957                 die "skip\n" if !$hotplug_features->{network};
4958                 vm_deviceunplug($vmid, $conf, $opt);
4959             } elsif (is_valid_drivename($opt)) {
4960                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4961                 vm_deviceunplug($vmid, $conf, $opt);
4962                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4963             } elsif ($opt =~ m/^memory$/) {
4964                 die "skip\n" if !$hotplug_features->{memory};
4965                 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
4966             } elsif ($opt eq 'cpuunits') {
4967                 cgroups_write("cpu", $vmid, "cpu.shares", $defaults->{cpuunits});
4968             } elsif ($opt eq 'cpulimit') {
4969                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", -1);
4970             } else {
4971                 die "skip\n";
4972             }
4973         };
4974         if (my $err = $@) {
4975             &$add_error($opt, $err) if $err ne "skip\n";
4976         } else {
4977             # save new config if hotplug was successful
4978             delete $conf->{$opt};
4979             vmconfig_undelete_pending_option($conf, $opt);
4980             PVE::QemuConfig->write_config($vmid, $conf);
4981             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4982         }
4983     }
4984
4985     my $apply_pending_cloudinit;
4986     $apply_pending_cloudinit = sub {
4987         my ($key, $value) = @_;
4988         $apply_pending_cloudinit = sub {}; # once is enough
4989
4990         my @cloudinit_opts = keys %$confdesc_cloudinit;
4991         foreach my $opt (keys %{$conf->{pending}}) {
4992             next if !grep { $_ eq $opt } @cloudinit_opts;
4993             $conf->{$opt} = delete $conf->{pending}->{$opt};
4994         }
4995
4996         my $new_conf = { %$conf };
4997         $new_conf->{$key} = $value;
4998         PVE::QemuServer::Cloudinit::generate_cloudinitconfig($new_conf, $vmid);
4999     };
5000
5001     foreach my $opt (keys %{$conf->{pending}}) {
5002         next if $selection && !$selection->{$opt};
5003         my $value = $conf->{pending}->{$opt};
5004         eval {
5005             if ($opt eq 'hotplug') {
5006                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
5007             } elsif ($opt eq 'tablet') {
5008                 die "skip\n" if !$hotplug_features->{usb};
5009                 if ($value == 1) {
5010                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
5011                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
5012                         if $arch eq 'aarch64';
5013                 } elsif ($value == 0) {
5014                     vm_deviceunplug($vmid, $conf, 'tablet');
5015                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
5016                 }
5017             } elsif ($opt =~ m/^usb\d+$/) {
5018                 die "skip\n";
5019                 # since we cannot reliably hot unplug usb devices
5020                 # we are disabling it
5021                 die "skip\n" if !$hotplug_features->{usb} || $value =~ m/spice/i;
5022                 my $d = eval { PVE::JSONSchema::parse_property_string($usbdesc->{format}, $value) };
5023                 die "skip\n" if !$d;
5024                 qemu_usb_hotplug($storecfg, $conf, $vmid, $opt, $d, $arch, $machine_type);
5025             } elsif ($opt eq 'vcpus') {
5026                 die "skip\n" if !$hotplug_features->{cpu};
5027                 qemu_cpu_hotplug($vmid, $conf, $value);
5028             } elsif ($opt eq 'balloon') {
5029                 # enable/disable balloning device is not hotpluggable
5030                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
5031                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
5032                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
5033
5034                 # allow manual ballooning if shares is set to zero
5035                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
5036                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
5037                     vm_mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
5038                 }
5039             } elsif ($opt =~ m/^net(\d+)$/) {
5040                 # some changes can be done without hotplug
5041                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
5042                                     $vmid, $opt, $value, $arch, $machine_type);
5043             } elsif (is_valid_drivename($opt)) {
5044                 # some changes can be done without hotplug
5045                 my $drive = parse_drive($opt, $value);
5046                 if (drive_is_cloudinit($drive)) {
5047                     &$apply_pending_cloudinit($opt, $value);
5048                 }
5049                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
5050                                      $vmid, $opt, $value, 1, $arch, $machine_type);
5051             } elsif ($opt =~ m/^memory$/) { #dimms
5052                 die "skip\n" if !$hotplug_features->{memory};
5053                 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
5054             } elsif ($opt eq 'cpuunits') {
5055                 cgroups_write("cpu", $vmid, "cpu.shares", $conf->{pending}->{$opt});
5056             } elsif ($opt eq 'cpulimit') {
5057                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
5058                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", $cpulimit);
5059             } else {
5060                 die "skip\n";  # skip non-hot-pluggable options
5061             }
5062         };
5063         if (my $err = $@) {
5064             &$add_error($opt, $err) if $err ne "skip\n";
5065         } else {
5066             # save new config if hotplug was successful
5067             $conf->{$opt} = $value;
5068             delete $conf->{pending}->{$opt};
5069             PVE::QemuConfig->write_config($vmid, $conf);
5070             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5071         }
5072     }
5073 }
5074
5075 sub try_deallocate_drive {
5076     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
5077
5078     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
5079         my $volid = $drive->{file};
5080         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
5081             my $sid = PVE::Storage::parse_volume_id($volid);
5082             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
5083
5084             # check if the disk is really unused
5085             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
5086                 if is_volume_in_use($storecfg, $conf, $key, $volid);
5087             PVE::Storage::vdisk_free($storecfg, $volid);
5088             return 1;
5089         } else {
5090             # If vm is not owner of this disk remove from config
5091             return 1;
5092         }
5093     }
5094
5095     return undef;
5096 }
5097
5098 sub vmconfig_delete_or_detach_drive {
5099     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
5100
5101     my $drive = parse_drive($opt, $conf->{$opt});
5102
5103     my $rpcenv = PVE::RPCEnvironment::get();
5104     my $authuser = $rpcenv->get_user();
5105
5106     if ($force) {
5107         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
5108         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
5109     } else {
5110         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
5111     }
5112 }
5113
5114 sub vmconfig_apply_pending {
5115     my ($vmid, $conf, $storecfg) = @_;
5116
5117     # cold plug
5118
5119     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
5120     while (my ($opt, $force) = each %$pending_delete_hash) {
5121         die "internal error" if $opt =~ m/^unused/;
5122         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5123         if (!defined($conf->{$opt})) {
5124             vmconfig_undelete_pending_option($conf, $opt);
5125             PVE::QemuConfig->write_config($vmid, $conf);
5126         } elsif (is_valid_drivename($opt)) {
5127             vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
5128             vmconfig_undelete_pending_option($conf, $opt);
5129             delete $conf->{$opt};
5130             PVE::QemuConfig->write_config($vmid, $conf);
5131         } else {
5132             vmconfig_undelete_pending_option($conf, $opt);
5133             delete $conf->{$opt};
5134             PVE::QemuConfig->write_config($vmid, $conf);
5135         }
5136     }
5137
5138     $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5139
5140     foreach my $opt (keys %{$conf->{pending}}) { # add/change
5141         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5142
5143         if (defined($conf->{$opt}) && ($conf->{$opt} eq $conf->{pending}->{$opt})) {
5144             # skip if nothing changed
5145         } elsif (is_valid_drivename($opt)) {
5146             vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
5147                 if defined($conf->{$opt});
5148             $conf->{$opt} = $conf->{pending}->{$opt};
5149         } else {
5150             $conf->{$opt} = $conf->{pending}->{$opt};
5151         }
5152
5153         delete $conf->{pending}->{$opt};
5154         PVE::QemuConfig->write_config($vmid, $conf);
5155     }
5156 }
5157
5158 my $safe_num_ne = sub {
5159     my ($a, $b) = @_;
5160
5161     return 0 if !defined($a) && !defined($b);
5162     return 1 if !defined($a);
5163     return 1 if !defined($b);
5164
5165     return $a != $b;
5166 };
5167
5168 my $safe_string_ne = sub {
5169     my ($a, $b) = @_;
5170
5171     return 0 if !defined($a) && !defined($b);
5172     return 1 if !defined($a);
5173     return 1 if !defined($b);
5174
5175     return $a ne $b;
5176 };
5177
5178 sub vmconfig_update_net {
5179     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
5180
5181     my $newnet = parse_net($value);
5182
5183     if ($conf->{$opt}) {
5184         my $oldnet = parse_net($conf->{$opt});
5185
5186         if (&$safe_string_ne($oldnet->{model}, $newnet->{model}) ||
5187             &$safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
5188             &$safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
5189             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
5190
5191             # for non online change, we try to hot-unplug
5192             die "skip\n" if !$hotplug;
5193             vm_deviceunplug($vmid, $conf, $opt);
5194         } else {
5195
5196             die "internal error" if $opt !~ m/net(\d+)/;
5197             my $iface = "tap${vmid}i$1";
5198
5199             if (&$safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
5200                 &$safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
5201                 &$safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
5202                 &$safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
5203                 PVE::Network::tap_unplug($iface);
5204                 PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
5205             } elsif (&$safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
5206                 # Rate can be applied on its own but any change above needs to
5207                 # include the rate in tap_plug since OVS resets everything.
5208                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
5209             }
5210
5211             if (&$safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
5212                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
5213             }
5214
5215             return 1;
5216         }
5217     }
5218
5219     if ($hotplug) {
5220         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet, $arch, $machine_type);
5221     } else {
5222         die "skip\n";
5223     }
5224 }
5225
5226 sub vmconfig_update_disk {
5227     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $force, $arch, $machine_type) = @_;
5228
5229     # fixme: do we need force?
5230
5231     my $drive = parse_drive($opt, $value);
5232
5233     if ($conf->{$opt}) {
5234
5235         if (my $old_drive = parse_drive($opt, $conf->{$opt}))  {
5236
5237             my $media = $drive->{media} || 'disk';
5238             my $oldmedia = $old_drive->{media} || 'disk';
5239             die "unable to change media type\n" if $media ne $oldmedia;
5240
5241             if (!drive_is_cdrom($old_drive)) {
5242
5243                 if ($drive->{file} ne $old_drive->{file}) {
5244
5245                     die "skip\n" if !$hotplug;
5246
5247                     # unplug and register as unused
5248                     vm_deviceunplug($vmid, $conf, $opt);
5249                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
5250
5251                 } else {
5252                     # update existing disk
5253
5254                     # skip non hotpluggable value
5255                     if (&$safe_string_ne($drive->{discard}, $old_drive->{discard}) ||
5256                         &$safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
5257                         &$safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
5258                         &$safe_string_ne($drive->{cache}, $old_drive->{cache})) {
5259                         die "skip\n";
5260                     }
5261
5262                     # apply throttle
5263                     if (&$safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
5264                         &$safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
5265                         &$safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
5266                         &$safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
5267                         &$safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
5268                         &$safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
5269                         &$safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
5270                         &$safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
5271                         &$safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
5272                         &$safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
5273                         &$safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
5274                         &$safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max}) ||
5275                         &$safe_num_ne($drive->{bps_max_length}, $old_drive->{bps_max_length}) ||
5276                         &$safe_num_ne($drive->{bps_rd_max_length}, $old_drive->{bps_rd_max_length}) ||
5277                         &$safe_num_ne($drive->{bps_wr_max_length}, $old_drive->{bps_wr_max_length}) ||
5278                         &$safe_num_ne($drive->{iops_max_length}, $old_drive->{iops_max_length}) ||
5279                         &$safe_num_ne($drive->{iops_rd_max_length}, $old_drive->{iops_rd_max_length}) ||
5280                         &$safe_num_ne($drive->{iops_wr_max_length}, $old_drive->{iops_wr_max_length})) {
5281
5282                         qemu_block_set_io_throttle($vmid,"drive-$opt",
5283                                                    ($drive->{mbps} || 0)*1024*1024,
5284                                                    ($drive->{mbps_rd} || 0)*1024*1024,
5285                                                    ($drive->{mbps_wr} || 0)*1024*1024,
5286                                                    $drive->{iops} || 0,
5287                                                    $drive->{iops_rd} || 0,
5288                                                    $drive->{iops_wr} || 0,
5289                                                    ($drive->{mbps_max} || 0)*1024*1024,
5290                                                    ($drive->{mbps_rd_max} || 0)*1024*1024,
5291                                                    ($drive->{mbps_wr_max} || 0)*1024*1024,
5292                                                    $drive->{iops_max} || 0,
5293                                                    $drive->{iops_rd_max} || 0,
5294                                                    $drive->{iops_wr_max} || 0,
5295                                                    $drive->{bps_max_length} || 1,
5296                                                    $drive->{bps_rd_max_length} || 1,
5297                                                    $drive->{bps_wr_max_length} || 1,
5298                                                    $drive->{iops_max_length} || 1,
5299                                                    $drive->{iops_rd_max_length} || 1,
5300                                                    $drive->{iops_wr_max_length} || 1);
5301
5302                     }
5303
5304                     return 1;
5305                 }
5306
5307             } else { # cdrom
5308
5309                 if ($drive->{file} eq 'none') {
5310                     vm_mon_cmd($vmid, "eject",force => JSON::true,device => "drive-$opt");
5311                     if (drive_is_cloudinit($old_drive)) {
5312                         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive);
5313                     }
5314                 } else {
5315                     my $path = get_iso_path($storecfg, $vmid, $drive->{file});
5316                     vm_mon_cmd($vmid, "eject", force => JSON::true,device => "drive-$opt"); # force eject if locked
5317                     vm_mon_cmd($vmid, "change", device => "drive-$opt",target => "$path") if $path;
5318                 }
5319
5320                 return 1;
5321             }
5322         }
5323     }
5324
5325     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
5326     # hotplug new disks
5327     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
5328     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive, $arch, $machine_type);
5329 }
5330
5331 sub vm_start {
5332     my ($storecfg, $vmid, $statefile, $skiplock, $migratedfrom, $paused,
5333         $forcemachine, $spice_ticket, $migration_network, $migration_type, $targetstorage) = @_;
5334
5335     PVE::QemuConfig->lock_config($vmid, sub {
5336         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
5337
5338         die "you can't start a vm if it's a template\n" if PVE::QemuConfig->is_template($conf);
5339
5340         my $is_suspended = PVE::QemuConfig->has_lock($conf, 'suspended');
5341
5342         PVE::QemuConfig->check_lock($conf)
5343             if !($skiplock || $is_suspended);
5344
5345         die "VM $vmid already running\n" if check_running($vmid, undef, $migratedfrom);
5346
5347         # clean up leftover reboot request files
5348         eval { clear_reboot_request($vmid); };
5349         warn $@ if $@;
5350
5351         if (!$statefile && scalar(keys %{$conf->{pending}})) {
5352             vmconfig_apply_pending($vmid, $conf, $storecfg);
5353             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5354         }
5355
5356         PVE::QemuServer::Cloudinit::generate_cloudinitconfig($conf, $vmid);
5357
5358         my $defaults = load_defaults();
5359
5360         # set environment variable useful inside network script
5361         $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
5362
5363         my $local_volumes = {};
5364
5365         if ($targetstorage) {
5366             foreach_drive($conf, sub {
5367                 my ($ds, $drive) = @_;
5368
5369                 return if drive_is_cdrom($drive);
5370
5371                 my $volid = $drive->{file};
5372
5373                 return if !$volid;
5374
5375                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
5376
5377                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5378                 return if $scfg->{shared};
5379                 $local_volumes->{$ds} = [$volid, $storeid, $volname];
5380             });
5381
5382             my $format = undef;
5383
5384             foreach my $opt (sort keys %$local_volumes) {
5385
5386                 my ($volid, $storeid, $volname) = @{$local_volumes->{$opt}};
5387                 my $drive = parse_drive($opt, $conf->{$opt});
5388
5389                 #if remote storage is specified, use default format
5390                 if ($targetstorage && $targetstorage ne "1") {
5391                     $storeid = $targetstorage;
5392                     my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
5393                     $format = $defFormat;
5394                 } else {
5395                     #else we use same format than original
5396                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5397                     $format = qemu_img_format($scfg, $volid);
5398                 }
5399
5400                 my $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $format, undef, ($drive->{size}/1024));
5401                 my $newdrive = $drive;
5402                 $newdrive->{format} = $format;
5403                 $newdrive->{file} = $newvolid;
5404                 my $drivestr = PVE::QemuServer::print_drive($vmid, $newdrive);
5405                 $local_volumes->{$opt} = $drivestr;
5406                 #pass drive to conf for command line
5407                 $conf->{$opt} = $drivestr;
5408             }
5409         }
5410
5411         PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-start', 1);
5412
5413         if ($is_suspended) {
5414             # enforce machine type on suspended vm to ensure HW compatibility
5415             $forcemachine = $conf->{runningmachine};
5416             print "Resuming suspended VM\n";
5417         }
5418
5419         my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine);
5420
5421         my $migrate_uri;
5422         if ($statefile) {
5423             if ($statefile eq 'tcp') {
5424                 my $localip = "localhost";
5425                 my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
5426                 my $nodename = PVE::INotify::nodename();
5427
5428                 if (!defined($migration_type)) {
5429                     if (defined($datacenterconf->{migration}->{type})) {
5430                         $migration_type = $datacenterconf->{migration}->{type};
5431                     } else {
5432                         $migration_type = 'secure';
5433                     }
5434                 }
5435
5436                 if ($migration_type eq 'insecure') {
5437                     my $migrate_network_addr = PVE::Cluster::get_local_migration_ip($migration_network);
5438                     if ($migrate_network_addr) {
5439                         $localip = $migrate_network_addr;
5440                     } else {
5441                         $localip = PVE::Cluster::remote_node_ip($nodename, 1);
5442                     }
5443
5444                     $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5445                 }
5446
5447                 my $pfamily = PVE::Tools::get_host_address_family($nodename);
5448                 my $migrate_port = PVE::Tools::next_migrate_port($pfamily);
5449                 $migrate_uri = "tcp:${localip}:${migrate_port}";
5450                 push @$cmd, '-incoming', $migrate_uri;
5451                 push @$cmd, '-S';
5452
5453             } elsif ($statefile eq 'unix') {
5454                 # should be default for secure migrations as a ssh TCP forward
5455                 # tunnel is not deterministic reliable ready and fails regurarly
5456                 # to set up in time, so use UNIX socket forwards
5457                 my $socket_addr = "/run/qemu-server/$vmid.migrate";
5458                 unlink $socket_addr;
5459
5460                 $migrate_uri = "unix:$socket_addr";
5461
5462                 push @$cmd, '-incoming', $migrate_uri;
5463                 push @$cmd, '-S';
5464
5465             } else {
5466                 push @$cmd, '-loadstate', $statefile;
5467             }
5468         } elsif ($paused) {
5469             push @$cmd, '-S';
5470         }
5471
5472         # host pci devices
5473         for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
5474           my $d = parse_hostpci($conf->{"hostpci$i"});
5475           next if !$d;
5476           my $pcidevices = $d->{pciid};
5477           foreach my $pcidevice (@$pcidevices) {
5478                 my $pciid = $pcidevice->{id};
5479
5480                 my $info = PVE::SysFSTools::pci_device_info("0000:$pciid");
5481                 die "IOMMU not present\n" if !PVE::SysFSTools::check_iommu_support();
5482                 die "no pci device info for device '$pciid'\n" if !$info;
5483
5484                 if ($d->{mdev}) {
5485                     my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $i);
5486                     PVE::SysFSTools::pci_create_mdev_device($pciid, $uuid, $d->{mdev});
5487                 } else {
5488                     die "can't unbind/bind pci group to vfio '$pciid'\n"
5489                         if !PVE::SysFSTools::pci_dev_group_bind_to_vfio($pciid);
5490                     die "can't reset pci device '$pciid'\n"
5491                         if $info->{has_fl_reset} and !PVE::SysFSTools::pci_dev_reset($info);
5492                 }
5493           }
5494         }
5495
5496         PVE::Storage::activate_volumes($storecfg, $vollist);
5497
5498         eval {
5499             run_command(['/bin/systemctl', 'stop', "$vmid.scope"],
5500                 outfunc => sub {}, errfunc => sub {});
5501         };
5502         # Issues with the above 'stop' not being fully completed are extremely rare, a very low
5503         # timeout should be more than enough here...
5504         PVE::Systemd::wait_for_unit_removed("$vmid.scope", 5);
5505
5506         my $cpuunits = defined($conf->{cpuunits}) ? $conf->{cpuunits}
5507                                                   : $defaults->{cpuunits};
5508
5509         my $start_timeout = ($conf->{hugepages} || $is_suspended) ? 300 : 30;
5510         my %run_params = (timeout => $statefile ? undef : $start_timeout, umask => 0077);
5511
5512         my %properties = (
5513             Slice => 'qemu.slice',
5514             KillMode => 'none',
5515             CPUShares => $cpuunits
5516         );
5517
5518         if (my $cpulimit = $conf->{cpulimit}) {
5519             $properties{CPUQuota} = int($cpulimit * 100);
5520         }
5521         $properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
5522
5523         my $run_qemu = sub {
5524             PVE::Tools::run_fork sub {
5525                 PVE::Systemd::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
5526                 run_command($cmd, %run_params);
5527             };
5528         };
5529
5530         if ($conf->{hugepages}) {
5531
5532             my $code = sub {
5533                 my $hugepages_topology = PVE::QemuServer::Memory::hugepages_topology($conf);
5534                 my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
5535
5536                 PVE::QemuServer::Memory::hugepages_mount();
5537                 PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
5538
5539                 eval { $run_qemu->() };
5540                 if (my $err = $@) {
5541                     PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology);
5542                     die $err;
5543                 }
5544
5545                 PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology);
5546             };
5547             eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
5548
5549         } else {
5550             eval { $run_qemu->() };
5551         }
5552
5553         if (my $err = $@) {
5554             # deactivate volumes if start fails
5555             eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
5556             die "start failed: $err";
5557         }
5558
5559         print "migration listens on $migrate_uri\n" if $migrate_uri;
5560
5561         if ($statefile && $statefile ne 'tcp' && $statefile ne 'unix')  {
5562             eval { vm_mon_cmd_nocheck($vmid, "cont"); };
5563             warn $@ if $@;
5564         }
5565
5566         #start nbd server for storage migration
5567         if ($targetstorage) {
5568             my $nodename = PVE::INotify::nodename();
5569             my $migrate_network_addr = PVE::Cluster::get_local_migration_ip($migration_network);
5570             my $localip = $migrate_network_addr ? $migrate_network_addr : PVE::Cluster::remote_node_ip($nodename, 1);
5571             my $pfamily = PVE::Tools::get_host_address_family($nodename);
5572             my $storage_migrate_port = PVE::Tools::next_migrate_port($pfamily);
5573
5574             vm_mon_cmd_nocheck($vmid, "nbd-server-start", addr => { type => 'inet', data => { host => "${localip}", port => "${storage_migrate_port}" } } );
5575
5576             $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5577
5578             foreach my $opt (sort keys %$local_volumes) {
5579                 my $volid = $local_volumes->{$opt};
5580                 vm_mon_cmd_nocheck($vmid, "nbd-server-add", device => "drive-$opt", writable => JSON::true );
5581                 my $migrate_storage_uri = "nbd:${localip}:${storage_migrate_port}:exportname=drive-$opt";
5582                 print "storage migration listens on $migrate_storage_uri volume:$volid\n";
5583             }
5584         }
5585
5586         if ($migratedfrom) {
5587             eval {
5588                 set_migration_caps($vmid);
5589             };
5590             warn $@ if $@;
5591
5592             if ($spice_port) {
5593                 print "spice listens on port $spice_port\n";
5594                 if ($spice_ticket) {
5595                     vm_mon_cmd_nocheck($vmid, "set_password", protocol => 'spice', password => $spice_ticket);
5596                     vm_mon_cmd_nocheck($vmid, "expire_password", protocol => 'spice', time => "+30");
5597                 }
5598             }
5599
5600         } else {
5601             vm_mon_cmd_nocheck($vmid, "balloon", value => $conf->{balloon}*1024*1024)
5602                 if !$statefile && $conf->{balloon};
5603
5604             foreach my $opt (keys %$conf) {
5605                 next if $opt !~  m/^net\d+$/;
5606                 my $nicconf = parse_net($conf->{$opt});
5607                 qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
5608             }
5609         }
5610
5611         vm_mon_cmd_nocheck($vmid, 'qom-set',
5612                     path => "machine/peripheral/balloon0",
5613                     property => "guest-stats-polling-interval",
5614                     value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
5615
5616         if ($is_suspended && (my $vmstate = $conf->{vmstate})) {
5617             print "Resumed VM, removing state\n";
5618             delete $conf->@{qw(lock vmstate runningmachine)};
5619             PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
5620             PVE::Storage::vdisk_free($storecfg, $vmstate);
5621             PVE::QemuConfig->write_config($vmid, $conf);
5622         }
5623
5624         PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'post-start');
5625     });
5626 }
5627
5628 sub vm_mon_cmd {
5629     my ($vmid, $execute, %params) = @_;
5630
5631     my $cmd = { execute => $execute, arguments => \%params };
5632     vm_qmp_command($vmid, $cmd);
5633 }
5634
5635 sub vm_mon_cmd_nocheck {
5636     my ($vmid, $execute, %params) = @_;
5637
5638     my $cmd = { execute => $execute, arguments => \%params };
5639     vm_qmp_command($vmid, $cmd, 1);
5640 }
5641
5642 sub vm_qmp_command {
5643     my ($vmid, $cmd, $nocheck) = @_;
5644
5645     my $res;
5646
5647     my $timeout;
5648     if ($cmd->{arguments}) {
5649         $timeout = delete $cmd->{arguments}->{timeout};
5650     }
5651
5652     eval {
5653         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
5654         my $sname = qmp_socket($vmid);
5655         if (-e $sname) { # test if VM is reasonambe new and supports qmp/qga
5656             my $qmpclient = PVE::QMPClient->new();
5657
5658             $res = $qmpclient->cmd($vmid, $cmd, $timeout);
5659         } else {
5660             die "unable to open monitor socket\n";
5661         }
5662     };
5663     if (my $err = $@) {
5664         syslog("err", "VM $vmid qmp command failed - $err");
5665         die $err;
5666     }
5667
5668     return $res;
5669 }
5670
5671 sub vm_human_monitor_command {
5672     my ($vmid, $cmdline) = @_;
5673
5674     my $cmd = {
5675         execute => 'human-monitor-command',
5676         arguments => { 'command-line' => $cmdline},
5677     };
5678
5679     return vm_qmp_command($vmid, $cmd);
5680 }
5681
5682 sub vm_commandline {
5683     my ($storecfg, $vmid, $snapname) = @_;
5684
5685     my $conf = PVE::QemuConfig->load_config($vmid);
5686
5687     if ($snapname) {
5688         my $snapshot = $conf->{snapshots}->{$snapname};
5689         die "snapshot '$snapname' does not exist\n" if !defined($snapshot);
5690
5691         $snapshot->{digest} = $conf->{digest}; # keep file digest for API
5692
5693         $conf = $snapshot;
5694     }
5695
5696     my $defaults = load_defaults();
5697
5698     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults);
5699
5700     return PVE::Tools::cmd2string($cmd);
5701 }
5702
5703 sub vm_reset {
5704     my ($vmid, $skiplock) = @_;
5705
5706     PVE::QemuConfig->lock_config($vmid, sub {
5707
5708         my $conf = PVE::QemuConfig->load_config($vmid);
5709
5710         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5711
5712         vm_mon_cmd($vmid, "system_reset");
5713     });
5714 }
5715
5716 sub get_vm_volumes {
5717     my ($conf) = @_;
5718
5719     my $vollist = [];
5720     foreach_volid($conf, sub {
5721         my ($volid, $attr) = @_;
5722
5723         return if $volid =~ m|^/|;
5724
5725         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5726         return if !$sid;
5727
5728         push @$vollist, $volid;
5729     });
5730
5731     return $vollist;
5732 }
5733
5734 sub vm_stop_cleanup {
5735     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
5736
5737     eval {
5738
5739         if (!$keepActive) {
5740             my $vollist = get_vm_volumes($conf);
5741             PVE::Storage::deactivate_volumes($storecfg, $vollist);
5742         }
5743
5744         foreach my $ext (qw(mon qmp pid vnc qga)) {
5745             unlink "/var/run/qemu-server/${vmid}.$ext";
5746         }
5747
5748         if ($conf->{ivshmem}) {
5749             my $ivshmem = PVE::JSONSchema::parse_property_string($ivshmem_fmt, $conf->{ivshmem});
5750             # just delete it for now, VMs which have this already open do not
5751             # are affected, but new VMs will get a separated one. If this
5752             # becomes an issue we either add some sort of ref-counting or just
5753             # add a "don't delete on stop" flag to the ivshmem format.
5754             unlink '/dev/shm/pve-shm-' . ($ivshmem->{name} // $vmid);
5755         }
5756
5757         foreach my $key (keys %$conf) {
5758             next if $key !~ m/^hostpci(\d+)$/;
5759             my $hostpciindex = $1;
5760             my $d = parse_hostpci($conf->{$key});
5761             my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $hostpciindex);
5762
5763             foreach my $pci (@{$d->{pciid}}) {
5764                 my $pciid = $pci->{id};
5765                 PVE::SysFSTools::pci_cleanup_mdev_device($pciid, $uuid);
5766             }
5767         }
5768
5769         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
5770     };
5771     warn $@ if $@; # avoid errors - just warn
5772 }
5773
5774 # call only in locked context
5775 sub _do_vm_stop {
5776     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive) = @_;
5777
5778     my $pid = check_running($vmid, $nocheck);
5779     return if !$pid;
5780
5781     my $conf;
5782     if (!$nocheck) {
5783         $conf = PVE::QemuConfig->load_config($vmid);
5784         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5785         if (!defined($timeout) && $shutdown && $conf->{startup}) {
5786             my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
5787             $timeout = $opts->{down} if $opts->{down};
5788         }
5789         PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-stop');
5790     }
5791
5792     eval {
5793         if ($shutdown) {
5794             if (defined($conf) && parse_guest_agent($conf)->{enabled}) {
5795                 vm_qmp_command($vmid, {
5796                         execute => "guest-shutdown",
5797                         arguments => { timeout => $timeout }
5798                     }, $nocheck);
5799             } else {
5800                 vm_qmp_command($vmid, { execute => "system_powerdown" }, $nocheck);
5801             }
5802         } else {
5803             vm_qmp_command($vmid, { execute => "quit" }, $nocheck);
5804         }
5805     };
5806     my $err = $@;
5807
5808     if (!$err) {
5809         $timeout = 60 if !defined($timeout);
5810
5811         my $count = 0;
5812         while (($count < $timeout) && check_running($vmid, $nocheck)) {
5813             $count++;
5814             sleep 1;
5815         }
5816
5817         if ($count >= $timeout) {
5818             if ($force) {
5819                 warn "VM still running - terminating now with SIGTERM\n";
5820                 kill 15, $pid;
5821             } else {
5822                 die "VM quit/powerdown failed - got timeout\n";
5823             }
5824         } else {
5825             vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5826             return;
5827         }
5828     } else {
5829         if ($force) {
5830             warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
5831             kill 15, $pid;
5832         } else {
5833             die "VM quit/powerdown failed\n";
5834         }
5835     }
5836
5837     # wait again
5838     $timeout = 10;
5839
5840     my $count = 0;
5841     while (($count < $timeout) && check_running($vmid, $nocheck)) {
5842         $count++;
5843         sleep 1;
5844     }
5845
5846     if ($count >= $timeout) {
5847         warn "VM still running - terminating now with SIGKILL\n";
5848         kill 9, $pid;
5849         sleep 1;
5850     }
5851
5852     vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5853 }
5854
5855 # Note: use $nocheck to skip tests if VM configuration file exists.
5856 # We need that when migration VMs to other nodes (files already moved)
5857 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
5858 sub vm_stop {
5859     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
5860
5861     $force = 1 if !defined($force) && !$shutdown;
5862
5863     if ($migratedfrom){
5864         my $pid = check_running($vmid, $nocheck, $migratedfrom);
5865         kill 15, $pid if $pid;
5866         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
5867         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
5868         return;
5869     }
5870
5871     PVE::QemuConfig->lock_config($vmid, sub {
5872         _do_vm_stop($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive);
5873    });
5874 }
5875
5876 sub vm_reboot {
5877     my ($vmid, $timeout) = @_;
5878
5879     PVE::QemuConfig->lock_config($vmid, sub {
5880
5881         # only reboot if running, as qmeventd starts it again on a stop event
5882         return if !check_running($vmid);
5883
5884         create_reboot_request($vmid);
5885
5886         my $storecfg = PVE::Storage::config();
5887         _do_vm_stop($storecfg, $vmid, undef, undef, $timeout, 1);
5888
5889    });
5890 }
5891
5892 sub vm_suspend {
5893     my ($vmid, $skiplock, $includestate, $statestorage) = @_;
5894
5895     my $conf;
5896     my $path;
5897     my $storecfg;
5898     my $vmstate;
5899
5900     PVE::QemuConfig->lock_config($vmid, sub {
5901
5902         $conf = PVE::QemuConfig->load_config($vmid);
5903
5904         my $is_backing_up = PVE::QemuConfig->has_lock($conf, 'backup');
5905         PVE::QemuConfig->check_lock($conf)
5906             if !($skiplock || $is_backing_up);
5907
5908         die "cannot suspend to disk during backup\n"
5909             if $is_backing_up && $includestate;
5910
5911         if ($includestate) {
5912             $conf->{lock} = 'suspending';
5913             my $date = strftime("%Y-%m-%d", localtime(time()));
5914             $storecfg = PVE::Storage::config();
5915             $vmstate = PVE::QemuConfig->__snapshot_save_vmstate($vmid, $conf, "suspend-$date", $storecfg, $statestorage, 1);
5916             $path = PVE::Storage::path($storecfg, $vmstate);
5917             PVE::QemuConfig->write_config($vmid, $conf);
5918         } else {
5919             vm_mon_cmd($vmid, "stop");
5920         }
5921     });
5922
5923     if ($includestate) {
5924         # save vm state
5925         PVE::Storage::activate_volumes($storecfg, [$vmstate]);
5926
5927         eval {
5928             vm_mon_cmd($vmid, "savevm-start", statefile => $path);
5929             for(;;) {
5930                 my $state = vm_mon_cmd_nocheck($vmid, "query-savevm");
5931                 if (!$state->{status}) {
5932                     die "savevm not active\n";
5933                 } elsif ($state->{status} eq 'active') {
5934                     sleep(1);
5935                     next;
5936                 } elsif ($state->{status} eq 'completed') {
5937                     print "State saved, quitting\n";
5938                     last;
5939                 } elsif ($state->{status} eq 'failed' && $state->{error}) {
5940                     die "query-savevm failed with error '$state->{error}'\n"
5941                 } else {
5942                     die "query-savevm returned status '$state->{status}'\n";
5943                 }
5944             }
5945         };
5946         my $err = $@;
5947
5948         PVE::QemuConfig->lock_config($vmid, sub {
5949             $conf = PVE::QemuConfig->load_config($vmid);
5950             if ($err) {
5951                 # cleanup, but leave suspending lock, to indicate something went wrong
5952                 eval {
5953                     vm_mon_cmd($vmid, "savevm-end");
5954                     PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
5955                     PVE::Storage::vdisk_free($storecfg, $vmstate);
5956                     delete $conf->@{qw(vmstate runningmachine)};
5957                     PVE::QemuConfig->write_config($vmid, $conf);
5958                 };
5959                 warn $@ if $@;
5960                 die $err;
5961             }
5962
5963             die "lock changed unexpectedly\n"
5964                 if !PVE::QemuConfig->has_lock($conf, 'suspending');
5965
5966             vm_qmp_command($vmid, { execute => "quit" });
5967             $conf->{lock} = 'suspended';
5968             PVE::QemuConfig->write_config($vmid, $conf);
5969         });
5970     }
5971 }
5972
5973 sub vm_resume {
5974     my ($vmid, $skiplock, $nocheck) = @_;
5975
5976     PVE::QemuConfig->lock_config($vmid, sub {
5977         my $vm_mon_cmd = $nocheck ? \&vm_mon_cmd_nocheck : \&vm_mon_cmd;
5978         my $res = $vm_mon_cmd->($vmid, 'query-status');
5979         my $resume_cmd = 'cont';
5980
5981         if ($res->{status} && $res->{status} eq 'suspended') {
5982             $resume_cmd = 'system_wakeup';
5983         }
5984
5985         if (!$nocheck) {
5986
5987             my $conf = PVE::QemuConfig->load_config($vmid);
5988
5989             PVE::QemuConfig->check_lock($conf)
5990                 if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
5991         }
5992
5993         $vm_mon_cmd->($vmid, $resume_cmd);
5994     });
5995 }
5996
5997 sub vm_sendkey {
5998     my ($vmid, $skiplock, $key) = @_;
5999
6000     PVE::QemuConfig->lock_config($vmid, sub {
6001
6002         my $conf = PVE::QemuConfig->load_config($vmid);
6003
6004         # there is no qmp command, so we use the human monitor command
6005         my $res = vm_human_monitor_command($vmid, "sendkey $key");
6006         die $res if $res ne '';
6007     });
6008 }
6009
6010 sub vm_destroy {
6011     my ($storecfg, $vmid, $skiplock) = @_;
6012
6013     PVE::QemuConfig->lock_config($vmid, sub {
6014
6015         my $conf = PVE::QemuConfig->load_config($vmid);
6016
6017         if (!check_running($vmid)) {
6018             destroy_vm($storecfg, $vmid, undef, $skiplock);
6019         } else {
6020             die "VM $vmid is running - destroy failed\n";
6021         }
6022     });
6023 }
6024
6025 # vzdump restore implementaion
6026
6027 sub tar_archive_read_firstfile {
6028     my $archive = shift;
6029
6030     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
6031
6032     # try to detect archive type first
6033     my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
6034         die "unable to open file '$archive'\n";
6035     my $firstfile = <$fh>;
6036     kill 15, $pid;
6037     close $fh;
6038
6039     die "ERROR: archive contaions no data\n" if !$firstfile;
6040     chomp $firstfile;
6041
6042     return $firstfile;
6043 }
6044
6045 sub tar_restore_cleanup {
6046     my ($storecfg, $statfile) = @_;
6047
6048     print STDERR "starting cleanup\n";
6049
6050     if (my $fd = IO::File->new($statfile, "r")) {
6051         while (defined(my $line = <$fd>)) {
6052             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
6053                 my $volid = $2;
6054                 eval {
6055                     if ($volid =~ m|^/|) {
6056                         unlink $volid || die 'unlink failed\n';
6057                     } else {
6058                         PVE::Storage::vdisk_free($storecfg, $volid);
6059                     }
6060                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
6061                 };
6062                 print STDERR "unable to cleanup '$volid' - $@" if $@;
6063             } else {
6064                 print STDERR "unable to parse line in statfile - $line";
6065             }
6066         }
6067         $fd->close();
6068     }
6069 }
6070
6071 sub restore_archive {
6072     my ($archive, $vmid, $user, $opts) = @_;
6073
6074     my $format = $opts->{format};
6075     my $comp;
6076
6077     if ($archive =~ m/\.tgz$/ || $archive =~ m/\.tar\.gz$/) {
6078         $format = 'tar' if !$format;
6079         $comp = 'gzip';
6080     } elsif ($archive =~ m/\.tar$/) {
6081         $format = 'tar' if !$format;
6082     } elsif ($archive =~ m/.tar.lzo$/) {
6083         $format = 'tar' if !$format;
6084         $comp = 'lzop';
6085     } elsif ($archive =~ m/\.vma$/) {
6086         $format = 'vma' if !$format;
6087     } elsif ($archive =~ m/\.vma\.gz$/) {
6088         $format = 'vma' if !$format;
6089         $comp = 'gzip';
6090     } elsif ($archive =~ m/\.vma\.lzo$/) {
6091         $format = 'vma' if !$format;
6092         $comp = 'lzop';
6093     } else {
6094         $format = 'vma' if !$format; # default
6095     }
6096
6097     # try to detect archive format
6098     if ($format eq 'tar') {
6099         return restore_tar_archive($archive, $vmid, $user, $opts);
6100     } else {
6101         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
6102     }
6103 }
6104
6105 sub restore_update_config_line {
6106     my ($outfd, $cookie, $vmid, $map, $line, $unique) = @_;
6107
6108     return if $line =~ m/^\#qmdump\#/;
6109     return if $line =~ m/^\#vzdump\#/;
6110     return if $line =~ m/^lock:/;
6111     return if $line =~ m/^unused\d+:/;
6112     return if $line =~ m/^parent:/;
6113
6114     my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
6115     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
6116         # try to convert old 1.X settings
6117         my ($id, $ind, $ethcfg) = ($1, $2, $3);
6118         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
6119             my ($model, $macaddr) = split(/\=/, $devconfig);
6120             $macaddr = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if !$macaddr || $unique;
6121             my $net = {
6122                 model => $model,
6123                 bridge => "vmbr$ind",
6124                 macaddr => $macaddr,
6125             };
6126             my $netstr = print_net($net);
6127
6128             print $outfd "net$cookie->{netcount}: $netstr\n";
6129             $cookie->{netcount}++;
6130         }
6131     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
6132         my ($id, $netstr) = ($1, $2);
6133         my $net = parse_net($netstr);
6134         $net->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if $net->{macaddr};
6135         $netstr = print_net($net);
6136         print $outfd "$id: $netstr\n";
6137     } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk)\d+):\s*(\S+)\s*$/) {
6138         my $virtdev = $1;
6139         my $value = $3;
6140         my $di = parse_drive($virtdev, $value);
6141         if (defined($di->{backup}) && !$di->{backup}) {
6142             print $outfd "#$line";
6143         } elsif ($map->{$virtdev}) {
6144             delete $di->{format}; # format can change on restore
6145             $di->{file} = $map->{$virtdev};
6146             $value = print_drive($vmid, $di);
6147             print $outfd "$virtdev: $value\n";
6148         } else {
6149             print $outfd $line;
6150         }
6151     } elsif (($line =~ m/^vmgenid: (.*)/)) {
6152         my $vmgenid = $1;
6153         if ($vmgenid ne '0') {
6154             # always generate a new vmgenid if there was a valid one setup
6155             $vmgenid = generate_uuid();
6156         }
6157         print $outfd "vmgenid: $vmgenid\n";
6158     } elsif (($line =~ m/^(smbios1: )(.*)/) && $unique) {
6159         my ($uuid, $uuid_str);
6160         UUID::generate($uuid);
6161         UUID::unparse($uuid, $uuid_str);
6162         my $smbios1 = parse_smbios1($2);
6163         $smbios1->{uuid} = $uuid_str;
6164         print $outfd $1.print_smbios1($smbios1)."\n";
6165     } else {
6166         print $outfd $line;
6167     }
6168 }
6169
6170 sub scan_volids {
6171     my ($cfg, $vmid) = @_;
6172
6173     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid);
6174
6175     my $volid_hash = {};
6176     foreach my $storeid (keys %$info) {
6177         foreach my $item (@{$info->{$storeid}}) {
6178             next if !($item->{volid} && $item->{size});
6179             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
6180             $volid_hash->{$item->{volid}} = $item;
6181         }
6182     }
6183
6184     return $volid_hash;
6185 }
6186
6187 sub is_volume_in_use {
6188     my ($storecfg, $conf, $skip_drive, $volid) = @_;
6189
6190     my $path = PVE::Storage::path($storecfg, $volid);
6191
6192     my $scan_config = sub {
6193         my ($cref, $snapname) = @_;
6194
6195         foreach my $key (keys %$cref) {
6196             my $value = $cref->{$key};
6197             if (is_valid_drivename($key)) {
6198                 next if $skip_drive && $key eq $skip_drive;
6199                 my $drive = parse_drive($key, $value);
6200                 next if !$drive || !$drive->{file} || drive_is_cdrom($drive);
6201                 return 1 if $volid eq $drive->{file};
6202                 if ($drive->{file} =~ m!^/!) {
6203                     return 1 if $drive->{file} eq $path;
6204                 } else {
6205                     my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file}, 1);
6206                     next if !$storeid;
6207                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid, 1);
6208                     next if !$scfg;
6209                     return 1 if $path eq PVE::Storage::path($storecfg, $drive->{file}, $snapname);
6210                 }
6211             }
6212         }
6213
6214         return 0;
6215     };
6216
6217     return 1 if &$scan_config($conf);
6218
6219     undef $skip_drive;
6220
6221     foreach my $snapname (keys %{$conf->{snapshots}}) {
6222         return 1 if &$scan_config($conf->{snapshots}->{$snapname}, $snapname);
6223     }
6224
6225     return 0;
6226 }
6227
6228 sub update_disksize {
6229     my ($vmid, $conf, $volid_hash) = @_;
6230
6231     my $changes;
6232     my $prefix = "VM $vmid:";
6233
6234     # used and unused disks
6235     my $referenced = {};
6236
6237     # Note: it is allowed to define multiple storages with same path (alias), so
6238     # we need to check both 'volid' and real 'path' (two different volid can point
6239     # to the same path).
6240
6241     my $referencedpath = {};
6242
6243     # update size info
6244     foreach my $opt (keys %$conf) {
6245         if (is_valid_drivename($opt)) {
6246             my $drive = parse_drive($opt, $conf->{$opt});
6247             my $volid = $drive->{file};
6248             next if !$volid;
6249
6250             $referenced->{$volid} = 1;
6251             if ($volid_hash->{$volid} &&
6252                 (my $path = $volid_hash->{$volid}->{path})) {
6253                 $referencedpath->{$path} = 1;
6254             }
6255
6256             next if drive_is_cdrom($drive);
6257             next if !$volid_hash->{$volid};
6258
6259             $drive->{size} = $volid_hash->{$volid}->{size};
6260             my $new = print_drive($vmid, $drive);
6261             if ($new ne $conf->{$opt}) {
6262                 $changes = 1;
6263                 $conf->{$opt} = $new;
6264                 print "$prefix update disk '$opt' information.\n";
6265             }
6266         }
6267     }
6268
6269     # remove 'unusedX' entry if volume is used
6270     foreach my $opt (keys %$conf) {
6271         next if $opt !~ m/^unused\d+$/;
6272         my $volid = $conf->{$opt};
6273         my $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
6274         if ($referenced->{$volid} || ($path && $referencedpath->{$path})) {
6275             print "$prefix remove entry '$opt', its volume '$volid' is in use.\n";
6276             $changes = 1;
6277             delete $conf->{$opt};
6278         }
6279
6280         $referenced->{$volid} = 1;
6281         $referencedpath->{$path} = 1 if $path;
6282     }
6283
6284     foreach my $volid (sort keys %$volid_hash) {
6285         next if $volid =~ m/vm-$vmid-state-/;
6286         next if $referenced->{$volid};
6287         my $path = $volid_hash->{$volid}->{path};
6288         next if !$path; # just to be sure
6289         next if $referencedpath->{$path};
6290         $changes = 1;
6291         my $key = PVE::QemuConfig->add_unused_volume($conf, $volid);
6292         print "$prefix add unreferenced volume '$volid' as '$key' to config.\n";
6293         $referencedpath->{$path} = 1; # avoid to add more than once (aliases)
6294     }
6295
6296     return $changes;
6297 }
6298
6299 sub rescan {
6300     my ($vmid, $nolock, $dryrun) = @_;
6301
6302     my $cfg = PVE::Storage::config();
6303
6304     # FIXME: Remove once our RBD plugin can handle CT and VM on a single storage
6305     # see: https://pve.proxmox.com/pipermail/pve-devel/2018-July/032900.html
6306     foreach my $stor (keys %{$cfg->{ids}}) {
6307         delete($cfg->{ids}->{$stor}) if ! $cfg->{ids}->{$stor}->{content}->{images};
6308     }
6309
6310     print "rescan volumes...\n";
6311     my $volid_hash = scan_volids($cfg, $vmid);
6312
6313     my $updatefn =  sub {
6314         my ($vmid) = @_;
6315
6316         my $conf = PVE::QemuConfig->load_config($vmid);
6317
6318         PVE::QemuConfig->check_lock($conf);
6319
6320         my $vm_volids = {};
6321         foreach my $volid (keys %$volid_hash) {
6322             my $info = $volid_hash->{$volid};
6323             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
6324         }
6325
6326         my $changes = update_disksize($vmid, $conf, $vm_volids);
6327
6328         PVE::QemuConfig->write_config($vmid, $conf) if $changes && !$dryrun;
6329     };
6330
6331     if (defined($vmid)) {
6332         if ($nolock) {
6333             &$updatefn($vmid);
6334         } else {
6335             PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6336         }
6337     } else {
6338         my $vmlist = config_list();
6339         foreach my $vmid (keys %$vmlist) {
6340             if ($nolock) {
6341                 &$updatefn($vmid);
6342             } else {
6343                 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6344             }
6345         }
6346     }
6347 }
6348
6349 sub restore_vma_archive {
6350     my ($archive, $vmid, $user, $opts, $comp) = @_;
6351
6352     my $readfrom = $archive;
6353
6354     my $cfg = PVE::Storage::config();
6355     my $commands = [];
6356     my $bwlimit = $opts->{bwlimit};
6357
6358     my $dbg_cmdstring = '';
6359     my $add_pipe = sub {
6360         my ($cmd) = @_;
6361         push @$commands, $cmd;
6362         $dbg_cmdstring .= ' | ' if length($dbg_cmdstring);
6363         $dbg_cmdstring .= PVE::Tools::cmd2string($cmd);
6364         $readfrom = '-';
6365     };
6366
6367     my $input = undef;
6368     if ($archive eq '-') {
6369         $input = '<&STDIN';
6370     } else {
6371         # If we use a backup from a PVE defined storage we also consider that
6372         # storage's rate limit:
6373         my (undef, $volid) = PVE::Storage::path_to_volume_id($cfg, $archive);
6374         if (defined($volid)) {
6375             my ($sid, undef) = PVE::Storage::parse_volume_id($volid);
6376             my $readlimit = PVE::Storage::get_bandwidth_limit('restore', [$sid], $bwlimit);
6377             if ($readlimit) {
6378                 print STDERR "applying read rate limit: $readlimit\n";
6379                 my $cstream = ['cstream', '-t', $readlimit*1024, '--', $readfrom];
6380                 $add_pipe->($cstream);
6381             }
6382         }
6383     }
6384
6385     if ($comp) {
6386         my $cmd;
6387         if ($comp eq 'gzip') {
6388             $cmd = ['zcat', $readfrom];
6389         } elsif ($comp eq 'lzop') {
6390             $cmd = ['lzop', '-d', '-c', $readfrom];
6391         } else {
6392             die "unknown compression method '$comp'\n";
6393         }
6394         $add_pipe->($cmd);
6395     }
6396
6397     my $tmpdir = "/var/tmp/vzdumptmp$$";
6398     rmtree $tmpdir;
6399
6400     # disable interrupts (always do cleanups)
6401     local $SIG{INT} =
6402         local $SIG{TERM} =
6403         local $SIG{QUIT} =
6404         local $SIG{HUP} = sub { warn "got interrupt - ignored\n"; };
6405
6406     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
6407     POSIX::mkfifo($mapfifo, 0600);
6408     my $fifofh;
6409
6410     my $openfifo = sub {
6411         open($fifofh, '>', $mapfifo) || die $!;
6412     };
6413
6414     $add_pipe->(['vma', 'extract', '-v', '-r', $mapfifo, $readfrom, $tmpdir]);
6415
6416     my $oldtimeout;
6417     my $timeout = 5;
6418
6419     my $devinfo = {};
6420
6421     my $rpcenv = PVE::RPCEnvironment::get();
6422
6423     my $conffile = PVE::QemuConfig->config_file($vmid);
6424     my $tmpfn = "$conffile.$$.tmp";
6425
6426     # Note: $oldconf is undef if VM does not exists
6427     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
6428     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
6429
6430     my %storage_limits;
6431
6432     my $print_devmap = sub {
6433         my $virtdev_hash = {};
6434
6435         my $cfgfn = "$tmpdir/qemu-server.conf";
6436
6437         # we can read the config - that is already extracted
6438         my $fh = IO::File->new($cfgfn, "r") ||
6439             "unable to read qemu-server.conf - $!\n";
6440
6441         my $fwcfgfn = "$tmpdir/qemu-server.fw";
6442         if (-f $fwcfgfn) {
6443             my $pve_firewall_dir = '/etc/pve/firewall';
6444             mkdir $pve_firewall_dir; # make sure the dir exists
6445             PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
6446         }
6447
6448         while (defined(my $line = <$fh>)) {
6449             if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
6450                 my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
6451                 die "archive does not contain data for drive '$virtdev'\n"
6452                     if !$devinfo->{$devname};
6453                 if (defined($opts->{storage})) {
6454                     $storeid = $opts->{storage} || 'local';
6455                 } elsif (!$storeid) {
6456                     $storeid = 'local';
6457                 }
6458                 $format = 'raw' if !$format;
6459                 $devinfo->{$devname}->{devname} = $devname;
6460                 $devinfo->{$devname}->{virtdev} = $virtdev;
6461                 $devinfo->{$devname}->{format} = $format;
6462                 $devinfo->{$devname}->{storeid} = $storeid;
6463
6464                 # check permission on storage
6465                 my $pool = $opts->{pool}; # todo: do we need that?
6466                 if ($user ne 'root@pam') {
6467                     $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
6468                 }
6469
6470                 $storage_limits{$storeid} = $bwlimit;
6471
6472                 $virtdev_hash->{$virtdev} = $devinfo->{$devname};
6473             } elsif ($line =~ m/^((?:ide|sata|scsi)\d+):\s*(.*)\s*$/) {
6474                 my $virtdev = $1;
6475                 my $drive = parse_drive($virtdev, $2);
6476                 if (drive_is_cloudinit($drive)) {
6477                     my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6478                     my $scfg = PVE::Storage::storage_config($cfg, $storeid);
6479                     my $format = qemu_img_format($scfg, $volname); # has 'raw' fallback
6480
6481                     my $d = {
6482                         format => $format,
6483                         storeid => $opts->{storage} // $storeid,
6484                         size => PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE,
6485                         file => $drive->{file}, # to make drive_is_cloudinit check possible
6486                         name => "vm-$vmid-cloudinit",
6487                         is_cloudinit => 1,
6488                     };
6489                     $virtdev_hash->{$virtdev} = $d;
6490                 }
6491             }
6492         }
6493
6494         foreach my $key (keys %storage_limits) {
6495             my $limit = PVE::Storage::get_bandwidth_limit('restore', [$key], $bwlimit);
6496             next if !$limit;
6497             print STDERR "rate limit for storage $key: $limit KiB/s\n";
6498             $storage_limits{$key} = $limit * 1024;
6499         }
6500
6501         foreach my $devname (keys %$devinfo) {
6502             die "found no device mapping information for device '$devname'\n"
6503                 if !$devinfo->{$devname}->{virtdev};
6504         }
6505
6506         # create empty/temp config
6507         if ($oldconf) {
6508             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
6509             foreach_drive($oldconf, sub {
6510                 my ($ds, $drive) = @_;
6511
6512                 return if drive_is_cdrom($drive, 1);
6513
6514                 my $volid = $drive->{file};
6515                 return if !$volid || $volid =~ m|^/|;
6516
6517                 my ($path, $owner) = PVE::Storage::path($cfg, $volid);
6518                 return if !$path || !$owner || ($owner != $vmid);
6519
6520                 # Note: only delete disk we want to restore
6521                 # other volumes will become unused
6522                 if ($virtdev_hash->{$ds}) {
6523                     eval { PVE::Storage::vdisk_free($cfg, $volid); };
6524                     if (my $err = $@) {
6525                         warn $err;
6526                     }
6527                 }
6528             });
6529
6530             # delete vmstate files, after the restore we have no snapshots anymore
6531             foreach my $snapname (keys %{$oldconf->{snapshots}}) {
6532                 my $snap = $oldconf->{snapshots}->{$snapname};
6533                 if ($snap->{vmstate}) {
6534                     eval { PVE::Storage::vdisk_free($cfg, $snap->{vmstate}); };
6535                     if (my $err = $@) {
6536                         warn $err;
6537                     }
6538                 }
6539             }
6540         }
6541
6542         my $map = {};
6543         foreach my $virtdev (sort keys %$virtdev_hash) {
6544             my $d = $virtdev_hash->{$virtdev};
6545             my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
6546             my $storeid = $d->{storeid};
6547             my $scfg = PVE::Storage::storage_config($cfg, $storeid);
6548
6549             my $map_opts = '';
6550             if (my $limit = $storage_limits{$storeid}) {
6551                 $map_opts .= "throttling.bps=$limit:throttling.group=$storeid:";
6552             }
6553
6554             # test if requested format is supported
6555             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($cfg, $storeid);
6556             my $supported = grep { $_ eq $d->{format} } @$validFormats;
6557             $d->{format} = $defFormat if !$supported;
6558
6559             my $name;
6560             if ($d->{is_cloudinit}) {
6561                 $name = $d->{name};
6562                 $name .= ".$d->{format}" if $d->{format} ne 'raw';
6563             }
6564
6565             my $volid = PVE::Storage::vdisk_alloc($cfg, $storeid, $vmid, $d->{format}, $name, $alloc_size);
6566             print STDERR "new volume ID is '$volid'\n";
6567             $d->{volid} = $volid;
6568
6569             PVE::Storage::activate_volumes($cfg, [$volid]);
6570
6571             my $write_zeros = 1;
6572             if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
6573                 $write_zeros = 0;
6574             }
6575
6576             if (!$d->{is_cloudinit}) {
6577                 my $path = PVE::Storage::path($cfg, $volid);
6578
6579                 print $fifofh "${map_opts}format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
6580
6581                 print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
6582             }
6583             $map->{$virtdev} = $volid;
6584         }
6585
6586         $fh->seek(0, 0) || die "seek failed - $!\n";
6587
6588         my $outfd = new IO::File ($tmpfn, "w") ||
6589             die "unable to write config for VM $vmid\n";
6590
6591         my $cookie = { netcount => 0 };
6592         while (defined(my $line = <$fh>)) {
6593             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
6594         }
6595
6596         $fh->close();
6597         $outfd->close();
6598     };
6599
6600     eval {
6601         # enable interrupts
6602         local $SIG{INT} =
6603             local $SIG{TERM} =
6604             local $SIG{QUIT} =
6605             local $SIG{HUP} =
6606             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6607         local $SIG{ALRM} = sub { die "got timeout\n"; };
6608
6609         $oldtimeout = alarm($timeout);
6610
6611         my $parser = sub {
6612             my $line = shift;
6613
6614             print "$line\n";
6615
6616             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
6617                 my ($dev_id, $size, $devname) = ($1, $2, $3);
6618                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
6619             } elsif ($line =~ m/^CTIME: /) {
6620                 # we correctly received the vma config, so we can disable
6621                 # the timeout now for disk allocation (set to 10 minutes, so
6622                 # that we always timeout if something goes wrong)
6623                 alarm(600);
6624                 &$print_devmap();
6625                 print $fifofh "done\n";
6626                 my $tmp = $oldtimeout || 0;
6627                 $oldtimeout = undef;
6628                 alarm($tmp);
6629                 close($fifofh);
6630             }
6631         };
6632
6633         print "restore vma archive: $dbg_cmdstring\n";
6634         run_command($commands, input => $input, outfunc => $parser, afterfork => $openfifo);
6635     };
6636     my $err = $@;
6637
6638     alarm($oldtimeout) if $oldtimeout;
6639
6640     my $vollist = [];
6641     foreach my $devname (keys %$devinfo) {
6642         my $volid = $devinfo->{$devname}->{volid};
6643         push @$vollist, $volid if $volid;
6644     }
6645
6646     PVE::Storage::deactivate_volumes($cfg, $vollist);
6647
6648     unlink $mapfifo;
6649
6650     if ($err) {
6651         rmtree $tmpdir;
6652         unlink $tmpfn;
6653
6654         foreach my $devname (keys %$devinfo) {
6655             my $volid = $devinfo->{$devname}->{volid};
6656             next if !$volid;
6657             eval {
6658                 if ($volid =~ m|^/|) {
6659                     unlink $volid || die 'unlink failed\n';
6660                 } else {
6661                     PVE::Storage::vdisk_free($cfg, $volid);
6662                 }
6663                 print STDERR "temporary volume '$volid' sucessfuly removed\n";
6664             };
6665             print STDERR "unable to cleanup '$volid' - $@" if $@;
6666         }
6667         die $err;
6668     }
6669
6670     rmtree $tmpdir;
6671
6672     rename($tmpfn, $conffile) ||
6673         die "unable to commit configuration file '$conffile'\n";
6674
6675     PVE::Cluster::cfs_update(); # make sure we read new file
6676
6677     eval { rescan($vmid, 1); };
6678     warn $@ if $@;
6679 }
6680
6681 sub restore_tar_archive {
6682     my ($archive, $vmid, $user, $opts) = @_;
6683
6684     if ($archive ne '-') {
6685         my $firstfile = tar_archive_read_firstfile($archive);
6686         die "ERROR: file '$archive' dos not lock like a QemuServer vzdump backup\n"
6687             if $firstfile ne 'qemu-server.conf';
6688     }
6689
6690     my $storecfg = PVE::Storage::config();
6691
6692     # destroy existing data - keep empty config
6693     my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
6694     destroy_vm($storecfg, $vmid, 1) if -f $vmcfgfn;
6695
6696     my $tocmd = "/usr/lib/qemu-server/qmextract";
6697
6698     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
6699     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
6700     $tocmd .= ' --prealloc' if $opts->{prealloc};
6701     $tocmd .= ' --info' if $opts->{info};
6702
6703     # tar option "xf" does not autodetect compression when read from STDIN,
6704     # so we pipe to zcat
6705     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
6706         PVE::Tools::shellquote("--to-command=$tocmd");
6707
6708     my $tmpdir = "/var/tmp/vzdumptmp$$";
6709     mkpath $tmpdir;
6710
6711     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
6712     local $ENV{VZDUMP_VMID} = $vmid;
6713     local $ENV{VZDUMP_USER} = $user;
6714
6715     my $conffile = PVE::QemuConfig->config_file($vmid);
6716     my $tmpfn = "$conffile.$$.tmp";
6717
6718     # disable interrupts (always do cleanups)
6719     local $SIG{INT} =
6720         local $SIG{TERM} =
6721         local $SIG{QUIT} =
6722         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
6723
6724     eval {
6725         # enable interrupts
6726         local $SIG{INT} =
6727             local $SIG{TERM} =
6728             local $SIG{QUIT} =
6729             local $SIG{HUP} =
6730             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6731
6732         if ($archive eq '-') {
6733             print "extracting archive from STDIN\n";
6734             run_command($cmd, input => "<&STDIN");
6735         } else {
6736             print "extracting archive '$archive'\n";
6737             run_command($cmd);
6738         }
6739
6740         return if $opts->{info};
6741
6742         # read new mapping
6743         my $map = {};
6744         my $statfile = "$tmpdir/qmrestore.stat";
6745         if (my $fd = IO::File->new($statfile, "r")) {
6746             while (defined (my $line = <$fd>)) {
6747                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
6748                     $map->{$1} = $2 if $1;
6749                 } else {
6750                     print STDERR "unable to parse line in statfile - $line\n";
6751                 }
6752             }
6753             $fd->close();
6754         }
6755
6756         my $confsrc = "$tmpdir/qemu-server.conf";
6757
6758         my $srcfd = new IO::File($confsrc, "r") ||
6759             die "unable to open file '$confsrc'\n";
6760
6761         my $outfd = new IO::File ($tmpfn, "w") ||
6762             die "unable to write config for VM $vmid\n";
6763
6764         my $cookie = { netcount => 0 };
6765         while (defined (my $line = <$srcfd>)) {
6766             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
6767         }
6768
6769         $srcfd->close();
6770         $outfd->close();
6771     };
6772     my $err = $@;
6773
6774     if ($err) {
6775
6776         unlink $tmpfn;
6777
6778         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
6779
6780         die $err;
6781     }
6782
6783     rmtree $tmpdir;
6784
6785     rename $tmpfn, $conffile ||
6786         die "unable to commit configuration file '$conffile'\n";
6787
6788     PVE::Cluster::cfs_update(); # make sure we read new file
6789
6790     eval { rescan($vmid, 1); };
6791     warn $@ if $@;
6792 };
6793
6794 sub foreach_storage_used_by_vm {
6795     my ($conf, $func) = @_;
6796
6797     my $sidhash = {};
6798
6799     foreach_drive($conf, sub {
6800         my ($ds, $drive) = @_;
6801         return if drive_is_cdrom($drive);
6802
6803         my $volid = $drive->{file};
6804
6805         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
6806         $sidhash->{$sid} = $sid if $sid;
6807     });
6808
6809     foreach my $sid (sort keys %$sidhash) {
6810         &$func($sid);
6811     }
6812 }
6813
6814 sub do_snapshots_with_qemu {
6815     my ($storecfg, $volid) = @_;
6816
6817     my $storage_name = PVE::Storage::parse_volume_id($volid);
6818     my $scfg = $storecfg->{ids}->{$storage_name};
6819
6820     if ($qemu_snap_storage->{$scfg->{type}} && !$scfg->{krbd}){
6821         return 1;
6822     }
6823
6824     if ($volid =~ m/\.(qcow2|qed)$/){
6825         return 1;
6826     }
6827
6828     return undef;
6829 }
6830
6831 sub qga_check_running {
6832     my ($vmid, $nowarn) = @_;
6833
6834     eval { vm_mon_cmd($vmid, "guest-ping", timeout => 3); };
6835     if ($@) {
6836         warn "Qemu Guest Agent is not running - $@" if !$nowarn;
6837         return 0;
6838     }
6839     return 1;
6840 }
6841
6842 sub template_create {
6843     my ($vmid, $conf, $disk) = @_;
6844
6845     my $storecfg = PVE::Storage::config();
6846
6847     foreach_drive($conf, sub {
6848         my ($ds, $drive) = @_;
6849
6850         return if drive_is_cdrom($drive);
6851         return if $disk && $ds ne $disk;
6852
6853         my $volid = $drive->{file};
6854         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
6855
6856         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
6857         $drive->{file} = $voliddst;
6858         $conf->{$ds} = print_drive($vmid, $drive);
6859         PVE::QemuConfig->write_config($vmid, $conf);
6860     });
6861 }
6862
6863 sub convert_iscsi_path {
6864     my ($path) = @_;
6865
6866     if ($path =~ m|^iscsi://([^/]+)/([^/]+)/(.+)$|) {
6867         my $portal = $1;
6868         my $target = $2;
6869         my $lun = $3;
6870
6871         my $initiator_name = get_initiator_name();
6872
6873         return "file.driver=iscsi,file.transport=tcp,file.initiator-name=$initiator_name,".
6874                "file.portal=$portal,file.target=$target,file.lun=$lun,driver=raw";
6875     }
6876
6877     die "cannot convert iscsi path '$path', unkown format\n";
6878 }
6879
6880 sub qemu_img_convert {
6881     my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized) = @_;
6882
6883     my $storecfg = PVE::Storage::config();
6884     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
6885     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
6886
6887     die "destination '$dst_volid' is not a valid volid form qemu-img convert\n" if !$dst_storeid;
6888
6889     my $cachemode;
6890     my $src_path;
6891     my $src_is_iscsi = 0;
6892     my $src_format = 'raw';
6893
6894     if ($src_storeid) {
6895         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
6896         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
6897         $src_format = qemu_img_format($src_scfg, $src_volname);
6898         $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
6899         $src_is_iscsi = ($src_path =~ m|^iscsi://|);
6900         $cachemode = 'none' if $src_scfg->{type} eq 'zfspool';
6901     } elsif (-f $src_volid) {
6902         $src_path = $src_volid;
6903         if ($src_path =~ m/\.($QEMU_FORMAT_RE)$/) {
6904             $src_format = $1;
6905         }
6906     }
6907
6908     die "source '$src_volid' is not a valid volid nor path for qemu-img convert\n" if !$src_path;
6909
6910     my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6911     my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
6912     my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6913     my $dst_is_iscsi = ($dst_path =~ m|^iscsi://|);
6914
6915     my $cmd = [];
6916     push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
6917     push @$cmd, '-l', "snapshot.name=$snapname" if($snapname && $src_format eq "qcow2");
6918     push @$cmd, '-t', 'none' if $dst_scfg->{type} eq 'zfspool';
6919     push @$cmd, '-T', $cachemode if defined($cachemode);
6920
6921     if ($src_is_iscsi) {
6922         push @$cmd, '--image-opts';
6923         $src_path = convert_iscsi_path($src_path);
6924     } else {
6925         push @$cmd, '-f', $src_format;
6926     }
6927
6928     if ($dst_is_iscsi) {
6929         push @$cmd, '--target-image-opts';
6930         $dst_path = convert_iscsi_path($dst_path);
6931     } else {
6932         push @$cmd, '-O', $dst_format;
6933     }
6934
6935     push @$cmd, $src_path;
6936
6937     if (!$dst_is_iscsi && $is_zero_initialized) {
6938         push @$cmd, "zeroinit:$dst_path";
6939     } else {
6940         push @$cmd, $dst_path;
6941     }
6942
6943     my $parser = sub {
6944         my $line = shift;
6945         if($line =~ m/\((\S+)\/100\%\)/){
6946             my $percent = $1;
6947             my $transferred = int($size * $percent / 100);
6948             my $remaining = $size - $transferred;
6949
6950             print "transferred: $transferred bytes remaining: $remaining bytes total: $size bytes progression: $percent %\n";
6951         }
6952
6953     };
6954
6955     eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
6956     my $err = $@;
6957     die "copy failed: $err" if $err;
6958 }
6959
6960 sub qemu_img_format {
6961     my ($scfg, $volname) = @_;
6962
6963     if ($scfg->{path} && $volname =~ m/\.($QEMU_FORMAT_RE)$/) {
6964         return $1;
6965     } else {
6966         return "raw";
6967     }
6968 }
6969
6970 sub qemu_drive_mirror {
6971     my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $skipcomplete, $qga, $bwlimit) = @_;
6972
6973     $jobs = {} if !$jobs;
6974
6975     my $qemu_target;
6976     my $format;
6977     $jobs->{"drive-$drive"} = {};
6978
6979     if ($dst_volid =~ /^nbd:/) {
6980         $qemu_target = $dst_volid;
6981         $format = "nbd";
6982     } else {
6983         my $storecfg = PVE::Storage::config();
6984         my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
6985
6986         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6987
6988         $format = qemu_img_format($dst_scfg, $dst_volname);
6989
6990         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6991
6992         $qemu_target = $is_zero_initialized ? "zeroinit:$dst_path" : $dst_path;
6993     }
6994
6995     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $qemu_target };
6996     $opts->{format} = $format if $format;
6997
6998     if (defined($bwlimit)) {
6999         $opts->{speed} = $bwlimit * 1024;
7000         print "drive mirror is starting for drive-$drive with bandwidth limit: ${bwlimit} KB/s\n";
7001     } else {
7002         print "drive mirror is starting for drive-$drive\n";
7003     }
7004
7005     # if a job already runs for this device we get an error, catch it for cleanup
7006     eval { vm_mon_cmd($vmid, "drive-mirror", %$opts); };
7007     if (my $err = $@) {
7008         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
7009         warn "$@\n" if $@;
7010         die "mirroring error: $err\n";
7011     }
7012
7013     qemu_drive_mirror_monitor ($vmid, $vmiddst, $jobs, $skipcomplete, $qga);
7014 }
7015
7016 sub qemu_drive_mirror_monitor {
7017     my ($vmid, $vmiddst, $jobs, $skipcomplete, $qga) = @_;
7018
7019     eval {
7020         my $err_complete = 0;
7021
7022         while (1) {
7023             die "storage migration timed out\n" if $err_complete > 300;
7024
7025             my $stats = vm_mon_cmd($vmid, "query-block-jobs");
7026
7027             my $running_mirror_jobs = {};
7028             foreach my $stat (@$stats) {
7029                 next if $stat->{type} ne 'mirror';
7030                 $running_mirror_jobs->{$stat->{device}} = $stat;
7031             }
7032
7033             my $readycounter = 0;
7034
7035             foreach my $job (keys %$jobs) {
7036
7037                 if(defined($jobs->{$job}->{complete}) && !defined($running_mirror_jobs->{$job})) {
7038                     print "$job : finished\n";
7039                     delete $jobs->{$job};
7040                     next;
7041                 }
7042
7043                 die "$job: mirroring has been cancelled\n" if !defined($running_mirror_jobs->{$job});
7044
7045                 my $busy = $running_mirror_jobs->{$job}->{busy};
7046                 my $ready = $running_mirror_jobs->{$job}->{ready};
7047                 if (my $total = $running_mirror_jobs->{$job}->{len}) {
7048                     my $transferred = $running_mirror_jobs->{$job}->{offset} || 0;
7049                     my $remaining = $total - $transferred;
7050                     my $percent = sprintf "%.2f", ($transferred * 100 / $total);
7051
7052                     print "$job: transferred: $transferred bytes remaining: $remaining bytes total: $total bytes progression: $percent % busy: $busy ready: $ready \n";
7053                 }
7054
7055                 $readycounter++ if $running_mirror_jobs->{$job}->{ready};
7056             }
7057
7058             last if scalar(keys %$jobs) == 0;
7059
7060             if ($readycounter == scalar(keys %$jobs)) {
7061                 print "all mirroring jobs are ready \n";
7062                 last if $skipcomplete; #do the complete later
7063
7064                 if ($vmiddst && $vmiddst != $vmid) {
7065                     my $agent_running = $qga && qga_check_running($vmid);
7066                     if ($agent_running) {
7067                         print "freeze filesystem\n";
7068                         eval { PVE::QemuServer::vm_mon_cmd($vmid, "guest-fsfreeze-freeze"); };
7069                     } else {
7070                         print "suspend vm\n";
7071                         eval { PVE::QemuServer::vm_suspend($vmid, 1); };
7072                     }
7073
7074                     # if we clone a disk for a new target vm, we don't switch the disk
7075                     PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs);
7076
7077                     if ($agent_running) {
7078                         print "unfreeze filesystem\n";
7079                         eval { PVE::QemuServer::vm_mon_cmd($vmid, "guest-fsfreeze-thaw"); };
7080                     } else {
7081                         print "resume vm\n";
7082                         eval {  PVE::QemuServer::vm_resume($vmid, 1, 1); };
7083                     }
7084
7085                     last;
7086                 } else {
7087
7088                     foreach my $job (keys %$jobs) {
7089                         # try to switch the disk if source and destination are on the same guest
7090                         print "$job: Completing block job...\n";
7091
7092                         eval { vm_mon_cmd($vmid, "block-job-complete", device => $job) };
7093                         if ($@ =~ m/cannot be completed/) {
7094                             print "$job: Block job cannot be completed, try again.\n";
7095                             $err_complete++;
7096                         }else {
7097                             print "$job: Completed successfully.\n";
7098                             $jobs->{$job}->{complete} = 1;
7099                         }
7100                     }
7101                 }
7102             }
7103             sleep 1;
7104         }
7105     };
7106     my $err = $@;
7107
7108     if ($err) {
7109         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
7110         die "mirroring error: $err";
7111     }
7112
7113 }
7114
7115 sub qemu_blockjobs_cancel {
7116     my ($vmid, $jobs) = @_;
7117
7118     foreach my $job (keys %$jobs) {
7119         print "$job: Cancelling block job\n";
7120         eval { vm_mon_cmd($vmid, "block-job-cancel", device => $job); };
7121         $jobs->{$job}->{cancel} = 1;
7122     }
7123
7124     while (1) {
7125         my $stats = vm_mon_cmd($vmid, "query-block-jobs");
7126
7127         my $running_jobs = {};
7128         foreach my $stat (@$stats) {
7129             $running_jobs->{$stat->{device}} = $stat;
7130         }
7131
7132         foreach my $job (keys %$jobs) {
7133
7134             if (defined($jobs->{$job}->{cancel}) && !defined($running_jobs->{$job})) {
7135                 print "$job: Done.\n";
7136                 delete $jobs->{$job};
7137             }
7138         }
7139
7140         last if scalar(keys %$jobs) == 0;
7141
7142         sleep 1;
7143     }
7144 }
7145
7146 sub clone_disk {
7147     my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
7148         $newvmid, $storage, $format, $full, $newvollist, $jobs, $skipcomplete, $qga, $bwlimit) = @_;
7149
7150     my $newvolid;
7151
7152     if (!$full) {
7153         print "create linked clone of drive $drivename ($drive->{file})\n";
7154         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
7155         push @$newvollist, $newvolid;
7156     } else {
7157
7158         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
7159         $storeid = $storage if $storage;
7160
7161         my $dst_format = resolve_dst_disk_format($storecfg, $storeid, $volname, $format);
7162         my ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 3);
7163
7164         print "create full clone of drive $drivename ($drive->{file})\n";
7165         my $name = undef;
7166         $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $newvmid, $dst_format, $name, ($size/1024));
7167         push @$newvollist, $newvolid;
7168
7169         PVE::Storage::activate_volumes($storecfg, [$newvolid]);
7170
7171         my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
7172         if (!$running || $snapname) {
7173             # TODO: handle bwlimits
7174             qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit);
7175         } else {
7176
7177             my $kvmver = get_running_qemu_version ($vmid);
7178             if (!qemu_machine_feature_enabled (undef, $kvmver, 2, 7)) {
7179                 die "drive-mirror with iothread requires qemu version 2.7 or higher\n"
7180                     if $drive->{iothread};
7181             }
7182
7183             qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid, $sparseinit, $jobs, $skipcomplete, $qga, $bwlimit);
7184         }
7185     }
7186
7187     my ($size) = PVE::Storage::volume_size_info($storecfg, $newvolid, 3);
7188
7189     my $disk = $drive;
7190     $disk->{format} = undef;
7191     $disk->{file} = $newvolid;
7192     $disk->{size} = $size;
7193
7194     return $disk;
7195 }
7196
7197 # this only works if VM is running
7198 sub get_current_qemu_machine {
7199     my ($vmid) = @_;
7200
7201     my $cmd = { execute => 'query-machines', arguments => {} };
7202     my $res = vm_qmp_command($vmid, $cmd);
7203
7204     my ($current, $default);
7205     foreach my $e (@$res) {
7206         $default = $e->{name} if $e->{'is-default'};
7207         $current = $e->{name} if $e->{'is-current'};
7208     }
7209
7210     # fallback to the default machine if current is not supported by qemu
7211     return $current || $default || 'pc';
7212 }
7213
7214 sub get_running_qemu_version {
7215     my ($vmid) = @_;
7216     my $cmd = { execute => 'query-version', arguments => {} };
7217     my $res = vm_qmp_command($vmid, $cmd);
7218     return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
7219 }
7220
7221 sub qemu_machine_feature_enabled {
7222     my ($machine, $kvmver, $version_major, $version_minor) = @_;
7223
7224     my $current_major;
7225     my $current_minor;
7226
7227     if ($machine && $machine =~ m/^((?:pc(-i440fx|-q35)?|virt)-(\d+)\.(\d+))/) {
7228
7229         $current_major = $3;
7230         $current_minor = $4;
7231
7232     } elsif ($kvmver =~ m/^(\d+)\.(\d+)/) {
7233
7234         $current_major = $1;
7235         $current_minor = $2;
7236     }
7237
7238     return 1 if $current_major > $version_major ||
7239                 ($current_major == $version_major &&
7240                  $current_minor >= $version_minor);
7241 }
7242
7243 sub qemu_machine_pxe {
7244     my ($vmid, $conf) = @_;
7245
7246     my $machine =  PVE::QemuServer::get_current_qemu_machine($vmid);
7247
7248     if ($conf->{machine} && $conf->{machine} =~ m/\.pxe$/) {
7249         $machine .= '.pxe';
7250     }
7251
7252     return $machine;
7253 }
7254
7255 sub qemu_use_old_bios_files {
7256     my ($machine_type) = @_;
7257
7258     return if !$machine_type;
7259
7260     my $use_old_bios_files = undef;
7261
7262     if ($machine_type =~ m/^(\S+)\.pxe$/) {
7263         $machine_type = $1;
7264         $use_old_bios_files = 1;
7265     } else {
7266         my $kvmver = kvm_user_version();
7267         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
7268         # load new efi bios files on migration. So this hack is required to allow
7269         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
7270         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
7271         $use_old_bios_files = !qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 4);
7272     }
7273
7274     return ($use_old_bios_files, $machine_type);
7275 }
7276
7277 sub create_efidisk($$$$$) {
7278     my ($storecfg, $storeid, $vmid, $fmt, $arch) = @_;
7279
7280     my (undef, $ovmf_vars) = get_ovmf_files($arch);
7281     die "EFI vars default image not found\n" if ! -f $ovmf_vars;
7282
7283     my $vars_size_b = -s $ovmf_vars;
7284     my $vars_size = PVE::Tools::convert_size($vars_size_b, 'b' => 'kb');
7285     my $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $fmt, undef, $vars_size);
7286     PVE::Storage::activate_volumes($storecfg, [$volid]);
7287
7288     qemu_img_convert($ovmf_vars, $volid, $vars_size_b, undef, 0);
7289
7290     return ($volid, $vars_size);
7291 }
7292
7293 sub vm_iothreads_list {
7294     my ($vmid) = @_;
7295
7296     my $res = vm_mon_cmd($vmid, 'query-iothreads');
7297
7298     my $iothreads = {};
7299     foreach my $iothread (@$res) {
7300         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
7301     }
7302
7303     return $iothreads;
7304 }
7305
7306 sub scsihw_infos {
7307     my ($conf, $drive) = @_;
7308
7309     my $maxdev = 0;
7310
7311     if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)) {
7312         $maxdev = 7;
7313     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
7314         $maxdev = 1;
7315     } else {
7316         $maxdev = 256;
7317     }
7318
7319     my $controller = int($drive->{index} / $maxdev);
7320     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single') ? "virtioscsi" : "scsihw";
7321
7322     return ($maxdev, $controller, $controller_prefix);
7323 }
7324
7325 sub add_hyperv_enlightenments {
7326     my ($cpuFlags, $winversion, $machine_type, $kvmver, $bios, $gpu_passthrough, $hv_vendor_id) = @_;
7327
7328     return if $winversion < 6;
7329     return if $bios && $bios eq 'ovmf' && $winversion < 8;
7330
7331     if ($gpu_passthrough || defined($hv_vendor_id)) {
7332         $hv_vendor_id //= 'proxmox';
7333         push @$cpuFlags , "hv_vendor_id=$hv_vendor_id";
7334     }
7335
7336     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
7337         push @$cpuFlags , 'hv_spinlocks=0x1fff';
7338         push @$cpuFlags , 'hv_vapic';
7339         push @$cpuFlags , 'hv_time';
7340     } else {
7341         push @$cpuFlags , 'hv_spinlocks=0xffff';
7342     }
7343
7344     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 6)) {
7345         push @$cpuFlags , 'hv_reset';
7346         push @$cpuFlags , 'hv_vpindex';
7347         push @$cpuFlags , 'hv_runtime';
7348     }
7349
7350     if ($winversion >= 7) {
7351         push @$cpuFlags , 'hv_relaxed';
7352
7353         if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 12)) {
7354             push @$cpuFlags , 'hv_synic';
7355             push @$cpuFlags , 'hv_stimer';
7356         }
7357
7358         if (qemu_machine_feature_enabled ($machine_type, $kvmver, 3, 1)) {
7359             push @$cpuFlags , 'hv_ipi';
7360         }
7361     }
7362 }
7363
7364 sub windows_version {
7365     my ($ostype) = @_;
7366
7367     return 0 if !$ostype;
7368
7369     my $winversion = 0;
7370
7371     if($ostype eq 'wxp' || $ostype eq 'w2k3' || $ostype eq 'w2k') {
7372         $winversion = 5;
7373     } elsif($ostype eq 'w2k8' || $ostype eq 'wvista') {
7374         $winversion = 6;
7375     } elsif ($ostype =~ m/^win(\d+)$/) {
7376         $winversion = $1;
7377     }
7378
7379     return $winversion;
7380 }
7381
7382 sub resolve_dst_disk_format {
7383         my ($storecfg, $storeid, $src_volname, $format) = @_;
7384         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
7385
7386         if (!$format) {
7387             # if no target format is specified, use the source disk format as hint
7388             if ($src_volname) {
7389                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
7390                 $format = qemu_img_format($scfg, $src_volname);
7391             } else {
7392                 return $defFormat;
7393             }
7394         }
7395
7396         # test if requested format is supported - else use default
7397         my $supported = grep { $_ eq $format } @$validFormats;
7398         $format = $defFormat if !$supported;
7399         return $format;
7400 }
7401
7402 sub resolve_first_disk {
7403     my $conf = shift;
7404     my @disks = PVE::QemuServer::valid_drive_names();
7405     my $firstdisk;
7406     foreach my $ds (reverse @disks) {
7407         next if !$conf->{$ds};
7408         my $disk = PVE::QemuServer::parse_drive($ds, $conf->{$ds});
7409         next if PVE::QemuServer::drive_is_cdrom($disk);
7410         $firstdisk = $ds;
7411     }
7412     return $firstdisk;
7413 }
7414
7415 sub generate_uuid {
7416     my ($uuid, $uuid_str);
7417     UUID::generate($uuid);
7418     UUID::unparse($uuid, $uuid_str);
7419     return $uuid_str;
7420 }
7421
7422 sub generate_smbios1_uuid {
7423     return "uuid=".generate_uuid();
7424 }
7425
7426 sub nbd_stop {
7427     my ($vmid) = @_;
7428
7429     vm_mon_cmd($vmid, 'nbd-server-stop');
7430 }
7431
7432 sub create_reboot_request {
7433     my ($vmid) = @_;
7434     open(my $fh, '>', "/run/qemu-server/$vmid.reboot")
7435         or die "failed to create reboot trigger file: $!\n";
7436     close($fh);
7437 }
7438
7439 sub clear_reboot_request {
7440     my ($vmid) = @_;
7441     my $path = "/run/qemu-server/$vmid.reboot";
7442     my $res = 0;
7443
7444     $res = unlink($path);
7445     die "could not remove reboot request for $vmid: $!"
7446         if !$res && $! != POSIX::ENOENT;
7447
7448     return $res;
7449 }
7450
7451 # bash completion helper
7452
7453 sub complete_backup_archives {
7454     my ($cmdname, $pname, $cvalue) = @_;
7455
7456     my $cfg = PVE::Storage::config();
7457
7458     my $storeid;
7459
7460     if ($cvalue =~ m/^([^:]+):/) {
7461         $storeid = $1;
7462     }
7463
7464     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
7465
7466     my $res = [];
7467     foreach my $id (keys %$data) {
7468         foreach my $item (@{$data->{$id}}) {
7469             next if $item->{format} !~ m/^vma\.(gz|lzo)$/;
7470             push @$res, $item->{volid} if defined($item->{volid});
7471         }
7472     }
7473
7474     return $res;
7475 }
7476
7477 my $complete_vmid_full = sub {
7478     my ($running) = @_;
7479
7480     my $idlist = vmstatus();
7481
7482     my $res = [];
7483
7484     foreach my $id (keys %$idlist) {
7485         my $d = $idlist->{$id};
7486         if (defined($running)) {
7487             next if $d->{template};
7488             next if $running && $d->{status} ne 'running';
7489             next if !$running && $d->{status} eq 'running';
7490         }
7491         push @$res, $id;
7492
7493     }
7494     return $res;
7495 };
7496
7497 sub complete_vmid {
7498     return &$complete_vmid_full();
7499 }
7500
7501 sub complete_vmid_stopped {
7502     return &$complete_vmid_full(0);
7503 }
7504
7505 sub complete_vmid_running {
7506     return &$complete_vmid_full(1);
7507 }
7508
7509 sub complete_storage {
7510
7511     my $cfg = PVE::Storage::config();
7512     my $ids = $cfg->{ids};
7513
7514     my $res = [];
7515     foreach my $sid (keys %$ids) {
7516         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
7517         next if !$ids->{$sid}->{content}->{images};
7518         push @$res, $sid;
7519     }
7520
7521     return $res;
7522 }
7523
7524 1;