PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5 use POSIX;
   6 use IO::Handle;
   7 use IO::Select;
   8 use IO::File;
   9 use IO::Dir;
  10 use IO::Socket::UNIX;
  11 use File::Basename;
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use Digest::SHA;
  16 use Fcntl ':flock';
  17 use Cwd 'abs_path';
  18 use IPC::Open3;
  19 use JSON;
  20 use Fcntl;
  21 use PVE::SafeSyslog;
  22 use Storable qw(dclone);
  23 use PVE::Exception qw(raise raise_param_exc);
  24 use PVE::Storage;
  25 use PVE::Tools qw(run_command lock_file lock_file_full file_read_firstline dir_glob_foreach $IPV6RE);
  26 use PVE::JSONSchema qw(get_standard_option);
  27 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file cfs_lock_file);
  28 use PVE::INotify;
  29 use PVE::ProcFSTools;
  30 use PVE::QemuConfig;
  31 use PVE::QMPClient;
  32 use PVE::RPCEnvironment;
  33 use PVE::GuestHelpers;
  34 use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr);
  35 use PVE::QemuServer::Memory;
  36 use PVE::QemuServer::USB qw(parse_usb_device);
  37 use PVE::QemuServer::Cloudinit;
  38 use PVE::SysFSTools;
  39 use PVE::Systemd;
  40 use Time::HiRes qw(gettimeofday);
  41 use File::Copy qw(copy);
  42 use URI::Escape;
  43
  44 my $EDK2_FW_BASE = '/usr/share/pve-edk2-firmware/';
  45 my $OVMF = {
  46     x86_64 => [
  47         "$EDK2_FW_BASE/OVMF_CODE.fd",
  48         "$EDK2_FW_BASE/OVMF_VARS.fd"
  49     ],
  50     aarch64 => [
  51         "$EDK2_FW_BASE/AAVMF_CODE.fd",
  52         "$EDK2_FW_BASE/AAVMF_VARS.fd"
  53     ],
  54 };
  55
  56 my $qemu_snap_storage = {rbd => 1, sheepdog => 1};
  57
  58 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  59
  60 my $QEMU_FORMAT_RE = qr/raw|cow|qcow|qcow2|qed|vmdk|cloop/;
  61
  62 # Note about locking: we use flock on the config file protect
  63 # against concurent actions.
  64 # Aditionaly, we have a 'lock' setting in the config file. This
  65 # can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
  66 # allowed when such lock is set. But you can ignore this kind of
  67 # lock with the --skiplock flag.
  68
  69 cfs_register_file('/qemu-server/',
  70                   \&parse_vm_config,
  71                   \&write_vm_config);
  72
  73 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
  74     description => "Some command save/restore state from this location.",
  75     type => 'string',
  76     maxLength => 128,
  77     optional => 1,
  78 });
  79
  80 PVE::JSONSchema::register_standard_option('pve-snapshot-name', {
  81     description => "The name of the snapshot.",
  82     type => 'string', format => 'pve-configid',
  83     maxLength => 40,
  84 });
  85
  86 PVE::JSONSchema::register_standard_option('pve-qm-image-format', {
  87     type => 'string',
  88     enum => [qw(raw cow qcow qed qcow2 vmdk cloop)],
  89     description => "The drive's backing file's data format.",
  90     optional => 1,
  91 });
  92
  93 PVE::JSONSchema::register_standard_option('pve-qemu-machine', {
  94         description => "Specifies the Qemu machine type.",
  95         type => 'string',
  96         pattern => '(pc|pc(-i440fx)?-\d+\.\d+(\.pxe)?|q35|pc-q35-\d+\.\d+(\.pxe)?|virt(?:-\d+\.\d+)?)',
  97         maxLength => 40,
  98         optional => 1,
  99 });
 100
 101 #no warnings 'redefine';
 102
 103 sub cgroups_write {
 104    my ($controller, $vmid, $option, $value) = @_;
 105
 106    my $path = "/sys/fs/cgroup/$controller/qemu.slice/$vmid.scope/$option";
 107    PVE::ProcFSTools::write_proc_entry($path, $value);
 108
 109 }
 110
 111 my $nodename = PVE::INotify::nodename();
 112
 113 mkdir "/etc/pve/nodes/$nodename";
 114 my $confdir = "/etc/pve/nodes/$nodename/qemu-server";
 115 mkdir $confdir;
 116
 117 my $var_run_tmpdir = "/var/run/qemu-server";
 118 mkdir $var_run_tmpdir;
 119
 120 my $lock_dir = "/var/lock/qemu-server";
 121 mkdir $lock_dir;
 122
 123 my $cpu_vendor_list = {
 124     # Intel CPUs
 125     486 => 'GenuineIntel',
 126     pentium => 'GenuineIntel',
 127     pentium2  => 'GenuineIntel',
 128     pentium3  => 'GenuineIntel',
 129     coreduo => 'GenuineIntel',
 130     core2duo => 'GenuineIntel',
 131     Conroe  => 'GenuineIntel',
 132     Penryn  => 'GenuineIntel',
 133     Nehalem  => 'GenuineIntel',
 134     'Nehalem-IBRS'  => 'GenuineIntel',
 135     Westmere => 'GenuineIntel',
 136     'Westmere-IBRS' => 'GenuineIntel',
 137     SandyBridge => 'GenuineIntel',
 138     'SandyBridge-IBRS' => 'GenuineIntel',
 139     IvyBridge => 'GenuineIntel',
 140     'IvyBridge-IBRS' => 'GenuineIntel',
 141     Haswell => 'GenuineIntel',
 142     'Haswell-IBRS' => 'GenuineIntel',
 143     'Haswell-noTSX' => 'GenuineIntel',
 144     'Haswell-noTSX-IBRS' => 'GenuineIntel',
 145     Broadwell => 'GenuineIntel',
 146     'Broadwell-IBRS' => 'GenuineIntel',
 147     'Broadwell-noTSX' => 'GenuineIntel',
 148     'Broadwell-noTSX-IBRS' => 'GenuineIntel',
 149     'Skylake-Client' => 'GenuineIntel',
 150     'Skylake-Client-IBRS' => 'GenuineIntel',
 151     'Skylake-Server' => 'GenuineIntel',
 152     'Skylake-Server-IBRS' => 'GenuineIntel',
 153
 154     # AMD CPUs
 155     athlon => 'AuthenticAMD',
 156     phenom  => 'AuthenticAMD',
 157     Opteron_G1  => 'AuthenticAMD',
 158     Opteron_G2  => 'AuthenticAMD',
 159     Opteron_G3  => 'AuthenticAMD',
 160     Opteron_G4  => 'AuthenticAMD',
 161     Opteron_G5  => 'AuthenticAMD',
 162     EPYC => 'AuthenticAMD',
 163     'EPYC-IBPB' => 'AuthenticAMD',
 164
 165     # generic types, use vendor from host node
 166     host => 'default',
 167     kvm32 => 'default',
 168     kvm64 => 'default',
 169     qemu32 => 'default',
 170     qemu64 => 'default',
 171     max => 'default',
 172 };
 173
 174 my $cpu_flag = qr/[+-](pcid|spec-ctrl|ibpb|ssbd|virt-ssbd|amd-ssbd|amd-no-ssb|pdpe1gb)/;
 175
 176 my $cpu_fmt = {
 177     cputype => {
 178         description => "Emulated CPU type.",
 179         type => 'string',
 180         enum => [ sort { "\L$a" cmp "\L$b" } keys %$cpu_vendor_list ],
 181         default => 'kvm64',
 182         default_key => 1,
 183     },
 184     hidden => {
 185         description => "Do not identify as a KVM virtual machine.",
 186         type => 'boolean',
 187         optional => 1,
 188         default => 0
 189     },
 190     'hv-vendor-id' => {
 191         type => 'string',
 192         pattern => qr/[a-zA-Z0-9]{1,12}/,
 193         format_description => 'vendor-id',
 194         description => 'The Hyper-V vendor ID. Some drivers or programs inside Windows guests need a specific ID.',
 195         optional => 1,
 196     },
 197     flags => {
 198         description => "List of additional CPU flags separated by ';'."
 199                      . " Use '+FLAG' to enable, '-FLAG' to disable a flag."
 200                      . " Currently supported flags: 'pcid', 'spec-ctrl', 'ibpb', 'ssbd', 'virt-ssbd', 'amd-ssbd', 'amd-no-ssb', 'pdpe1gb'.",
 201         format_description => '+FLAG[;-FLAG...]',
 202         type => 'string',
 203         pattern => qr/$cpu_flag(;$cpu_flag)*/,
 204         optional => 1,
 205     },
 206 };
 207
 208 my $watchdog_fmt = {
 209     model => {
 210         default_key => 1,
 211         type => 'string',
 212         enum => [qw(i6300esb ib700)],
 213         description => "Watchdog type to emulate.",
 214         default => 'i6300esb',
 215         optional => 1,
 216     },
 217     action => {
 218         type => 'string',
 219         enum => [qw(reset shutdown poweroff pause debug none)],
 220         description => "The action to perform if after activation the guest fails to poll the watchdog in time.",
 221         optional => 1,
 222     },
 223 };
 224 PVE::JSONSchema::register_format('pve-qm-watchdog', $watchdog_fmt);
 225
 226 my $agent_fmt = {
 227     enabled => {
 228         description => "Enable/disable Qemu GuestAgent.",
 229         type => 'boolean',
 230         default => 0,
 231         default_key => 1,
 232     },
 233     fstrim_cloned_disks => {
 234         description => "Run fstrim after cloning/moving a disk.",
 235         type => 'boolean',
 236         optional => 1,
 237         default => 0
 238     },
 239 };
 240
 241 my $vga_fmt = {
 242     type => {
 243         description => "Select the VGA type.",
 244         type => 'string',
 245         default => 'std',
 246         optional => 1,
 247         default_key => 1,
 248         enum => [qw(cirrus qxl qxl2 qxl3 qxl4 none serial0 serial1 serial2 serial3 std virtio vmware)],
 249     },
 250     memory => {
 251         description => "Sets the VGA memory (in MiB). Has no effect with serial display.",
 252         type => 'integer',
 253         optional => 1,
 254         minimum => 4,
 255         maximum => 512,
 256     },
 257 };
 258
 259 my $ivshmem_fmt = {
 260     size => {
 261         type => 'integer',
 262         minimum => 1,
 263         description => "The size of the file in MB.",
 264     },
 265     name => {
 266         type => 'string',
 267         pattern => '[a-zA-Z0-9\-]+',
 268         optional => 1,
 269         format_description => 'string',
 270         description => "The name of the file. Will be prefixed with 'pve-shm-'. Default is the VMID. Will be deleted when the VM is stopped.",
 271     },
 272 };
 273
 274 my $confdesc = {
 275     onboot => {
 276         optional => 1,
 277         type => 'boolean',
 278         description => "Specifies whether a VM will be started during system bootup.",
 279         default => 0,
 280     },
 281     autostart => {
 282         optional => 1,
 283         type => 'boolean',
 284         description => "Automatic restart after crash (currently ignored).",
 285         default => 0,
 286     },
 287     hotplug => {
 288         optional => 1,
 289         type => 'string', format => 'pve-hotplug-features',
 290         description => "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
 291         default => 'network,disk,usb',
 292     },
 293     reboot => {
 294         optional => 1,
 295         type => 'boolean',
 296         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 297         default => 1,
 298     },
 299     lock => {
 300         optional => 1,
 301         type => 'string',
 302         description => "Lock/unlock the VM.",
 303         enum => [qw(backup clone create migrate rollback snapshot snapshot-delete)],
 304     },
 305     cpulimit => {
 306         optional => 1,
 307         type => 'number',
 308         description => "Limit of CPU usage.",
 309         verbose_description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
 310         minimum => 0,
 311         maximum => 128,
 312         default => 0,
 313     },
 314     cpuunits => {
 315         optional => 1,
 316         type => 'integer',
 317         description => "CPU weight for a VM.",
 318         verbose_description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.",
 319         minimum => 2,
 320         maximum => 262144,
 321         default => 1024,
 322     },
 323     memory => {
 324         optional => 1,
 325         type => 'integer',
 326         description => "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
 327         minimum => 16,
 328         default => 512,
 329     },
 330     balloon => {
 331         optional => 1,
 332         type => 'integer',
 333         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 334         minimum => 0,
 335     },
 336     shares => {
 337         optional => 1,
 338         type => 'integer',
 339         description => "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning. Auto-ballooning is done by pvestatd.",
 340         minimum => 0,
 341         maximum => 50000,
 342         default => 1000,
 343     },
 344     keyboard => {
 345         optional => 1,
 346         type => 'string',
 347         description => "Keybord layout for vnc server. Default is read from the '/etc/pve/datacenter.cfg' configuration file.".
 348                        "It should not be necessary to set it.",
 349         enum => PVE::Tools::kvmkeymaplist(),
 350         default => undef,
 351     },
 352     name => {
 353         optional => 1,
 354         type => 'string', format => 'dns-name',
 355         description => "Set a name for the VM. Only used on the configuration web interface.",
 356     },
 357     scsihw => {
 358         optional => 1,
 359         type => 'string',
 360         description => "SCSI controller model",
 361         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 362         default => 'lsi',
 363     },
 364     description => {
 365         optional => 1,
 366         type => 'string',
 367         description => "Description for the VM. Only used on the configuration web interface. This is saved as comment inside the configuration file.",
 368     },
 369     ostype => {
 370         optional => 1,
 371         type => 'string',
 372         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 l24 l26 solaris)],
 373         description => "Specify guest operating system.",
 374         verbose_description => <<EODESC,
 375 Specify guest operating system. This is used to enable special
 376 optimization/features for specific operating systems:
 377
 378 [horizontal]
 379 other;; unspecified OS
 380 wxp;; Microsoft Windows XP
 381 w2k;; Microsoft Windows 2000
 382 w2k3;; Microsoft Windows 2003
 383 w2k8;; Microsoft Windows 2008
 384 wvista;; Microsoft Windows Vista
 385 win7;; Microsoft Windows 7
 386 win8;; Microsoft Windows 8/2012/2012r2
 387 win10;; Microsoft Windows 10/2016
 388 l24;; Linux 2.4 Kernel
 389 l26;; Linux 2.6/3.X Kernel
 390 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
 391 EODESC
 392     },
 393     boot => {
 394         optional => 1,
 395         type => 'string',
 396         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n).",
 397         pattern => '[acdn]{1,4}',
 398         default => 'cdn',
 399     },
 400     bootdisk => {
 401         optional => 1,
 402         type => 'string', format => 'pve-qm-bootdisk',
 403         description => "Enable booting from specified disk.",
 404         pattern => '(ide|sata|scsi|virtio)\d+',
 405     },
 406     smp => {
 407         optional => 1,
 408         type => 'integer',
 409         description => "The number of CPUs. Please use option -sockets instead.",
 410         minimum => 1,
 411         default => 1,
 412     },
 413     sockets => {
 414         optional => 1,
 415         type => 'integer',
 416         description => "The number of CPU sockets.",
 417         minimum => 1,
 418         default => 1,
 419     },
 420     cores => {
 421         optional => 1,
 422         type => 'integer',
 423         description => "The number of cores per socket.",
 424         minimum => 1,
 425         default => 1,
 426     },
 427     numa => {
 428         optional => 1,
 429         type => 'boolean',
 430         description => "Enable/disable NUMA.",
 431         default => 0,
 432     },
 433     hugepages => {
 434         optional => 1,
 435         type => 'string',
 436         description => "Enable/disable hugepages memory.",
 437         enum => [qw(any 2 1024)],
 438     },
 439     vcpus => {
 440         optional => 1,
 441         type => 'integer',
 442         description => "Number of hotplugged vcpus.",
 443         minimum => 1,
 444         default => 0,
 445     },
 446     acpi => {
 447         optional => 1,
 448         type => 'boolean',
 449         description => "Enable/disable ACPI.",
 450         default => 1,
 451     },
 452     agent => {
 453         optional => 1,
 454         description => "Enable/disable Qemu GuestAgent and its properties.",
 455         type => 'string',
 456         format => $agent_fmt,
 457     },
 458     kvm => {
 459         optional => 1,
 460         type => 'boolean',
 461         description => "Enable/disable KVM hardware virtualization.",
 462         default => 1,
 463     },
 464     tdf => {
 465         optional => 1,
 466         type => 'boolean',
 467         description => "Enable/disable time drift fix.",
 468         default => 0,
 469     },
 470     localtime => {
 471         optional => 1,
 472         type => 'boolean',
 473         description => "Set the real time clock to local time. This is enabled by default if ostype indicates a Microsoft OS.",
 474     },
 475     freeze => {
 476         optional => 1,
 477         type => 'boolean',
 478         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 479     },
 480     vga => {
 481         optional => 1,
 482         type => 'string', format => $vga_fmt,
 483         description => "Configure the VGA hardware.",
 484         verbose_description => "Configure the VGA Hardware. If you want to use ".
 485             "high resolution modes (>= 1280x1024x16) you may need to increase " .
 486             "the vga memory option. Since QEMU 2.9 the default VGA display type " .
 487             "is 'std' for all OS types besides some Windows versions (XP and " .
 488             "older) which use 'cirrus'. The 'qxl' option enables the SPICE " .
 489             "display server. For win* OS you can select how many independent " .
 490             "displays you want, Linux guests can add displays them self.\n".
 491             "You can also run without any graphic card, using a serial device as terminal.",
 492     },
 493     watchdog => {
 494         optional => 1,
 495         type => 'string', format => 'pve-qm-watchdog',
 496         description => "Create a virtual hardware watchdog device.",
 497         verbose_description => "Create a virtual hardware watchdog device. Once enabled" .
 498             " (by a guest action), the watchdog must be periodically polled " .
 499             "by an agent inside the guest or else the watchdog will reset " .
 500             "the guest (or execute the respective action specified)",
 501     },
 502     startdate => {
 503         optional => 1,
 504         type => 'string',
 505         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 506         description => "Set the initial date of the real time clock. Valid format for date are: 'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 507         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 508         default => 'now',
 509     },
 510     startup =>  get_standard_option('pve-startup-order'),
 511     template => {
 512         optional => 1,
 513         type => 'boolean',
 514         description => "Enable/disable Template.",
 515         default => 0,
 516     },
 517     args => {
 518         optional => 1,
 519         type => 'string',
 520         description => "Arbitrary arguments passed to kvm.",
 521         verbose_description => <<EODESCR,
 522 Arbitrary arguments passed to kvm, for example:
 523
 524 args: -no-reboot -no-hpet
 525
 526 NOTE: this option is for experts only.
 527 EODESCR
 528     },
 529     tablet => {
 530         optional => 1,
 531         type => 'boolean',
 532         default => 1,
 533         description => "Enable/disable the USB tablet device.",
 534         verbose_description => "Enable/disable the USB tablet device. This device is " .
 535             "usually needed to allow absolute mouse positioning with VNC. " .
 536             "Else the mouse runs out of sync with normal VNC clients. " .
 537             "If you're running lots of console-only guests on one host, " .
 538             "you may consider disabling this to save some context switches. " .
 539             "This is turned off by default if you use spice (-vga=qxl).",
 540     },
 541     migrate_speed => {
 542         optional => 1,
 543         type => 'integer',
 544         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 545         minimum => 0,
 546         default => 0,
 547     },
 548     migrate_downtime => {
 549         optional => 1,
 550         type => 'number',
 551         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 552         minimum => 0,
 553         default => 0.1,
 554     },
 555     cdrom => {
 556         optional => 1,
 557         type => 'string', format => 'pve-qm-ide',
 558         typetext => '<volume>',
 559         description => "This is an alias for option -ide2",
 560     },
 561     cpu => {
 562         optional => 1,
 563         description => "Emulated CPU type.",
 564         type => 'string',
 565         format => $cpu_fmt,
 566     },
 567     parent => get_standard_option('pve-snapshot-name', {
 568         optional => 1,
 569         description => "Parent snapshot name. This is used internally, and should not be modified.",
 570     }),
 571     snaptime => {
 572         optional => 1,
 573         description => "Timestamp for snapshots.",
 574         type => 'integer',
 575         minimum => 0,
 576     },
 577     vmstate => {
 578         optional => 1,
 579         type => 'string', format => 'pve-volume-id',
 580         description => "Reference to a volume which stores the VM state. This is used internally for snapshots.",
 581     },
 582     vmstatestorage => get_standard_option('pve-storage-id', {
 583         description => "Default storage for VM state volumes/files.",
 584         optional => 1,
 585     }),
 586     runningmachine => get_standard_option('pve-qemu-machine', {
 587         description => "Specifies the Qemu machine type of the running vm. This is used internally for snapshots.",
 588     }),
 589     machine => get_standard_option('pve-qemu-machine'),
 590     arch => {
 591         description => "Virtual processor architecture. Defaults to the host.",
 592         optional => 1,
 593         type => 'string',
 594         enum => [qw(x86_64 aarch64)],
 595     },
 596     smbios1 => {
 597         description => "Specify SMBIOS type 1 fields.",
 598         type => 'string', format => 'pve-qm-smbios1',
 599         maxLength => 256,
 600         optional => 1,
 601     },
 602     protection => {
 603         optional => 1,
 604         type => 'boolean',
 605         description => "Sets the protection flag of the VM. This will disable the remove VM and remove disk operations.",
 606         default => 0,
 607     },
 608     bios => {
 609         optional => 1,
 610         type => 'string',
 611         enum => [ qw(seabios ovmf) ],
 612         description => "Select BIOS implementation.",
 613         default => 'seabios',
 614     },
 615     vmgenid => {
 616         type => 'string',
 617         pattern => '(?:[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}|[01])',
 618         format_description => 'UUID',
 619         description => "Set VM Generation ID. Use '1' to autogenerate on create or update, pass '0' to disable explicitly.",
 620         verbose_description => "The VM generation ID (vmgenid) device exposes a".
 621             " 128-bit integer value identifier to the guest OS. This allows to".
 622             " notify the guest operating system when the virtual machine is".
 623             " executed with a different configuration (e.g. snapshot execution".
 624             " or creation from a template). The guest operating system notices".
 625             " the change, and is then able to react as appropriate by marking".
 626             " its copies of distributed databases as dirty, re-initializing its".
 627             " random number generator, etc.\n".
 628             "Note that auto-creation only works when done throug API/CLI create".
 629             " or update methods, but not when manually editing the config file.",
 630         default => "1 (autogenerated)",
 631         optional => 1,
 632     },
 633     hookscript => {
 634         type => 'string',
 635         format => 'pve-volume-id',
 636         optional => 1,
 637         description => "Script that will be executed during various steps in the vms lifetime.",
 638     },
 639     ivshmem => {
 640         type => 'string',
 641         format => $ivshmem_fmt,
 642         description => "Inter-VM shared memory. Useful for direct communication between VMs, or to the host.",
 643         optional => 1,
 644     }
 645 };
 646
 647 my $confdesc_cloudinit = {
 648     citype => {
 649         optional => 1,
 650         type => 'string',
 651         description => 'Specifies the cloud-init configuration format. The default depends on the configured operating system type (`ostype`. We use the `nocloud` format for Linux, and `configdrive2` for windows.',
 652         enum => ['configdrive2', 'nocloud'],
 653     },
 654     ciuser => {
 655         optional => 1,
 656         type => 'string',
 657         description => "cloud-init: User name to change ssh keys and password for instead of the image's configured default user.",
 658     },
 659     cipassword => {
 660         optional => 1,
 661         type => 'string',
 662         description => 'cloud-init: Password to assign the user. Using this is generally not recommended. Use ssh keys instead. Also note that older cloud-init versions do not support hashed passwords.',
 663     },
 664     searchdomain => {
 665         optional => 1,
 666         type => 'string',
 667         description => "cloud-init: Sets DNS search domains for a container. Create will automatically use the setting from the host if neither searchdomain nor nameserver are set.",
 668     },
 669     nameserver => {
 670         optional => 1,
 671         type => 'string', format => 'address-list',
 672         description => "cloud-init: Sets DNS server IP address for a container. Create will automatically use the setting from the host if neither searchdomain nor nameserver are set.",
 673     },
 674     sshkeys => {
 675         optional => 1,
 676         type => 'string',
 677         format => 'urlencoded',
 678         description => "cloud-init: Setup public SSH keys (one key per line, OpenSSH format).",
 679     },
 680 };
 681
 682 # what about other qemu settings ?
 683 #cpu => 'string',
 684 #machine => 'string',
 685 #fda => 'file',
 686 #fdb => 'file',
 687 #mtdblock => 'file',
 688 #sd => 'file',
 689 #pflash => 'file',
 690 #snapshot => 'bool',
 691 #bootp => 'file',
 692 ##tftp => 'dir',
 693 ##smb => 'dir',
 694 #kernel => 'file',
 695 #append => 'string',
 696 #initrd => 'file',
 697 ##soundhw => 'string',
 698
 699 while (my ($k, $v) = each %$confdesc) {
 700     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 701 }
 702
 703 my $MAX_IDE_DISKS = 4;
 704 my $MAX_SCSI_DISKS = 14;
 705 my $MAX_VIRTIO_DISKS = 16;
 706 my $MAX_SATA_DISKS = 6;
 707 my $MAX_USB_DEVICES = 5;
 708 my $MAX_NETS = 32;
 709 my $MAX_UNUSED_DISKS = 256;
 710 my $MAX_HOSTPCI_DEVICES = 4;
 711 my $MAX_SERIAL_PORTS = 4;
 712 my $MAX_PARALLEL_PORTS = 3;
 713 my $MAX_NUMA = 8;
 714
 715 my $numa_fmt = {
 716     cpus => {
 717         type => "string",
 718         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 719         description => "CPUs accessing this NUMA node.",
 720         format_description => "id[-id];...",
 721     },
 722     memory => {
 723         type => "number",
 724         description => "Amount of memory this NUMA node provides.",
 725         optional => 1,
 726     },
 727     hostnodes => {
 728         type => "string",
 729         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 730         description => "Host NUMA nodes to use.",
 731         format_description => "id[-id];...",
 732         optional => 1,
 733     },
 734     policy => {
 735         type => 'string',
 736         enum => [qw(preferred bind interleave)],
 737         description => "NUMA allocation policy.",
 738         optional => 1,
 739     },
 740 };
 741 PVE::JSONSchema::register_format('pve-qm-numanode', $numa_fmt);
 742 my $numadesc = {
 743     optional => 1,
 744     type => 'string', format => $numa_fmt,
 745     description => "NUMA topology.",
 746 };
 747 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 748
 749 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 750     $confdesc->{"numa$i"} = $numadesc;
 751 }
 752
 753 my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000',  'pcnet',  'virtio',
 754                       'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3',
 755                       'e1000-82540em', 'e1000-82544gc', 'e1000-82545em'];
 756 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 757
 758 my $net_fmt_bridge_descr = <<__EOD__;
 759 Bridge to attach the network device to. The Proxmox VE standard bridge
 760 is called 'vmbr0'.
 761
 762 If you do not specify a bridge, we create a kvm user (NATed) network
 763 device, which provides DHCP and DNS services. The following addresses
 764 are used:
 765
 766  10.0.2.2   Gateway
 767  10.0.2.3   DNS Server
 768  10.0.2.4   SMB Server
 769
 770 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 771 __EOD__
 772
 773 my $net_fmt = {
 774     macaddr => {
 775         type => 'string',
 776         pattern => qr/[0-9a-f]{2}(?::[0-9a-f]{2}){5}/i,
 777         description => "MAC address. That address must be unique withing your network. This is automatically generated if not specified.",
 778         format_description => "XX:XX:XX:XX:XX:XX",
 779         optional => 1,
 780     },
 781     model => {
 782         type => 'string',
 783         description => "Network Card Model. The 'virtio' model provides the best performance with very low CPU overhead. If your guest does not support this driver, it is usually best to use 'e1000'.",
 784         enum => $nic_model_list,
 785         default_key => 1,
 786     },
 787     (map { $_ => { keyAlias => 'model', alias => 'macaddr' }} @$nic_model_list),
 788     bridge => {
 789         type => 'string',
 790         description => $net_fmt_bridge_descr,
 791         format_description => 'bridge',
 792         optional => 1,
 793     },
 794     queues => {
 795         type => 'integer',
 796         minimum => 0, maximum => 16,
 797         description => 'Number of packet queues to be used on the device.',
 798         optional => 1,
 799     },
 800     rate => {
 801         type => 'number',
 802         minimum => 0,
 803         description => "Rate limit in mbps (megabytes per second) as floating point number.",
 804         optional => 1,
 805     },
 806     tag => {
 807         type => 'integer',
 808         minimum => 1, maximum => 4094,
 809         description => 'VLAN tag to apply to packets on this interface.',
 810         optional => 1,
 811     },
 812     trunks => {
 813         type => 'string',
 814         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 815         description => 'VLAN trunks to pass through this interface.',
 816         format_description => 'vlanid[;vlanid...]',
 817         optional => 1,
 818     },
 819     firewall => {
 820         type => 'boolean',
 821         description => 'Whether this interface should be protected by the firewall.',
 822         optional => 1,
 823     },
 824     link_down => {
 825         type => 'boolean',
 826         description => 'Whether this interface should be disconnected (like pulling the plug).',
 827         optional => 1,
 828     },
 829 };
 830
 831 my $netdesc = {
 832     optional => 1,
 833     type => 'string', format => $net_fmt,
 834     description => "Specify network devices.",
 835 };
 836
 837 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 838
 839 my $ipconfig_fmt = {
 840     ip => {
 841         type => 'string',
 842         format => 'pve-ipv4-config',
 843         format_description => 'IPv4Format/CIDR',
 844         description => 'IPv4 address in CIDR format.',
 845         optional => 1,
 846         default => 'dhcp',
 847     },
 848     gw => {
 849         type => 'string',
 850         format => 'ipv4',
 851         format_description => 'GatewayIPv4',
 852         description => 'Default gateway for IPv4 traffic.',
 853         optional => 1,
 854         requires => 'ip',
 855     },
 856     ip6 => {
 857         type => 'string',
 858         format => 'pve-ipv6-config',
 859         format_description => 'IPv6Format/CIDR',
 860         description => 'IPv6 address in CIDR format.',
 861         optional => 1,
 862         default => 'dhcp',
 863     },
 864     gw6 => {
 865         type => 'string',
 866         format => 'ipv6',
 867         format_description => 'GatewayIPv6',
 868         description => 'Default gateway for IPv6 traffic.',
 869         optional => 1,
 870         requires => 'ip6',
 871     },
 872 };
 873 PVE::JSONSchema::register_format('pve-qm-ipconfig', $ipconfig_fmt);
 874 my $ipconfigdesc = {
 875     optional => 1,
 876     type => 'string', format => 'pve-qm-ipconfig',
 877     description => <<'EODESCR',
 878 cloud-init: Specify IP addresses and gateways for the corresponding interface.
 879
 880 IP addresses use CIDR notation, gateways are optional but need an IP of the same type specified.
 881
 882 The special string 'dhcp' can be used for IP addresses to use DHCP, in which case no explicit gateway should be provided.
 883 For IPv6 the special string 'auto' can be used to use stateless autoconfiguration.
 884
 885 If cloud-init is enabled and neither an IPv4 nor an IPv6 address is specified, it defaults to using dhcp on IPv4.
 886 EODESCR
 887 };
 888 PVE::JSONSchema::register_standard_option("pve-qm-ipconfig", $netdesc);
 889
 890 for (my $i = 0; $i < $MAX_NETS; $i++)  {
 891     $confdesc->{"net$i"} = $netdesc;
 892     $confdesc_cloudinit->{"ipconfig$i"} = $ipconfigdesc;
 893 }
 894
 895 foreach my $key (keys %$confdesc_cloudinit) {
 896     $confdesc->{$key} = $confdesc_cloudinit->{$key};
 897 }
 898
 899 PVE::JSONSchema::register_format('pve-volume-id-or-qm-path', \&verify_volume_id_or_qm_path);
 900 sub verify_volume_id_or_qm_path {
 901     my ($volid, $noerr) = @_;
 902
 903     if ($volid eq 'none' || $volid eq 'cdrom' || $volid =~ m|^/|) {
 904         return $volid;
 905     }
 906
 907     # if its neither 'none' nor 'cdrom' nor a path, check if its a volume-id
 908     $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') };
 909     if ($@) {
 910         return undef if $noerr;
 911         die $@;
 912     }
 913     return $volid;
 914 }
 915
 916 my $drivename_hash;
 917
 918 my %drivedesc_base = (
 919     volume => { alias => 'file' },
 920     file => {
 921         type => 'string',
 922         format => 'pve-volume-id-or-qm-path',
 923         default_key => 1,
 924         format_description => 'volume',
 925         description => "The drive's backing volume.",
 926     },
 927     media => {
 928         type => 'string',
 929         enum => [qw(cdrom disk)],
 930         description => "The drive's media type.",
 931         default => 'disk',
 932         optional => 1
 933     },
 934     cyls => {
 935         type => 'integer',
 936         description => "Force the drive's physical geometry to have a specific cylinder count.",
 937         optional => 1
 938     },
 939     heads => {
 940         type => 'integer',
 941         description => "Force the drive's physical geometry to have a specific head count.",
 942         optional => 1
 943     },
 944     secs => {
 945         type => 'integer',
 946         description => "Force the drive's physical geometry to have a specific sector count.",
 947         optional => 1
 948     },
 949     trans => {
 950         type => 'string',
 951         enum => [qw(none lba auto)],
 952         description => "Force disk geometry bios translation mode.",
 953         optional => 1,
 954     },
 955     snapshot => {
 956         type => 'boolean',
 957         description => "Controls qemu's snapshot mode feature."
 958             . " If activated, changes made to the disk are temporary and will"
 959             . " be discarded when the VM is shutdown.",
 960         optional => 1,
 961     },
 962     cache => {
 963         type => 'string',
 964         enum => [qw(none writethrough writeback unsafe directsync)],
 965         description => "The drive's cache mode",
 966         optional => 1,
 967     },
 968     format => get_standard_option('pve-qm-image-format'),
 969     size => {
 970         type => 'string',
 971         format => 'disk-size',
 972         format_description => 'DiskSize',
 973         description => "Disk size. This is purely informational and has no effect.",
 974         optional => 1,
 975     },
 976     backup => {
 977         type => 'boolean',
 978         description => "Whether the drive should be included when making backups.",
 979         optional => 1,
 980     },
 981     replicate => {
 982         type => 'boolean',
 983         description => 'Whether the drive should considered for replication jobs.',
 984         optional => 1,
 985         default => 1,
 986     },
 987     rerror => {
 988         type => 'string',
 989         enum => [qw(ignore report stop)],
 990         description => 'Read error action.',
 991         optional => 1,
 992     },
 993     werror => {
 994         type => 'string',
 995         enum => [qw(enospc ignore report stop)],
 996         description => 'Write error action.',
 997         optional => 1,
 998     },
 999     aio => {
1000         type => 'string',
1001         enum => [qw(native threads)],
1002         description => 'AIO type to use.',
1003         optional => 1,
1004     },
1005     discard => {
1006         type => 'string',
1007         enum => [qw(ignore on)],
1008         description => 'Controls whether to pass discard/trim requests to the underlying storage.',
1009         optional => 1,
1010     },
1011     detect_zeroes => {
1012         type => 'boolean',
1013         description => 'Controls whether to detect and try to optimize writes of zeroes.',
1014         optional => 1,
1015     },
1016     serial => {
1017         type => 'string',
1018         format => 'urlencoded',
1019         format_description => 'serial',
1020         maxLength => 20*3, # *3 since it's %xx url enoded
1021         description => "The drive's reported serial number, url-encoded, up to 20 bytes long.",
1022         optional => 1,
1023     },
1024     shared => {
1025         type => 'boolean',
1026         description => 'Mark this locally-managed volume as available on all nodes',
1027         verbose_description => "Mark this locally-managed volume as available on all nodes.\n\nWARNING: This option does not share the volume automatically, it assumes it is shared already!",
1028         optional => 1,
1029         default => 0,
1030     }
1031 );
1032
1033 my %iothread_fmt = ( iothread => {
1034         type => 'boolean',
1035         description => "Whether to use iothreads for this drive",
1036         optional => 1,
1037 });
1038
1039 my %model_fmt = (
1040     model => {
1041         type => 'string',
1042         format => 'urlencoded',
1043         format_description => 'model',
1044         maxLength => 40*3, # *3 since it's %xx url enoded
1045         description => "The drive's reported model name, url-encoded, up to 40 bytes long.",
1046         optional => 1,
1047     },
1048 );
1049
1050 my %queues_fmt = (
1051     queues => {
1052         type => 'integer',
1053         description => "Number of queues.",
1054         minimum => 2,
1055         optional => 1
1056     }
1057 );
1058
1059 my %scsiblock_fmt = (
1060     scsiblock => {
1061         type => 'boolean',
1062         description => "whether to use scsi-block for full passthrough of host block device\n\nWARNING: can lead to I/O errors in combination with low memory or high memory fragmentation on host",
1063         optional => 1,
1064         default => 0,
1065     },
1066 );
1067
1068 my %ssd_fmt = (
1069     ssd => {
1070         type => 'boolean',
1071         description => "Whether to expose this drive as an SSD, rather than a rotational hard disk.",
1072         optional => 1,
1073     },
1074 );
1075
1076 my %wwn_fmt = (
1077     wwn => {
1078         type => 'string',
1079         pattern => qr/^(0x)[0-9a-fA-F]{16}/,
1080         format_description => 'wwn',
1081         description => "The drive's worldwide name, encoded as 16 bytes hex string, prefixed by '0x'.",
1082         optional => 1,
1083     },
1084 );
1085
1086 my $add_throttle_desc = sub {
1087     my ($key, $type, $what, $unit, $longunit, $minimum) = @_;
1088     my $d = {
1089         type => $type,
1090         format_description => $unit,
1091         description => "Maximum $what in $longunit.",
1092         optional => 1,
1093     };
1094     $d->{minimum} = $minimum if defined($minimum);
1095     $drivedesc_base{$key} = $d;
1096 };
1097 # throughput: (leaky bucket)
1098 $add_throttle_desc->('bps',     'integer', 'r/w speed',   'bps',  'bytes per second');
1099 $add_throttle_desc->('bps_rd',  'integer', 'read speed',  'bps',  'bytes per second');
1100 $add_throttle_desc->('bps_wr',  'integer', 'write speed', 'bps',  'bytes per second');
1101 $add_throttle_desc->('mbps',    'number',  'r/w speed',   'mbps', 'megabytes per second');
1102 $add_throttle_desc->('mbps_rd', 'number',  'read speed',  'mbps', 'megabytes per second');
1103 $add_throttle_desc->('mbps_wr', 'number',  'write speed', 'mbps', 'megabytes per second');
1104 $add_throttle_desc->('iops',    'integer', 'r/w I/O',     'iops', 'operations per second');
1105 $add_throttle_desc->('iops_rd', 'integer', 'read I/O',    'iops', 'operations per second');
1106 $add_throttle_desc->('iops_wr', 'integer', 'write I/O',   'iops', 'operations per second');
1107
1108 # pools: (pool of IO before throttling starts taking effect)
1109 $add_throttle_desc->('mbps_max',    'number',  'unthrottled r/w pool',       'mbps', 'megabytes per second');
1110 $add_throttle_desc->('mbps_rd_max', 'number',  'unthrottled read pool',      'mbps', 'megabytes per second');
1111 $add_throttle_desc->('mbps_wr_max', 'number',  'unthrottled write pool',     'mbps', 'megabytes per second');
1112 $add_throttle_desc->('iops_max',    'integer', 'unthrottled r/w I/O pool',   'iops', 'operations per second');
1113 $add_throttle_desc->('iops_rd_max', 'integer', 'unthrottled read I/O pool',  'iops', 'operations per second');
1114 $add_throttle_desc->('iops_wr_max', 'integer', 'unthrottled write I/O pool', 'iops', 'operations per second');
1115
1116 # burst lengths
1117 $add_throttle_desc->('bps_max_length',     'integer', 'length of I/O bursts',       'seconds', 'seconds', 1);
1118 $add_throttle_desc->('bps_rd_max_length',  'integer', 'length of read I/O bursts',  'seconds', 'seconds', 1);
1119 $add_throttle_desc->('bps_wr_max_length',  'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
1120 $add_throttle_desc->('iops_max_length',    'integer', 'length of I/O bursts',       'seconds', 'seconds', 1);
1121 $add_throttle_desc->('iops_rd_max_length', 'integer', 'length of read I/O bursts',  'seconds', 'seconds', 1);
1122 $add_throttle_desc->('iops_wr_max_length', 'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
1123
1124 # legacy support
1125 $drivedesc_base{'bps_rd_length'} = { alias => 'bps_rd_max_length' };
1126 $drivedesc_base{'bps_wr_length'} = { alias => 'bps_wr_max_length' };
1127 $drivedesc_base{'iops_rd_length'} = { alias => 'iops_rd_max_length' };
1128 $drivedesc_base{'iops_wr_length'} = { alias => 'iops_wr_max_length' };
1129
1130 my $ide_fmt = {
1131     %drivedesc_base,
1132     %model_fmt,
1133     %ssd_fmt,
1134     %wwn_fmt,
1135 };
1136 PVE::JSONSchema::register_format("pve-qm-ide", $ide_fmt);
1137
1138 my $idedesc = {
1139     optional => 1,
1140     type => 'string', format => $ide_fmt,
1141     description => "Use volume as IDE hard disk or CD-ROM (n is 0 to " .($MAX_IDE_DISKS -1) . ").",
1142 };
1143 PVE::JSONSchema::register_standard_option("pve-qm-ide", $idedesc);
1144
1145 my $scsi_fmt = {
1146     %drivedesc_base,
1147     %iothread_fmt,
1148     %queues_fmt,
1149     %scsiblock_fmt,
1150     %ssd_fmt,
1151     %wwn_fmt,
1152 };
1153 my $scsidesc = {
1154     optional => 1,
1155     type => 'string', format => $scsi_fmt,
1156     description => "Use volume as SCSI hard disk or CD-ROM (n is 0 to " . ($MAX_SCSI_DISKS - 1) . ").",
1157 };
1158 PVE::JSONSchema::register_standard_option("pve-qm-scsi", $scsidesc);
1159
1160 my $sata_fmt = {
1161     %drivedesc_base,
1162     %ssd_fmt,
1163     %wwn_fmt,
1164 };
1165 my $satadesc = {
1166     optional => 1,
1167     type => 'string', format => $sata_fmt,
1168     description => "Use volume as SATA hard disk or CD-ROM (n is 0 to " . ($MAX_SATA_DISKS - 1). ").",
1169 };
1170 PVE::JSONSchema::register_standard_option("pve-qm-sata", $satadesc);
1171
1172 my $virtio_fmt = {
1173     %drivedesc_base,
1174     %iothread_fmt,
1175 };
1176 my $virtiodesc = {
1177     optional => 1,
1178     type => 'string', format => $virtio_fmt,
1179     description => "Use volume as VIRTIO hard disk (n is 0 to " . ($MAX_VIRTIO_DISKS - 1) . ").",
1180 };
1181 PVE::JSONSchema::register_standard_option("pve-qm-virtio", $virtiodesc);
1182
1183 my $alldrive_fmt = {
1184     %drivedesc_base,
1185     %iothread_fmt,
1186     %model_fmt,
1187     %queues_fmt,
1188     %scsiblock_fmt,
1189     %ssd_fmt,
1190     %wwn_fmt,
1191 };
1192
1193 my $efidisk_fmt = {
1194     volume => { alias => 'file' },
1195     file => {
1196         type => 'string',
1197         format => 'pve-volume-id-or-qm-path',
1198         default_key => 1,
1199         format_description => 'volume',
1200         description => "The drive's backing volume.",
1201     },
1202     format => get_standard_option('pve-qm-image-format'),
1203     size => {
1204         type => 'string',
1205         format => 'disk-size',
1206         format_description => 'DiskSize',
1207         description => "Disk size. This is purely informational and has no effect.",
1208         optional => 1,
1209     },
1210 };
1211
1212 my $efidisk_desc = {
1213     optional => 1,
1214     type => 'string', format => $efidisk_fmt,
1215     description => "Configure a Disk for storing EFI vars",
1216 };
1217
1218 PVE::JSONSchema::register_standard_option("pve-qm-efidisk", $efidisk_desc);
1219
1220 my $usb_fmt = {
1221     host => {
1222         default_key => 1,
1223         type => 'string', format => 'pve-qm-usb-device',
1224         format_description => 'HOSTUSBDEVICE|spice',
1225         description => <<EODESCR,
1226 The Host USB device or port or the value 'spice'. HOSTUSBDEVICE syntax is:
1227
1228  'bus-port(.port)*' (decimal numbers) or
1229  'vendor_id:product_id' (hexadeciaml numbers) or
1230  'spice'
1231
1232 You can use the 'lsusb -t' command to list existing usb devices.
1233
1234 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1235
1236 The value 'spice' can be used to add a usb redirection devices for spice.
1237 EODESCR
1238     },
1239     usb3 => {
1240         optional => 1,
1241         type => 'boolean',
1242         description => "Specifies whether if given host option is a USB3 device or port (this does currently not work reliably with spice redirection and is then ignored).",
1243         default => 0,
1244     },
1245 };
1246
1247 my $usbdesc = {
1248     optional => 1,
1249     type => 'string', format => $usb_fmt,
1250     description => "Configure an USB device (n is 0 to 4).",
1251 };
1252 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
1253
1254 my $PCIRE = qr/[a-f0-9]{2}:[a-f0-9]{2}(?:\.[a-f0-9])?/;
1255 my $hostpci_fmt = {
1256     host => {
1257         default_key => 1,
1258         type => 'string',
1259         pattern => qr/$PCIRE(;$PCIRE)*/,
1260         format_description => 'HOSTPCIID[;HOSTPCIID2...]',
1261         description => <<EODESCR,
1262 Host PCI device pass through. The PCI ID of a host's PCI device or a list
1263 of PCI virtual functions of the host. HOSTPCIID syntax is:
1264
1265 'bus:dev.func' (hexadecimal numbers)
1266
1267 You can us the 'lspci' command to list existing PCI devices.
1268 EODESCR
1269     },
1270     rombar => {
1271         type => 'boolean',
1272         description =>  "Specify whether or not the device's ROM will be visible in the guest's memory map.",
1273         optional => 1,
1274         default => 1,
1275     },
1276     romfile => {
1277         type => 'string',
1278         pattern => '[^,;]+',
1279         format_description => 'string',
1280         description => "Custom pci device rom filename (must be located in /usr/share/kvm/).",
1281         optional => 1,
1282     },
1283     pcie => {
1284         type => 'boolean',
1285         description =>  "Choose the PCI-express bus (needs the 'q35' machine model).",
1286         optional => 1,
1287         default => 0,
1288     },
1289     'x-vga' => {
1290         type => 'boolean',
1291         description =>  "Enable vfio-vga device support.",
1292         optional => 1,
1293         default => 0,
1294     },
1295     'mdev' => {
1296         type => 'string',
1297         format_description => 'string',
1298         pattern => '[^/\.:]+',
1299         optional => 1,
1300         description => <<EODESCR
1301 The type of mediated device to use.
1302 An instance of this type will be created on startup of the VM and
1303 will be cleaned up when the VM stops.
1304 EODESCR
1305     }
1306 };
1307 PVE::JSONSchema::register_format('pve-qm-hostpci', $hostpci_fmt);
1308
1309 my $hostpcidesc = {
1310         optional => 1,
1311         type => 'string', format => 'pve-qm-hostpci',
1312         description => "Map host PCI devices into guest.",
1313         verbose_description =>  <<EODESCR,
1314 Map host PCI devices into guest.
1315
1316 NOTE: This option allows direct access to host hardware. So it is no longer
1317 possible to migrate such machines - use with special care.
1318
1319 CAUTION: Experimental! User reported problems with this option.
1320 EODESCR
1321 };
1322 PVE::JSONSchema::register_standard_option("pve-qm-hostpci", $hostpcidesc);
1323
1324 my $serialdesc = {
1325         optional => 1,
1326         type => 'string',
1327         pattern => '(/dev/.+|socket)',
1328         description =>  "Create a serial device inside the VM (n is 0 to 3)",
1329         verbose_description =>  <<EODESCR,
1330 Create a serial device inside the VM (n is 0 to 3), and pass through a
1331 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1332 host side (use 'qm terminal' to open a terminal connection).
1333
1334 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines - use with special care.
1335
1336 CAUTION: Experimental! User reported problems with this option.
1337 EODESCR
1338 };
1339
1340 my $paralleldesc= {
1341         optional => 1,
1342         type => 'string',
1343         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
1344         description =>  "Map host parallel devices (n is 0 to 2).",
1345         verbose_description =>  <<EODESCR,
1346 Map host parallel devices (n is 0 to 2).
1347
1348 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1349
1350 CAUTION: Experimental! User reported problems with this option.
1351 EODESCR
1352 };
1353
1354 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
1355     $confdesc->{"parallel$i"} = $paralleldesc;
1356 }
1357
1358 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
1359     $confdesc->{"serial$i"} = $serialdesc;
1360 }
1361
1362 for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
1363     $confdesc->{"hostpci$i"} = $hostpcidesc;
1364 }
1365
1366 for (my $i = 0; $i < $MAX_IDE_DISKS; $i++)  {
1367     $drivename_hash->{"ide$i"} = 1;
1368     $confdesc->{"ide$i"} = $idedesc;
1369 }
1370
1371 for (my $i = 0; $i < $MAX_SATA_DISKS; $i++)  {
1372     $drivename_hash->{"sata$i"} = 1;
1373     $confdesc->{"sata$i"} = $satadesc;
1374 }
1375
1376 for (my $i = 0; $i < $MAX_SCSI_DISKS; $i++)  {
1377     $drivename_hash->{"scsi$i"} = 1;
1378     $confdesc->{"scsi$i"} = $scsidesc ;
1379 }
1380
1381 for (my $i = 0; $i < $MAX_VIRTIO_DISKS; $i++)  {
1382     $drivename_hash->{"virtio$i"} = 1;
1383     $confdesc->{"virtio$i"} = $virtiodesc;
1384 }
1385
1386 $drivename_hash->{efidisk0} = 1;
1387 $confdesc->{efidisk0} = $efidisk_desc;
1388
1389 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
1390     $confdesc->{"usb$i"} = $usbdesc;
1391 }
1392
1393 my $unuseddesc = {
1394     optional => 1,
1395     type => 'string', format => 'pve-volume-id',
1396     description => "Reference to unused volumes. This is used internally, and should not be modified manually.",
1397 };
1398
1399 for (my $i = 0; $i < $MAX_UNUSED_DISKS; $i++)  {
1400     $confdesc->{"unused$i"} = $unuseddesc;
1401 }
1402
1403 my $kvm_api_version = 0;
1404
1405 sub kvm_version {
1406     return $kvm_api_version if $kvm_api_version;
1407
1408     open my $fh, '<', '/dev/kvm'
1409         or return undef;
1410
1411     # 0xae00 => KVM_GET_API_VERSION
1412     $kvm_api_version = ioctl($fh, 0xae00, 0);
1413
1414     return $kvm_api_version;
1415 }
1416
1417 my $kvm_user_version;
1418
1419 sub kvm_user_version {
1420
1421     return $kvm_user_version if $kvm_user_version;
1422
1423     $kvm_user_version = 'unknown';
1424
1425     my $code = sub {
1426         my $line = shift;
1427         if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1428             $kvm_user_version = $2;
1429         }
1430     };
1431
1432     eval { run_command("kvm -version", outfunc => $code); };
1433     warn $@ if $@;
1434
1435     return $kvm_user_version;
1436
1437 }
1438
1439 sub kernel_has_vhost_net {
1440     return -c '/dev/vhost-net';
1441 }
1442
1443 sub valid_drive_names {
1444     # order is important - used to autoselect boot disk
1445     return ((map { "ide$_" } (0 .. ($MAX_IDE_DISKS - 1))),
1446             (map { "scsi$_" } (0 .. ($MAX_SCSI_DISKS - 1))),
1447             (map { "virtio$_" } (0 .. ($MAX_VIRTIO_DISKS - 1))),
1448             (map { "sata$_" } (0 .. ($MAX_SATA_DISKS - 1))),
1449             'efidisk0');
1450 }
1451
1452 sub is_valid_drivename {
1453     my $dev = shift;
1454
1455     return defined($drivename_hash->{$dev});
1456 }
1457
1458 sub option_exists {
1459     my $key = shift;
1460     return defined($confdesc->{$key});
1461 }
1462
1463 sub nic_models {
1464     return $nic_model_list;
1465 }
1466
1467 sub os_list_description {
1468
1469     return {
1470         other => 'Other',
1471         wxp => 'Windows XP',
1472         w2k => 'Windows 2000',
1473         w2k3 =>, 'Windows 2003',
1474         w2k8 => 'Windows 2008',
1475         wvista => 'Windows Vista',
1476         win7 => 'Windows 7',
1477         win8 => 'Windows 8/2012',
1478         win10 => 'Windows 10/2016',
1479         l24 => 'Linux 2.4',
1480         l26 => 'Linux 2.6',
1481     };
1482 }
1483
1484 my $cdrom_path;
1485
1486 sub get_cdrom_path {
1487
1488     return  $cdrom_path if $cdrom_path;
1489
1490     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
1491     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
1492     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
1493 }
1494
1495 sub get_iso_path {
1496     my ($storecfg, $vmid, $cdrom) = @_;
1497
1498     if ($cdrom eq 'cdrom') {
1499         return get_cdrom_path();
1500     } elsif ($cdrom eq 'none') {
1501         return '';
1502     } elsif ($cdrom =~ m|^/|) {
1503         return $cdrom;
1504     } else {
1505         return PVE::Storage::path($storecfg, $cdrom);
1506     }
1507 }
1508
1509 # try to convert old style file names to volume IDs
1510 sub filename_to_volume_id {
1511     my ($vmid, $file, $media) = @_;
1512
1513      if (!($file eq 'none' || $file eq 'cdrom' ||
1514           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1515
1516         return undef if $file =~ m|/|;
1517
1518         if ($media && $media eq 'cdrom') {
1519             $file = "local:iso/$file";
1520         } else {
1521             $file = "local:$vmid/$file";
1522         }
1523     }
1524
1525     return $file;
1526 }
1527
1528 sub verify_media_type {
1529     my ($opt, $vtype, $media) = @_;
1530
1531     return if !$media;
1532
1533     my $etype;
1534     if ($media eq 'disk') {
1535         $etype = 'images';
1536     } elsif ($media eq 'cdrom') {
1537         $etype = 'iso';
1538     } else {
1539         die "internal error";
1540     }
1541
1542     return if ($vtype eq $etype);
1543
1544     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1545 }
1546
1547 sub cleanup_drive_path {
1548     my ($opt, $storecfg, $drive) = @_;
1549
1550     # try to convert filesystem paths to volume IDs
1551
1552     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1553         ($drive->{file} !~ m|^/dev/.+|) &&
1554         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1555         ($drive->{file} !~ m/^\d+$/)) {
1556         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1557         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"}) if !$vtype;
1558         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1559         verify_media_type($opt, $vtype, $drive->{media});
1560         $drive->{file} = $volid;
1561     }
1562
1563     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1564 }
1565
1566 sub parse_hotplug_features {
1567     my ($data) = @_;
1568
1569     my $res = {};
1570
1571     return $res if $data eq '0';
1572
1573     $data = $confdesc->{hotplug}->{default} if $data eq '1';
1574
1575     foreach my $feature (PVE::Tools::split_list($data)) {
1576         if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
1577             $res->{$1} = 1;
1578         } else {
1579             die "invalid hotplug feature '$feature'\n";
1580         }
1581     }
1582     return $res;
1583 }
1584
1585 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1586 sub pve_verify_hotplug_features {
1587     my ($value, $noerr) = @_;
1588
1589     return $value if parse_hotplug_features($value);
1590
1591     return undef if $noerr;
1592
1593     die "unable to parse hotplug option\n";
1594 }
1595
1596 # ideX = [volume=]volume-id[,media=d][,cyls=c,heads=h,secs=s[,trans=t]]
1597 #        [,snapshot=on|off][,cache=on|off][,format=f][,backup=yes|no]
1598 #        [,rerror=ignore|report|stop][,werror=enospc|ignore|report|stop]
1599 #        [,aio=native|threads][,discard=ignore|on][,detect_zeroes=on|off]
1600 #        [,iothread=on][,serial=serial][,model=model]
1601
1602 sub parse_drive {
1603     my ($key, $data) = @_;
1604
1605     my ($interface, $index);
1606
1607     if ($key =~ m/^([^\d]+)(\d+)$/) {
1608         $interface = $1;
1609         $index = $2;
1610     } else {
1611         return undef;
1612     }
1613
1614     my $desc = $key =~ /^unused\d+$/ ? $alldrive_fmt
1615                                      : $confdesc->{$key}->{format};
1616     if (!$desc) {
1617         warn "invalid drive key: $key\n";
1618         return undef;
1619     }
1620     my $res = eval { PVE::JSONSchema::parse_property_string($desc, $data) };
1621     return undef if !$res;
1622     $res->{interface} = $interface;
1623     $res->{index} = $index;
1624
1625     my $error = 0;
1626     foreach my $opt (qw(bps bps_rd bps_wr)) {
1627         if (my $bps = defined(delete $res->{$opt})) {
1628             if (defined($res->{"m$opt"})) {
1629                 warn "both $opt and m$opt specified\n";
1630                 ++$error;
1631                 next;
1632             }
1633             $res->{"m$opt"} = sprintf("%.3f", $bps / (1024*1024.0));
1634         }
1635     }
1636
1637     # can't use the schema's 'requires' because of the mbps* => bps* "transforming aliases"
1638     for my $requirement (
1639         [mbps_max => 'mbps'],
1640         [mbps_rd_max => 'mbps_rd'],
1641         [mbps_wr_max => 'mbps_wr'],
1642         [miops_max => 'miops'],
1643         [miops_rd_max => 'miops_rd'],
1644         [miops_wr_max => 'miops_wr'],
1645         [bps_max_length => 'mbps_max'],
1646         [bps_rd_max_length => 'mbps_rd_max'],
1647         [bps_wr_max_length => 'mbps_wr_max'],
1648         [iops_max_length => 'iops_max'],
1649         [iops_rd_max_length => 'iops_rd_max'],
1650         [iops_wr_max_length => 'iops_wr_max']) {
1651         my ($option, $requires) = @$requirement;
1652         if ($res->{$option} && !$res->{$requires}) {
1653             warn "$option requires $requires\n";
1654             ++$error;
1655         }
1656     }
1657
1658     return undef if $error;
1659
1660     return undef if $res->{mbps_rd} && $res->{mbps};
1661     return undef if $res->{mbps_wr} && $res->{mbps};
1662     return undef if $res->{iops_rd} && $res->{iops};
1663     return undef if $res->{iops_wr} && $res->{iops};
1664
1665     if ($res->{media} && ($res->{media} eq 'cdrom')) {
1666         return undef if $res->{snapshot} || $res->{trans} || $res->{format};
1667         return undef if $res->{heads} || $res->{secs} || $res->{cyls};
1668         return undef if $res->{interface} eq 'virtio';
1669     }
1670
1671     if (my $size = $res->{size}) {
1672         return undef if !defined($res->{size} = PVE::JSONSchema::parse_size($size));
1673     }
1674
1675     return $res;
1676 }
1677
1678 sub print_drive {
1679     my ($vmid, $drive) = @_;
1680     my $data = { %$drive };
1681     delete $data->{$_} for qw(index interface);
1682     return PVE::JSONSchema::print_property_string($data, $alldrive_fmt);
1683 }
1684
1685 sub scsi_inquiry {
1686     my($fh, $noerr) = @_;
1687
1688     my $SG_IO = 0x2285;
1689     my $SG_GET_VERSION_NUM = 0x2282;
1690
1691     my $versionbuf = "\x00" x 8;
1692     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1693     if (!$ret) {
1694         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1695         return undef;
1696     }
1697     my $version = unpack("I", $versionbuf);
1698     if ($version < 30000) {
1699         die "scsi generic interface too old\n"  if !$noerr;
1700         return undef;
1701     }
1702
1703     my $buf = "\x00" x 36;
1704     my $sensebuf = "\x00" x 8;
1705     my $cmd = pack("C x3 C x1", 0x12, 36);
1706
1707     # see /usr/include/scsi/sg.h
1708     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1709
1710     my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1711                       length($sensebuf), 0, length($buf), $buf,
1712                       $cmd, $sensebuf, 6000);
1713
1714     $ret = ioctl($fh, $SG_IO, $packet);
1715     if (!$ret) {
1716         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1717         return undef;
1718     }
1719
1720     my @res = unpack($sg_io_hdr_t, $packet);
1721     if ($res[17] || $res[18]) {
1722         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1723         return undef;
1724     }
1725
1726     my $res = {};
1727     (my $byte0, my $byte1, $res->{vendor},
1728      $res->{product}, $res->{revision}) = unpack("C C x6 A8 A16 A4", $buf);
1729
1730     $res->{removable} = $byte1 & 128 ? 1 : 0;
1731     $res->{type} = $byte0 & 31;
1732
1733     return $res;
1734 }
1735
1736 sub path_is_scsi {
1737     my ($path) = @_;
1738
1739     my $fh = IO::File->new("+<$path") || return undef;
1740     my $res = scsi_inquiry($fh, 1);
1741     close($fh);
1742
1743     return $res;
1744 }
1745
1746 sub machine_type_is_q35 {
1747     my ($conf) = @_;
1748
1749     return $conf->{machine} && ($conf->{machine} =~ m/q35/) ? 1 : 0;
1750 }
1751
1752 sub print_tabletdevice_full {
1753     my ($conf, $arch) = @_;
1754
1755     my $q35 = machine_type_is_q35($conf);
1756
1757     # we use uhci for old VMs because tablet driver was buggy in older qemu
1758     my $usbbus;
1759     if (machine_type_is_q35($conf) || $arch eq 'aarch64') {
1760         $usbbus = 'ehci';
1761     } else {
1762         $usbbus = 'uhci';
1763     }
1764
1765     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1766 }
1767
1768 sub print_keyboarddevice_full {
1769     my ($conf, $arch, $machine) = @_;
1770
1771     return undef if $arch ne 'aarch64';
1772
1773     return "usb-kbd,id=keyboard,bus=ehci.0,port=2";
1774 }
1775
1776 sub print_drivedevice_full {
1777     my ($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type) = @_;
1778
1779     my $device = '';
1780     my $maxdev = 0;
1781
1782     if ($drive->{interface} eq 'virtio') {
1783         my $pciaddr = print_pci_addr("$drive->{interface}$drive->{index}", $bridges, $arch, $machine_type);
1784         $device = "virtio-blk-pci,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}$pciaddr";
1785         $device .= ",iothread=iothread-$drive->{interface}$drive->{index}" if $drive->{iothread};
1786     } elsif ($drive->{interface} eq 'scsi') {
1787
1788         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1789         my $unit = $drive->{index} % $maxdev;
1790         my $devicetype = 'hd';
1791         my $path = '';
1792         if (drive_is_cdrom($drive)) {
1793             $devicetype = 'cd';
1794         } else {
1795             if ($drive->{file} =~ m|^/|) {
1796                 $path = $drive->{file};
1797                 if (my $info = path_is_scsi($path)) {
1798                     if ($info->{type} == 0 && $drive->{scsiblock}) {
1799                         $devicetype = 'block';
1800                     } elsif ($info->{type} == 1) { # tape
1801                         $devicetype = 'generic';
1802                     }
1803                 }
1804             } else {
1805                  $path = PVE::Storage::path($storecfg, $drive->{file});
1806             }
1807
1808             if($path =~ m/^iscsi\:\/\//){
1809                 $devicetype = 'generic';
1810             }
1811         }
1812
1813         if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)){
1814             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1815         } else {
1816             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0,lun=$drive->{index},drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1817         }
1818
1819         if ($drive->{ssd} && ($devicetype eq 'block' || $devicetype eq 'hd')) {
1820             $device .= ",rotation_rate=1";
1821         }
1822         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1823
1824     } elsif ($drive->{interface} eq 'ide' || $drive->{interface} eq 'sata') {
1825         my $maxdev = ($drive->{interface} eq 'sata') ? $MAX_SATA_DISKS : 2;
1826         my $controller = int($drive->{index} / $maxdev);
1827         my $unit = $drive->{index} % $maxdev;
1828         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1829
1830         $device = "ide-$devicetype";
1831         if ($drive->{interface} eq 'ide') {
1832             $device .= ",bus=ide.$controller,unit=$unit";
1833         } else {
1834             $device .= ",bus=ahci$controller.$unit";
1835         }
1836         $device .= ",drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1837
1838         if ($devicetype eq 'hd') {
1839             if (my $model = $drive->{model}) {
1840                 $model = URI::Escape::uri_unescape($model);
1841                 $device .= ",model=$model";
1842             }
1843             if ($drive->{ssd}) {
1844                 $device .= ",rotation_rate=1";
1845             }
1846         }
1847         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1848     } elsif ($drive->{interface} eq 'usb') {
1849         die "implement me";
1850         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1851     } else {
1852         die "unsupported interface type";
1853     }
1854
1855     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1856
1857     if (my $serial = $drive->{serial}) {
1858         $serial = URI::Escape::uri_unescape($serial);
1859         $device .= ",serial=$serial";
1860     }
1861
1862
1863     return $device;
1864 }
1865
1866 sub get_initiator_name {
1867     my $initiator;
1868
1869     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return undef;
1870     while (defined(my $line = <$fh>)) {
1871         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1872         $initiator = $1;
1873         last;
1874     }
1875     $fh->close();
1876
1877     return $initiator;
1878 }
1879
1880 sub print_drive_full {
1881     my ($storecfg, $vmid, $drive) = @_;
1882
1883     my $path;
1884     my $volid = $drive->{file};
1885     my $format;
1886
1887     if (drive_is_cdrom($drive)) {
1888         $path = get_iso_path($storecfg, $vmid, $volid);
1889     } else {
1890         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1891         if ($storeid) {
1892             $path = PVE::Storage::path($storecfg, $volid);
1893             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
1894             $format = qemu_img_format($scfg, $volname);
1895         } else {
1896             $path = $volid;
1897             $format = "raw";
1898         }
1899    }
1900
1901     my $opts = '';
1902     my @qemu_drive_options = qw(heads secs cyls trans media format cache rerror werror aio discard);
1903     foreach my $o (@qemu_drive_options) {
1904         $opts .= ",$o=$drive->{$o}" if defined($drive->{$o});
1905     }
1906
1907     # snapshot only accepts on|off
1908     if (defined($drive->{snapshot})) {
1909         my $v = $drive->{snapshot} ? 'on' : 'off';
1910         $opts .= ",snapshot=$v";
1911     }
1912
1913     foreach my $type (['', '-total'], [_rd => '-read'], [_wr => '-write']) {
1914         my ($dir, $qmpname) = @$type;
1915         if (my $v = $drive->{"mbps$dir"}) {
1916             $opts .= ",throttling.bps$qmpname=".int($v*1024*1024);
1917         }
1918         if (my $v = $drive->{"mbps${dir}_max"}) {
1919             $opts .= ",throttling.bps$qmpname-max=".int($v*1024*1024);
1920         }
1921         if (my $v = $drive->{"bps${dir}_max_length"}) {
1922             $opts .= ",throttling.bps$qmpname-max-length=$v";
1923         }
1924         if (my $v = $drive->{"iops${dir}"}) {
1925             $opts .= ",throttling.iops$qmpname=$v";
1926         }
1927         if (my $v = $drive->{"iops${dir}_max"}) {
1928             $opts .= ",throttling.iops$qmpname-max=$v";
1929         }
1930         if (my $v = $drive->{"iops${dir}_max_length"}) {
1931             $opts .= ",throttling.iops$qmpname-max-length=$v";
1932         }
1933     }
1934
1935     $opts .= ",format=$format" if $format && !$drive->{format};
1936
1937     my $cache_direct = 0;
1938
1939     if (my $cache = $drive->{cache}) {
1940         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1941     } elsif (!drive_is_cdrom($drive)) {
1942         $opts .= ",cache=none";
1943         $cache_direct = 1;
1944     }
1945
1946     # aio native works only with O_DIRECT
1947     if (!$drive->{aio}) {
1948         if($cache_direct) {
1949             $opts .= ",aio=native";
1950         } else {
1951             $opts .= ",aio=threads";
1952         }
1953     }
1954
1955     if (!drive_is_cdrom($drive)) {
1956         my $detectzeroes;
1957         if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
1958             $detectzeroes = 'off';
1959         } elsif ($drive->{discard}) {
1960             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
1961         } else {
1962             # This used to be our default with discard not being specified:
1963             $detectzeroes = 'on';
1964         }
1965         $opts .= ",detect-zeroes=$detectzeroes" if $detectzeroes;
1966     }
1967
1968     my $pathinfo = $path ? "file=$path," : '';
1969
1970     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1971 }
1972
1973 sub print_netdevice_full {
1974     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files, $arch, $machine_type) = @_;
1975
1976     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
1977
1978     my $device = $net->{model};
1979     if ($net->{model} eq 'virtio') {
1980          $device = 'virtio-net-pci';
1981      };
1982
1983     my $pciaddr = print_pci_addr("$netid", $bridges, $arch, $machine_type);
1984     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1985     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
1986         #Consider we have N queues, the number of vectors needed is 2*N + 2 (plus one config interrupt and control vq)
1987         my $vectors = $net->{queues} * 2 + 2;
1988         $tmpstr .= ",vectors=$vectors,mq=on";
1989     }
1990     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
1991
1992     if ($use_old_bios_files) {
1993         my $romfile;
1994         if ($device eq 'virtio-net-pci') {
1995             $romfile = 'pxe-virtio.rom';
1996         } elsif ($device eq 'e1000') {
1997             $romfile = 'pxe-e1000.rom';
1998         } elsif ($device eq 'ne2k') {
1999             $romfile = 'pxe-ne2k_pci.rom';
2000         } elsif ($device eq 'pcnet') {
2001             $romfile = 'pxe-pcnet.rom';
2002         } elsif ($device eq 'rtl8139') {
2003             $romfile = 'pxe-rtl8139.rom';
2004         }
2005         $tmpstr .= ",romfile=$romfile" if $romfile;
2006     }
2007
2008     return $tmpstr;
2009 }
2010
2011 sub print_netdev_full {
2012     my ($vmid, $conf, $arch, $net, $netid, $hotplug) = @_;
2013
2014     my $i = '';
2015     if ($netid =~ m/^net(\d+)$/) {
2016         $i = int($1);
2017     }
2018
2019     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
2020
2021     my $ifname = "tap${vmid}i$i";
2022
2023     # kvm uses TUNSETIFF ioctl, and that limits ifname length
2024     die "interface name '$ifname' is too long (max 15 character)\n"
2025         if length($ifname) >= 16;
2026
2027     my $vhostparam = '';
2028     if (is_native($arch)) {
2029         $vhostparam = ',vhost=on' if kernel_has_vhost_net() && $net->{model} eq 'virtio';
2030     }
2031
2032     my $vmname = $conf->{name} || "vm$vmid";
2033
2034     my $netdev = "";
2035     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
2036
2037     if ($net->{bridge}) {
2038         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script,downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
2039     } else {
2040         $netdev = "type=user,id=$netid,hostname=$vmname";
2041     }
2042
2043     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
2044
2045     return $netdev;
2046 }
2047
2048
2049 sub print_cpu_device {
2050     my ($conf, $id) = @_;
2051
2052     my $kvm = $conf->{kvm} // 1;
2053     my $cpu = $kvm ? "kvm64" : "qemu64";
2054     if (my $cputype = $conf->{cpu}) {
2055         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
2056             or die "Cannot parse cpu description: $cputype\n";
2057         $cpu = $cpuconf->{cputype};
2058     }
2059
2060     my $cores = $conf->{cores} || 1;
2061
2062     my $current_core = ($id - 1) % $cores;
2063     my $current_socket = int(($id - 1 - $current_core)/$cores);
2064
2065     return "$cpu-x86_64-cpu,id=cpu$id,socket-id=$current_socket,core-id=$current_core,thread-id=0";
2066 }
2067
2068 my $vga_map = {
2069     'cirrus' => 'cirrus-vga',
2070     'std' => 'VGA',
2071     'vmware' => 'vmware-svga',
2072     'virtio' => 'virtio-vga',
2073 };
2074
2075 sub print_vga_device {
2076     my ($conf, $vga, $arch, $machine, $id, $qxlnum, $bridges) = @_;
2077
2078     my $type = $vga_map->{$vga->{type}};
2079     if ($arch eq 'aarch64' && defined($type) && $type eq 'virtio-vga') {
2080         $type = 'virtio-gpu';
2081     }
2082     my $vgamem_mb = $vga->{memory};
2083     if ($qxlnum) {
2084         $type = $id ? 'qxl' : 'qxl-vga';
2085     }
2086     die "no devicetype for $vga->{type}\n" if !$type;
2087
2088     my $memory = "";
2089     if ($vgamem_mb) {
2090         if ($vga->{type} eq 'virtio') {
2091             my $bytes = PVE::Tools::convert_size($vgamem_mb, "mb" => "b");
2092             $memory = ",max_hostmem=$bytes";
2093         } elsif ($qxlnum) {
2094             # from https://www.spice-space.org/multiple-monitors.html
2095             $memory = ",vgamem_mb=$vga->{memory}";
2096             my $ram = $vgamem_mb * 4;
2097             my $vram = $vgamem_mb * 2;
2098             $memory .= ",ram_size_mb=$ram,vram_size_mb=$vram";
2099         } else {
2100             $memory = ",vgamem_mb=$vga->{memory}";
2101         }
2102     } elsif ($qxlnum && $id) {
2103         $memory = ",ram_size=67108864,vram_size=33554432";
2104     }
2105
2106     my $q35 = machine_type_is_q35($conf);
2107     my $vgaid = "vga" . ($id // '');
2108     my $pciaddr;
2109
2110     if ($q35 && $vgaid eq 'vga') {
2111         # the first display uses pcie.0 bus on q35 machines
2112         $pciaddr = print_pcie_addr($vgaid, $bridges, $arch, $machine);
2113     } else {
2114         $pciaddr = print_pci_addr($vgaid, $bridges, $arch, $machine);
2115     }
2116
2117     return "$type,id=${vgaid}${memory}${pciaddr}";
2118 }
2119
2120 sub drive_is_cloudinit {
2121     my ($drive) = @_;
2122     return $drive->{file} =~ m@[:/]vm-\d+-cloudinit(?:\.$QEMU_FORMAT_RE)?$@;
2123 }
2124
2125 sub drive_is_cdrom {
2126     my ($drive, $exclude_cloudinit) = @_;
2127
2128     return 0 if $exclude_cloudinit && drive_is_cloudinit($drive);
2129
2130     return $drive && $drive->{media} && ($drive->{media} eq 'cdrom');
2131
2132 }
2133
2134 sub parse_number_sets {
2135     my ($set) = @_;
2136     my $res = [];
2137     foreach my $part (split(/;/, $set)) {
2138         if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
2139             die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
2140             push @$res, [ $1, $2 ];
2141         } else {
2142             die "invalid range: $part\n";
2143         }
2144     }
2145     return $res;
2146 }
2147
2148 sub parse_numa {
2149     my ($data) = @_;
2150
2151     my $res = PVE::JSONSchema::parse_property_string($numa_fmt, $data);
2152     $res->{cpus} = parse_number_sets($res->{cpus}) if defined($res->{cpus});
2153     $res->{hostnodes} = parse_number_sets($res->{hostnodes}) if defined($res->{hostnodes});
2154     return $res;
2155 }
2156
2157 sub parse_hostpci {
2158     my ($value) = @_;
2159
2160     return undef if !$value;
2161
2162     my $res = PVE::JSONSchema::parse_property_string($hostpci_fmt, $value);
2163
2164     my @idlist = split(/;/, $res->{host});
2165     delete $res->{host};
2166     foreach my $id (@idlist) {
2167         if ($id =~ m/\./) { # full id 00:00.1
2168             push @{$res->{pciid}}, {
2169                 id => $id,
2170             };
2171         } else { # partial id 00:00
2172             $res->{pciid} = PVE::SysFSTools::lspci($id);
2173         }
2174     }
2175     return $res;
2176 }
2177
2178 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
2179 sub parse_net {
2180     my ($data) = @_;
2181
2182     my $res = eval { PVE::JSONSchema::parse_property_string($net_fmt, $data) };
2183     if ($@) {
2184         warn $@;
2185         return undef;
2186     }
2187     if (!defined($res->{macaddr})) {
2188         my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
2189         $res->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix});
2190     }
2191     return $res;
2192 }
2193
2194 # ipconfigX ip=cidr,gw=ip,ip6=cidr,gw6=ip
2195 sub parse_ipconfig {
2196     my ($data) = @_;
2197
2198     my $res = eval { PVE::JSONSchema::parse_property_string($ipconfig_fmt, $data) };
2199     if ($@) {
2200         warn $@;
2201         return undef;
2202     }
2203
2204     if ($res->{gw} && !$res->{ip}) {
2205         warn 'gateway specified without specifying an IP address';
2206         return undef;
2207     }
2208     if ($res->{gw6} && !$res->{ip6}) {
2209         warn 'IPv6 gateway specified without specifying an IPv6 address';
2210         return undef;
2211     }
2212     if ($res->{gw} && $res->{ip} eq 'dhcp') {
2213         warn 'gateway specified together with DHCP';
2214         return undef;
2215     }
2216     if ($res->{gw6} && $res->{ip6} !~ /^$IPV6RE/) {
2217         # gw6 + auto/dhcp
2218         warn "IPv6 gateway specified together with $res->{ip6} address";
2219         return undef;
2220     }
2221
2222     if (!$res->{ip} && !$res->{ip6}) {
2223         return { ip => 'dhcp', ip6 => 'dhcp' };
2224     }
2225
2226     return $res;
2227 }
2228
2229 sub print_net {
2230     my $net = shift;
2231
2232     return PVE::JSONSchema::print_property_string($net, $net_fmt);
2233 }
2234
2235 sub add_random_macs {
2236     my ($settings) = @_;
2237
2238     foreach my $opt (keys %$settings) {
2239         next if $opt !~ m/^net(\d+)$/;
2240         my $net = parse_net($settings->{$opt});
2241         next if !$net;
2242         $settings->{$opt} = print_net($net);
2243     }
2244 }
2245
2246 sub vm_is_volid_owner {
2247     my ($storecfg, $vmid, $volid) = @_;
2248
2249     if ($volid !~  m|^/|) {
2250         my ($path, $owner);
2251         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
2252         if ($owner && ($owner == $vmid)) {
2253             return 1;
2254         }
2255     }
2256
2257     return undef;
2258 }
2259
2260 sub split_flagged_list {
2261     my $text = shift || '';
2262     $text =~ s/[,;]/ /g;
2263     $text =~ s/^\s+//;
2264     return { map { /^(!?)(.*)$/ && ($2, $1) } ($text =~ /\S+/g) };
2265 }
2266
2267 sub join_flagged_list {
2268     my ($how, $lst) = @_;
2269     join $how, map { $lst->{$_} . $_ } keys %$lst;
2270 }
2271
2272 sub vmconfig_delete_pending_option {
2273     my ($conf, $key, $force) = @_;
2274
2275     delete $conf->{pending}->{$key};
2276     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
2277     $pending_delete_hash->{$key} = $force ? '!' : '';
2278     $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
2279 }
2280
2281 sub vmconfig_undelete_pending_option {
2282     my ($conf, $key) = @_;
2283
2284     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
2285     delete $pending_delete_hash->{$key};
2286
2287     if (%$pending_delete_hash) {
2288         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
2289     } else {
2290         delete $conf->{pending}->{delete};
2291     }
2292 }
2293
2294 sub vmconfig_register_unused_drive {
2295     my ($storecfg, $vmid, $conf, $drive) = @_;
2296
2297     if (drive_is_cloudinit($drive)) {
2298         eval { PVE::Storage::vdisk_free($storecfg, $drive->{file}) };
2299         warn $@ if $@;
2300     } elsif (!drive_is_cdrom($drive)) {
2301         my $volid = $drive->{file};
2302         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
2303             PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
2304         }
2305     }
2306 }
2307
2308 sub vmconfig_cleanup_pending {
2309     my ($conf) = @_;
2310
2311     # remove pending changes when nothing changed
2312     my $changes;
2313     foreach my $opt (keys %{$conf->{pending}}) {
2314         if (defined($conf->{$opt}) && ($conf->{pending}->{$opt} eq  $conf->{$opt})) {
2315             $changes = 1;
2316             delete $conf->{pending}->{$opt};
2317         }
2318     }
2319
2320     my $current_delete_hash = split_flagged_list($conf->{pending}->{delete});
2321     my $pending_delete_hash = {};
2322     while (my ($opt, $force) = each %$current_delete_hash) {
2323         if (defined($conf->{$opt})) {
2324             $pending_delete_hash->{$opt} = $force;
2325         } else {
2326             $changes = 1;
2327         }
2328     }
2329
2330     if (%$pending_delete_hash) {
2331         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
2332     } else {
2333         delete $conf->{pending}->{delete};
2334     }
2335
2336     return $changes;
2337 }
2338
2339 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str]
2340 my $smbios1_fmt = {
2341     uuid => {
2342         type => 'string',
2343         pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
2344         format_description => 'UUID',
2345         description => "Set SMBIOS1 UUID.",
2346         optional => 1,
2347     },
2348     version => {
2349         type => 'string',
2350         pattern => '\S+',
2351         format_description => 'string',
2352         description => "Set SMBIOS1 version.",
2353         optional => 1,
2354     },
2355     serial => {
2356         type => 'string',
2357         pattern => '\S+',
2358         format_description => 'string',
2359         description => "Set SMBIOS1 serial number.",
2360         optional => 1,
2361     },
2362     manufacturer => {
2363         type => 'string',
2364         pattern => '\S+',
2365         format_description => 'string',
2366         description => "Set SMBIOS1 manufacturer.",
2367         optional => 1,
2368     },
2369     product => {
2370         type => 'string',
2371         pattern => '\S+',
2372         format_description => 'string',
2373         description => "Set SMBIOS1 product ID.",
2374         optional => 1,
2375     },
2376     sku => {
2377         type => 'string',
2378         pattern => '\S+',
2379         format_description => 'string',
2380         description => "Set SMBIOS1 SKU string.",
2381         optional => 1,
2382     },
2383     family => {
2384         type => 'string',
2385         pattern => '\S+',
2386         format_description => 'string',
2387         description => "Set SMBIOS1 family string.",
2388         optional => 1,
2389     },
2390 };
2391
2392 sub parse_smbios1 {
2393     my ($data) = @_;
2394
2395     my $res = eval { PVE::JSONSchema::parse_property_string($smbios1_fmt, $data) };
2396     warn $@ if $@;
2397     return $res;
2398 }
2399
2400 sub print_smbios1 {
2401     my ($smbios1) = @_;
2402     return PVE::JSONSchema::print_property_string($smbios1, $smbios1_fmt);
2403 }
2404
2405 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_fmt);
2406
2407 PVE::JSONSchema::register_format('pve-qm-bootdisk', \&verify_bootdisk);
2408 sub verify_bootdisk {
2409     my ($value, $noerr) = @_;
2410
2411     return $value if is_valid_drivename($value);
2412
2413     return undef if $noerr;
2414
2415     die "invalid boot disk '$value'\n";
2416 }
2417
2418 sub parse_watchdog {
2419     my ($value) = @_;
2420
2421     return undef if !$value;
2422
2423     my $res = eval { PVE::JSONSchema::parse_property_string($watchdog_fmt, $value) };
2424     warn $@ if $@;
2425     return $res;
2426 }
2427
2428 sub parse_guest_agent {
2429     my ($value) = @_;
2430
2431     return {} if !defined($value->{agent});
2432
2433     my $res = eval { PVE::JSONSchema::parse_property_string($agent_fmt, $value->{agent}) };
2434     warn $@ if $@;
2435
2436     # if the agent is disabled ignore the other potentially set properties
2437     return {} if !$res->{enabled};
2438     return $res;
2439 }
2440
2441 sub parse_vga {
2442     my ($value) = @_;
2443
2444     return {} if !$value;
2445     my $res = eval { PVE::JSONSchema::parse_property_string($vga_fmt, $value) };
2446     warn $@ if $@;
2447     return $res;
2448 }
2449
2450 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
2451 sub verify_usb_device {
2452     my ($value, $noerr) = @_;
2453
2454     return $value if parse_usb_device($value);
2455
2456     return undef if $noerr;
2457
2458     die "unable to parse usb device\n";
2459 }
2460
2461 # add JSON properties for create and set function
2462 sub json_config_properties {
2463     my $prop = shift;
2464
2465     foreach my $opt (keys %$confdesc) {
2466         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate' || $opt eq 'runningmachine';
2467         $prop->{$opt} = $confdesc->{$opt};
2468     }
2469
2470     return $prop;
2471 }
2472
2473 # return copy of $confdesc_cloudinit to generate documentation
2474 sub cloudinit_config_properties {
2475
2476     return dclone($confdesc_cloudinit);
2477 }
2478
2479 sub check_type {
2480     my ($key, $value) = @_;
2481
2482     die "unknown setting '$key'\n" if !$confdesc->{$key};
2483
2484     my $type = $confdesc->{$key}->{type};
2485
2486     if (!defined($value)) {
2487         die "got undefined value\n";
2488     }
2489
2490     if ($value =~ m/[\n\r]/) {
2491         die "property contains a line feed\n";
2492     }
2493
2494     if ($type eq 'boolean') {
2495         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
2496         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
2497         die "type check ('boolean') failed - got '$value'\n";
2498     } elsif ($type eq 'integer') {
2499         return int($1) if $value =~ m/^(\d+)$/;
2500         die "type check ('integer') failed - got '$value'\n";
2501     } elsif ($type eq 'number') {
2502         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
2503         die "type check ('number') failed - got '$value'\n";
2504     } elsif ($type eq 'string') {
2505         if (my $fmt = $confdesc->{$key}->{format}) {
2506             PVE::JSONSchema::check_format($fmt, $value);
2507             return $value;
2508         }
2509         $value =~ s/^\"(.*)\"$/$1/;
2510         return $value;
2511     } else {
2512         die "internal error"
2513     }
2514 }
2515
2516 sub touch_config {
2517     my ($vmid) = @_;
2518
2519     my $conf = PVE::QemuConfig->config_file($vmid);
2520     utime undef, undef, $conf;
2521 }
2522
2523 sub destroy_vm {
2524     my ($storecfg, $vmid, $keep_empty_config, $skiplock) = @_;
2525
2526     my $conffile = PVE::QemuConfig->config_file($vmid);
2527
2528     my $conf = PVE::QemuConfig->load_config($vmid);
2529
2530     PVE::QemuConfig->check_lock($conf) if !$skiplock;
2531
2532     if ($conf->{template}) {
2533         # check if any base image is still used by a linked clone
2534         foreach_drive($conf, sub {
2535                 my ($ds, $drive) = @_;
2536
2537                 return if drive_is_cdrom($drive);
2538
2539                 my $volid = $drive->{file};
2540
2541                 return if !$volid || $volid =~ m|^/|;
2542
2543                 die "base volume '$volid' is still in use by linked cloned\n"
2544                     if PVE::Storage::volume_is_base_and_used($storecfg, $volid);
2545
2546         });
2547     }
2548
2549     # only remove disks owned by this VM
2550     foreach_drive($conf, sub {
2551         my ($ds, $drive) = @_;
2552
2553         return if drive_is_cdrom($drive, 1);
2554
2555         my $volid = $drive->{file};
2556
2557         return if !$volid || $volid =~ m|^/|;
2558
2559         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
2560         return if !$path || !$owner || ($owner != $vmid);
2561
2562         eval {
2563             PVE::Storage::vdisk_free($storecfg, $volid);
2564         };
2565         warn "Could not remove disk '$volid', check manually: $@" if $@;
2566
2567     });
2568
2569     if ($keep_empty_config) {
2570         PVE::Tools::file_set_contents($conffile, "memory: 128\n");
2571     } else {
2572         unlink $conffile;
2573     }
2574
2575     # also remove unused disk
2576     eval {
2577         my $dl = PVE::Storage::vdisk_list($storecfg, undef, $vmid);
2578
2579         eval {
2580             PVE::Storage::foreach_volid($dl, sub {
2581                 my ($volid, $sid, $volname, $d) = @_;
2582                 PVE::Storage::vdisk_free($storecfg, $volid);
2583             });
2584         };
2585         warn $@ if $@;
2586
2587     };
2588     warn $@ if $@;
2589 }
2590
2591 sub parse_vm_config {
2592     my ($filename, $raw) = @_;
2593
2594     return undef if !defined($raw);
2595
2596     my $res = {
2597         digest => Digest::SHA::sha1_hex($raw),
2598         snapshots => {},
2599         pending => {},
2600     };
2601
2602     $filename =~ m|/qemu-server/(\d+)\.conf$|
2603         || die "got strange filename '$filename'";
2604
2605     my $vmid = $1;
2606
2607     my $conf = $res;
2608     my $descr;
2609     my $section = '';
2610
2611     my @lines = split(/\n/, $raw);
2612     foreach my $line (@lines) {
2613         next if $line =~ m/^\s*$/;
2614
2615         if ($line =~ m/^\[PENDING\]\s*$/i) {
2616             $section = 'pending';
2617             if (defined($descr)) {
2618                 $descr =~ s/\s+$//;
2619                 $conf->{description} = $descr;
2620             }
2621             $descr = undef;
2622             $conf = $res->{$section} = {};
2623             next;
2624
2625         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2626             $section = $1;
2627             if (defined($descr)) {
2628                 $descr =~ s/\s+$//;
2629                 $conf->{description} = $descr;
2630             }
2631             $descr = undef;
2632             $conf = $res->{snapshots}->{$section} = {};
2633             next;
2634         }
2635
2636         if ($line =~ m/^\#(.*)\s*$/) {
2637             $descr = '' if !defined($descr);
2638             $descr .= PVE::Tools::decode_text($1) . "\n";
2639             next;
2640         }
2641
2642         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2643             $descr = '' if !defined($descr);
2644             $descr .= PVE::Tools::decode_text($2);
2645         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2646             $conf->{snapstate} = $1;
2647         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2648             my $key = $1;
2649             my $value = $2;
2650             $conf->{$key} = $value;
2651         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2652             my $value = $1;
2653             if ($section eq 'pending') {
2654                 $conf->{delete} = $value; # we parse this later
2655             } else {
2656                 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
2657             }
2658         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(.+?)\s*$/) {
2659             my $key = $1;
2660             my $value = $2;
2661             eval { $value = check_type($key, $value); };
2662             if ($@) {
2663                 warn "vm $vmid - unable to parse value of '$key' - $@";
2664             } else {
2665                 $key = 'ide2' if $key eq 'cdrom';
2666                 my $fmt = $confdesc->{$key}->{format};
2667                 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2668                     my $v = parse_drive($key, $value);
2669                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2670                         $v->{file} = $volid;
2671                         $value = print_drive($vmid, $v);
2672                     } else {
2673                         warn "vm $vmid - unable to parse value of '$key'\n";
2674                         next;
2675                     }
2676                 }
2677
2678                 $conf->{$key} = $value;
2679             }
2680         }
2681     }
2682
2683     if (defined($descr)) {
2684         $descr =~ s/\s+$//;
2685         $conf->{description} = $descr;
2686     }
2687     delete $res->{snapstate}; # just to be sure
2688
2689     return $res;
2690 }
2691
2692 sub write_vm_config {
2693     my ($filename, $conf) = @_;
2694
2695     delete $conf->{snapstate}; # just to be sure
2696
2697     if ($conf->{cdrom}) {
2698         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2699         $conf->{ide2} = $conf->{cdrom};
2700         delete $conf->{cdrom};
2701     }
2702
2703     # we do not use 'smp' any longer
2704     if ($conf->{sockets}) {
2705         delete $conf->{smp};
2706     } elsif ($conf->{smp}) {
2707         $conf->{sockets} = $conf->{smp};
2708         delete $conf->{cores};
2709         delete $conf->{smp};
2710     }
2711
2712     my $used_volids = {};
2713
2714     my $cleanup_config = sub {
2715         my ($cref, $pending, $snapname) = @_;
2716
2717         foreach my $key (keys %$cref) {
2718             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2719                 $key eq 'snapstate' || $key eq 'pending';
2720             my $value = $cref->{$key};
2721             if ($key eq 'delete') {
2722                 die "propertry 'delete' is only allowed in [PENDING]\n"
2723                     if !$pending;
2724                 # fixme: check syntax?
2725                 next;
2726             }
2727             eval { $value = check_type($key, $value); };
2728             die "unable to parse value of '$key' - $@" if $@;
2729
2730             $cref->{$key} = $value;
2731
2732             if (!$snapname && is_valid_drivename($key)) {
2733                 my $drive = parse_drive($key, $value);
2734                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2735             }
2736         }
2737     };
2738
2739     &$cleanup_config($conf);
2740
2741     &$cleanup_config($conf->{pending}, 1);
2742
2743     foreach my $snapname (keys %{$conf->{snapshots}}) {
2744         die "internal error" if $snapname eq 'pending';
2745         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2746     }
2747
2748     # remove 'unusedX' settings if we re-add a volume
2749     foreach my $key (keys %$conf) {
2750         my $value = $conf->{$key};
2751         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2752             delete $conf->{$key};
2753         }
2754     }
2755
2756     my $generate_raw_config = sub {
2757         my ($conf, $pending) = @_;
2758
2759         my $raw = '';
2760
2761         # add description as comment to top of file
2762         if (defined(my $descr = $conf->{description})) {
2763             if ($descr) {
2764                 foreach my $cl (split(/\n/, $descr)) {
2765                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2766                 }
2767             } else {
2768                 $raw .= "#\n" if $pending;
2769             }
2770         }
2771
2772         foreach my $key (sort keys %$conf) {
2773             next if $key eq 'digest' || $key eq 'description' || $key eq 'pending' || $key eq 'snapshots';
2774             $raw .= "$key: $conf->{$key}\n";
2775         }
2776         return $raw;
2777     };
2778
2779     my $raw = &$generate_raw_config($conf);
2780
2781     if (scalar(keys %{$conf->{pending}})){
2782         $raw .= "\n[PENDING]\n";
2783         $raw .= &$generate_raw_config($conf->{pending}, 1);
2784     }
2785
2786     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2787         $raw .= "\n[$snapname]\n";
2788         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2789     }
2790
2791     return $raw;
2792 }
2793
2794 sub load_defaults {
2795
2796     my $res = {};
2797
2798     # we use static defaults from our JSON schema configuration
2799     foreach my $key (keys %$confdesc) {
2800         if (defined(my $default = $confdesc->{$key}->{default})) {
2801             $res->{$key} = $default;
2802         }
2803     }
2804
2805     return $res;
2806 }
2807
2808 sub config_list {
2809     my $vmlist = PVE::Cluster::get_vmlist();
2810     my $res = {};
2811     return $res if !$vmlist || !$vmlist->{ids};
2812     my $ids = $vmlist->{ids};
2813
2814     foreach my $vmid (keys %$ids) {
2815         my $d = $ids->{$vmid};
2816         next if !$d->{node} || $d->{node} ne $nodename;
2817         next if !$d->{type} || $d->{type} ne 'qemu';
2818         $res->{$vmid}->{exists} = 1;
2819     }
2820     return $res;
2821 }
2822
2823 # test if VM uses local resources (to prevent migration)
2824 sub check_local_resources {
2825     my ($conf, $noerr) = @_;
2826
2827     my $loc_res = 0;
2828
2829     $loc_res = 1 if $conf->{hostusb}; # old syntax
2830     $loc_res = 1 if $conf->{hostpci}; # old syntax
2831
2832     $loc_res = 1 if $conf->{ivshmem};
2833
2834     foreach my $k (keys %$conf) {
2835         next if $k =~ m/^usb/ && ($conf->{$k} eq 'spice');
2836         # sockets are safe: they will recreated be on the target side post-migrate
2837         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2838         $loc_res = 1 if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2839     }
2840
2841     die "VM uses local resources\n" if $loc_res && !$noerr;
2842
2843     return $loc_res;
2844 }
2845
2846 # check if used storages are available on all nodes (use by migrate)
2847 sub check_storage_availability {
2848     my ($storecfg, $conf, $node) = @_;
2849
2850     foreach_drive($conf, sub {
2851         my ($ds, $drive) = @_;
2852
2853         my $volid = $drive->{file};
2854         return if !$volid;
2855
2856         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2857         return if !$sid;
2858
2859         # check if storage is available on both nodes
2860         my $scfg = PVE::Storage::storage_check_node($storecfg, $sid);
2861         PVE::Storage::storage_check_node($storecfg, $sid, $node);
2862    });
2863 }
2864
2865 # list nodes where all VM images are available (used by has_feature API)
2866 sub shared_nodes {
2867     my ($conf, $storecfg) = @_;
2868
2869     my $nodelist = PVE::Cluster::get_nodelist();
2870     my $nodehash = { map { $_ => 1 } @$nodelist };
2871     my $nodename = PVE::INotify::nodename();
2872
2873     foreach_drive($conf, sub {
2874         my ($ds, $drive) = @_;
2875
2876         my $volid = $drive->{file};
2877         return if !$volid;
2878
2879         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2880         if ($storeid) {
2881             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2882             if ($scfg->{disable}) {
2883                 $nodehash = {};
2884             } elsif (my $avail = $scfg->{nodes}) {
2885                 foreach my $node (keys %$nodehash) {
2886                     delete $nodehash->{$node} if !$avail->{$node};
2887                 }
2888             } elsif (!$scfg->{shared}) {
2889                 foreach my $node (keys %$nodehash) {
2890                     delete $nodehash->{$node} if $node ne $nodename
2891                 }
2892             }
2893         }
2894     });
2895
2896     return $nodehash
2897 }
2898
2899 sub check_cmdline {
2900     my ($pidfile, $pid) = @_;
2901
2902     my $fh = IO::File->new("/proc/$pid/cmdline", "r");
2903     if (defined($fh)) {
2904         my $line = <$fh>;
2905         $fh->close;
2906         return undef if !$line;
2907         my @param = split(/\0/, $line);
2908
2909         my $cmd = $param[0];
2910         return if !$cmd || ($cmd !~ m|kvm$| && $cmd !~ m@(?:^|/)qemu-system-[^/]+$@);
2911
2912         for (my $i = 0; $i < scalar (@param); $i++) {
2913             my $p = $param[$i];
2914             next if !$p;
2915             if (($p eq '-pidfile') || ($p eq '--pidfile')) {
2916                 my $p = $param[$i+1];
2917                 return 1 if $p && ($p eq $pidfile);
2918                 return undef;
2919             }
2920         }
2921     }
2922     return undef;
2923 }
2924
2925 sub check_running {
2926     my ($vmid, $nocheck, $node) = @_;
2927
2928     my $filename = PVE::QemuConfig->config_file($vmid, $node);
2929
2930     die "unable to find configuration file for VM $vmid - no such machine\n"
2931         if !$nocheck && ! -f $filename;
2932
2933     my $pidfile = pidfile_name($vmid);
2934
2935     if (my $fd = IO::File->new("<$pidfile")) {
2936         my $st = stat($fd);
2937         my $line = <$fd>;
2938         close($fd);
2939
2940         my $mtime = $st->mtime;
2941         if ($mtime > time()) {
2942             warn "file '$filename' modified in future\n";
2943         }
2944
2945         if ($line =~ m/^(\d+)$/) {
2946             my $pid = $1;
2947             if (check_cmdline($pidfile, $pid)) {
2948                 if (my $pinfo = PVE::ProcFSTools::check_process_running($pid)) {
2949                     return $pid;
2950                 }
2951             }
2952         }
2953     }
2954
2955     return undef;
2956 }
2957
2958 sub vzlist {
2959
2960     my $vzlist = config_list();
2961
2962     my $fd = IO::Dir->new($var_run_tmpdir) || return $vzlist;
2963
2964     while (defined(my $de = $fd->read)) {
2965         next if $de !~ m/^(\d+)\.pid$/;
2966         my $vmid = $1;
2967         next if !defined($vzlist->{$vmid});
2968         if (my $pid = check_running($vmid)) {
2969             $vzlist->{$vmid}->{pid} = $pid;
2970         }
2971     }
2972
2973     return $vzlist;
2974 }
2975
2976 sub disksize {
2977     my ($storecfg, $conf) = @_;
2978
2979     my $bootdisk = $conf->{bootdisk};
2980     return undef if !$bootdisk;
2981     return undef if !is_valid_drivename($bootdisk);
2982
2983     return undef if !$conf->{$bootdisk};
2984
2985     my $drive = parse_drive($bootdisk, $conf->{$bootdisk});
2986     return undef if !defined($drive);
2987
2988     return undef if drive_is_cdrom($drive);
2989
2990     my $volid = $drive->{file};
2991     return undef if !$volid;
2992
2993     return $drive->{size};
2994 }
2995
2996 our $vmstatus_return_properties = {
2997     vmid => get_standard_option('pve-vmid'),
2998     status => {
2999         description => "Qemu process status.",
3000         type => 'string',
3001         enum => ['stopped', 'running'],
3002     },
3003     maxmem => {
3004         description => "Maximum memory in bytes.",
3005         type => 'integer',
3006         optional => 1,
3007         renderer => 'bytes',
3008     },
3009     maxdisk => {
3010         description => "Root disk size in bytes.",
3011         type => 'integer',
3012         optional => 1,
3013         renderer => 'bytes',
3014     },
3015     name => {
3016         description => "VM name.",
3017         type => 'string',
3018         optional => 1,
3019     },
3020     qmpstatus => {
3021         description => "Qemu QMP agent status.",
3022         type => 'string',
3023         optional => 1,
3024     },
3025     pid => {
3026         description => "PID of running qemu process.",
3027         type => 'integer',
3028         optional => 1,
3029     },
3030     uptime => {
3031         description => "Uptime.",
3032         type => 'integer',
3033         optional => 1,
3034         renderer => 'duration',
3035     },
3036     cpus => {
3037         description => "Maximum usable CPUs.",
3038         type => 'number',
3039         optional => 1,
3040     },
3041 };
3042
3043 my $last_proc_pid_stat;
3044
3045 # get VM status information
3046 # This must be fast and should not block ($full == false)
3047 # We only query KVM using QMP if $full == true (this can be slow)
3048 sub vmstatus {
3049     my ($opt_vmid, $full) = @_;
3050
3051     my $res = {};
3052
3053     my $storecfg = PVE::Storage::config();
3054
3055     my $list = vzlist();
3056     my $defaults = load_defaults();
3057
3058     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
3059
3060     my $cpucount = $cpuinfo->{cpus} || 1;
3061
3062     foreach my $vmid (keys %$list) {
3063         next if $opt_vmid && ($vmid ne $opt_vmid);
3064
3065         my $cfspath = PVE::QemuConfig->cfs_config_path($vmid);
3066         my $conf = PVE::Cluster::cfs_read_file($cfspath) || {};
3067
3068         my $d = { vmid => $vmid };
3069         $d->{pid} = $list->{$vmid}->{pid};
3070
3071         # fixme: better status?
3072         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
3073
3074         my $size = disksize($storecfg, $conf);
3075         if (defined($size)) {
3076             $d->{disk} = 0; # no info available
3077             $d->{maxdisk} = $size;
3078         } else {
3079             $d->{disk} = 0;
3080             $d->{maxdisk} = 0;
3081         }
3082
3083         $d->{cpus} = ($conf->{sockets} || $defaults->{sockets})
3084             * ($conf->{cores} || $defaults->{cores});
3085         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
3086         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
3087
3088         $d->{name} = $conf->{name} || "VM $vmid";
3089         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024)
3090             : $defaults->{memory}*(1024*1024);
3091
3092         if ($conf->{balloon}) {
3093             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
3094             $d->{shares} = defined($conf->{shares}) ? $conf->{shares}
3095                 : $defaults->{shares};
3096         }
3097
3098         $d->{uptime} = 0;
3099         $d->{cpu} = 0;
3100         $d->{mem} = 0;
3101
3102         $d->{netout} = 0;
3103         $d->{netin} = 0;
3104
3105         $d->{diskread} = 0;
3106         $d->{diskwrite} = 0;
3107
3108         $d->{template} = PVE::QemuConfig->is_template($conf);
3109
3110         $d->{serial} = 1 if conf_has_serial($conf);
3111
3112         $res->{$vmid} = $d;
3113     }
3114
3115     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
3116     foreach my $dev (keys %$netdev) {
3117         next if $dev !~ m/^tap([1-9]\d*)i/;
3118         my $vmid = $1;
3119         my $d = $res->{$vmid};
3120         next if !$d;
3121
3122         $d->{netout} += $netdev->{$dev}->{receive};
3123         $d->{netin} += $netdev->{$dev}->{transmit};
3124
3125         if ($full) {
3126             $d->{nics}->{$dev}->{netout} = $netdev->{$dev}->{receive};
3127             $d->{nics}->{$dev}->{netin} = $netdev->{$dev}->{transmit};
3128         }
3129
3130     }
3131
3132     my $ctime = gettimeofday;
3133
3134     foreach my $vmid (keys %$list) {
3135
3136         my $d = $res->{$vmid};
3137         my $pid = $d->{pid};
3138         next if !$pid;
3139
3140         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
3141         next if !$pstat; # not running
3142
3143         my $used = $pstat->{utime} + $pstat->{stime};
3144
3145         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
3146
3147         if ($pstat->{vsize}) {
3148             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
3149         }
3150
3151         my $old = $last_proc_pid_stat->{$pid};
3152         if (!$old) {
3153             $last_proc_pid_stat->{$pid} = {
3154                 time => $ctime,
3155                 used => $used,
3156                 cpu => 0,
3157             };
3158             next;
3159         }
3160
3161         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
3162
3163         if ($dtime > 1000) {
3164             my $dutime = $used -  $old->{used};
3165
3166             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
3167             $last_proc_pid_stat->{$pid} = {
3168                 time => $ctime,
3169                 used => $used,
3170                 cpu => $d->{cpu},
3171             };
3172         } else {
3173             $d->{cpu} = $old->{cpu};
3174         }
3175     }
3176
3177     return $res if !$full;
3178
3179     my $qmpclient = PVE::QMPClient->new();
3180
3181     my $ballooncb = sub {
3182         my ($vmid, $resp) = @_;
3183
3184         my $info = $resp->{'return'};
3185         return if !$info->{max_mem};
3186
3187         my $d = $res->{$vmid};
3188
3189         # use memory assigned to VM
3190         $d->{maxmem} = $info->{max_mem};
3191         $d->{balloon} = $info->{actual};
3192
3193         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
3194             $d->{mem} = $info->{total_mem} - $info->{free_mem};
3195             $d->{freemem} = $info->{free_mem};
3196         }
3197
3198         $d->{ballooninfo} = $info;
3199     };
3200
3201     my $blockstatscb = sub {
3202         my ($vmid, $resp) = @_;
3203         my $data = $resp->{'return'} || [];
3204         my $totalrdbytes = 0;
3205         my $totalwrbytes = 0;
3206
3207         for my $blockstat (@$data) {
3208             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
3209             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
3210
3211             $blockstat->{device} =~ s/drive-//;
3212             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
3213         }
3214         $res->{$vmid}->{diskread} = $totalrdbytes;
3215         $res->{$vmid}->{diskwrite} = $totalwrbytes;
3216     };
3217
3218     my $statuscb = sub {
3219         my ($vmid, $resp) = @_;
3220
3221         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
3222         # this fails if ballon driver is not loaded, so this must be
3223         # the last commnand (following command are aborted if this fails).
3224         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
3225
3226         my $status = 'unknown';
3227         if (!defined($status = $resp->{'return'}->{status})) {
3228             warn "unable to get VM status\n";
3229             return;
3230         }
3231
3232         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
3233     };
3234
3235     foreach my $vmid (keys %$list) {
3236         next if $opt_vmid && ($vmid ne $opt_vmid);
3237         next if !$res->{$vmid}->{pid}; # not running
3238         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
3239     }
3240
3241     $qmpclient->queue_execute(undef, 2);
3242
3243     foreach my $vmid (keys %$list) {
3244         next if $opt_vmid && ($vmid ne $opt_vmid);
3245         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
3246     }
3247
3248     return $res;
3249 }
3250
3251 sub foreach_drive {
3252     my ($conf, $func, @param) = @_;
3253
3254     foreach my $ds (valid_drive_names()) {
3255         next if !defined($conf->{$ds});
3256
3257         my $drive = parse_drive($ds, $conf->{$ds});
3258         next if !$drive;
3259
3260         &$func($ds, $drive, @param);
3261     }
3262 }
3263
3264 sub foreach_volid {
3265     my ($conf, $func, @param) = @_;
3266
3267     my $volhash = {};
3268
3269     my $test_volid = sub {
3270         my ($volid, $is_cdrom, $replicate, $shared, $snapname) = @_;
3271
3272         return if !$volid;
3273
3274         $volhash->{$volid}->{cdrom} //= 1;
3275         $volhash->{$volid}->{cdrom} = 0 if !$is_cdrom;
3276
3277         $volhash->{$volid}->{replicate} //= 0;
3278         $volhash->{$volid}->{replicate} = 1 if $replicate;
3279
3280         $volhash->{$volid}->{shared} //= 0;
3281         $volhash->{$volid}->{shared} = 1 if $shared;
3282
3283         $volhash->{$volid}->{referenced_in_config} //= 0;
3284         $volhash->{$volid}->{referenced_in_config} = 1 if !defined($snapname);
3285
3286         $volhash->{$volid}->{referenced_in_snapshot}->{$snapname} = 1
3287             if defined($snapname);
3288     };
3289
3290     foreach_drive($conf, sub {
3291         my ($ds, $drive) = @_;
3292         $test_volid->($drive->{file}, drive_is_cdrom($drive), $drive->{replicate} // 1, $drive->{shared}, undef);
3293     });
3294
3295     foreach my $snapname (keys %{$conf->{snapshots}}) {
3296         my $snap = $conf->{snapshots}->{$snapname};
3297         $test_volid->($snap->{vmstate}, 0, 1, $snapname);
3298         foreach_drive($snap, sub {
3299             my ($ds, $drive) = @_;
3300             $test_volid->($drive->{file}, drive_is_cdrom($drive), $drive->{replicate} // 1, $drive->{shared}, $snapname);
3301         });
3302     }
3303
3304     foreach my $volid (keys %$volhash) {
3305         &$func($volid, $volhash->{$volid}, @param);
3306     }
3307 }
3308
3309 sub conf_has_serial {
3310     my ($conf) = @_;
3311
3312     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3313         if ($conf->{"serial$i"}) {
3314             return 1;
3315         }
3316     }
3317
3318     return 0;
3319 }
3320
3321 sub vga_conf_has_spice {
3322     my ($vga) = @_;
3323
3324     my $vgaconf = parse_vga($vga);
3325     my $vgatype = $vgaconf->{type};
3326     return 0 if !$vgatype || $vgatype !~ m/^qxl([234])?$/;
3327
3328     return $1 || 1;
3329 }
3330
3331 my $host_arch; # FIXME: fix PVE::Tools::get_host_arch
3332 sub get_host_arch() {
3333     $host_arch = (POSIX::uname())[4] if !$host_arch;
3334     return $host_arch;
3335 }
3336
3337 sub is_native($) {
3338     my ($arch) = @_;
3339     return get_host_arch() eq $arch;
3340 }
3341
3342 my $default_machines = {
3343     x86_64 => 'pc',
3344     aarch64 => 'virt',
3345 };
3346
3347 sub get_basic_machine_info {
3348     my ($conf, $forcemachine) = @_;
3349
3350     my $arch = $conf->{arch} // get_host_arch();
3351     my $machine = $forcemachine || $conf->{machine} || $default_machines->{$arch};
3352     return ($arch, $machine);
3353 }
3354
3355 sub get_ovmf_files($) {
3356     my ($arch) = @_;
3357
3358     my $ovmf = $OVMF->{$arch}
3359         or die "no OVMF images known for architecture '$arch'\n";
3360
3361     return @$ovmf;
3362 }
3363
3364 my $Arch2Qemu = {
3365     aarch64 => '/usr/bin/qemu-system-aarch64',
3366     x86_64 => '/usr/bin/qemu-system-x86_64',
3367 };
3368 sub get_command_for_arch($) {
3369     my ($arch) = @_;
3370     return '/usr/bin/kvm' if is_native($arch);
3371
3372     my $cmd = $Arch2Qemu->{$arch}
3373         or die "don't know how to emulate architecture '$arch'\n";
3374     return $cmd;
3375 }
3376
3377 sub get_cpu_options {
3378     my ($conf, $arch, $kvm, $machine_type, $kvm_off, $kvmver, $winversion, $gpu_passthrough) = @_;
3379
3380     my $cpuFlags = [];
3381     my $ostype = $conf->{ostype};
3382
3383     my $cpu = $kvm ? "kvm64" : "qemu64";
3384     if ($arch eq 'aarch64') {
3385         $cpu = 'cortex-a57';
3386     }
3387     my $hv_vendor_id;
3388     if (my $cputype = $conf->{cpu}) {
3389         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
3390             or die "Cannot parse cpu description: $cputype\n";
3391         $cpu = $cpuconf->{cputype};
3392         $kvm_off = 1 if $cpuconf->{hidden};
3393         $hv_vendor_id = $cpuconf->{'hv-vendor-id'};
3394
3395         if (defined(my $flags = $cpuconf->{flags})) {
3396             push @$cpuFlags, split(";", $flags);
3397         }
3398     }
3399
3400     push @$cpuFlags , '+lahf_lm' if $cpu eq 'kvm64' && $arch eq 'x86_64';
3401
3402     push @$cpuFlags , '-x2apic'
3403         if $conf->{ostype} && $conf->{ostype} eq 'solaris';
3404
3405     push @$cpuFlags, '+sep' if $cpu eq 'kvm64' || $cpu eq 'kvm32';
3406
3407     push @$cpuFlags, '-rdtscp' if $cpu =~ m/^Opteron/;
3408
3409     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3) && $arch eq 'x86_64') {
3410
3411         push @$cpuFlags , '+kvm_pv_unhalt' if $kvm;
3412         push @$cpuFlags , '+kvm_pv_eoi' if $kvm;
3413     }
3414
3415     add_hyperv_enlightenments($cpuFlags, $winversion, $machine_type, $kvmver, $conf->{bios}, $gpu_passthrough, $hv_vendor_id) if $kvm;
3416
3417     push @$cpuFlags, 'enforce' if $cpu ne 'host' && $kvm && $arch eq 'x86_64';
3418
3419     push @$cpuFlags, 'kvm=off' if $kvm_off;
3420
3421     if (my $cpu_vendor = $cpu_vendor_list->{$cpu}) {
3422         push @$cpuFlags, "vendor=${cpu_vendor}"
3423             if $cpu_vendor ne 'default';
3424     } elsif ($arch ne 'aarch64') {
3425         die "internal error"; # should not happen
3426     }
3427
3428     $cpu .= "," . join(',', @$cpuFlags) if scalar(@$cpuFlags);
3429
3430     return ('-cpu', $cpu);
3431 }
3432
3433 sub config_to_command {
3434     my ($storecfg, $vmid, $conf, $defaults, $forcemachine) = @_;
3435
3436     my $cmd = [];
3437     my $globalFlags = [];
3438     my $machineFlags = [];
3439     my $rtcFlags = [];
3440     my $devices = [];
3441     my $pciaddr = '';
3442     my $bridges = {};
3443     my $kvmver = kvm_user_version();
3444     my $vernum = 0; # unknown
3445     my $ostype = $conf->{ostype};
3446     my $winversion = windows_version($ostype);
3447     my $kvm = $conf->{kvm};
3448
3449     my ($arch, $machine_type) = get_basic_machine_info($conf, $forcemachine);
3450     $kvm //= 1 if is_native($arch);
3451
3452     if ($kvm) {
3453         die "KVM virtualisation configured, but not available. Either disable in VM configuration or enable in BIOS.\n"
3454             if !defined kvm_version();
3455     }
3456
3457     if ($kvmver =~ m/^(\d+)\.(\d+)$/) {
3458         $vernum = $1*1000000+$2*1000;
3459     } elsif ($kvmver =~ m/^(\d+)\.(\d+)\.(\d+)$/) {
3460         $vernum = $1*1000000+$2*1000+$3;
3461     }
3462
3463     die "detected old qemu-kvm binary ($kvmver)\n" if $vernum < 15000;
3464
3465     my $have_ovz = -f '/proc/vz/vestat';
3466
3467     my $q35 = machine_type_is_q35($conf);
3468     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
3469     my $use_old_bios_files = undef;
3470     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3471
3472     my $cpuunits = defined($conf->{cpuunits}) ?
3473             $conf->{cpuunits} : $defaults->{cpuunits};
3474
3475     push @$cmd, get_command_for_arch($arch);
3476
3477     push @$cmd, '-id', $vmid;
3478
3479     my $vmname = $conf->{name} || "vm$vmid";
3480
3481     push @$cmd, '-name', $vmname;
3482
3483     my $use_virtio = 0;
3484
3485     my $qmpsocket = qmp_socket($vmid);
3486     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server,nowait";
3487     push @$cmd, '-mon', "chardev=qmp,mode=control";
3488
3489     if (qemu_machine_feature_enabled($machine_type, $kvmver, 2, 12)) {
3490         push @$cmd, '-chardev', "socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5";
3491         push @$cmd, '-mon', "chardev=qmp-event,mode=control";
3492     }
3493
3494     push @$cmd, '-pidfile' , pidfile_name($vmid);
3495
3496     push @$cmd, '-daemonize';
3497
3498     if ($conf->{smbios1}) {
3499         push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
3500     }
3501
3502     if ($conf->{vmgenid}) {
3503         push @$devices, '-device', 'vmgenid,guid='.$conf->{vmgenid};
3504     }
3505
3506     my ($ovmf_code, $ovmf_vars) = get_ovmf_files($arch);
3507     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3508         die "uefi base image not found\n" if ! -f $ovmf_code;
3509
3510         my $path;
3511         my $format;
3512         if (my $efidisk = $conf->{efidisk0}) {
3513             my $d = PVE::JSONSchema::parse_property_string($efidisk_fmt, $efidisk);
3514             my ($storeid, $volname) = PVE::Storage::parse_volume_id($d->{file}, 1);
3515             $format = $d->{format};
3516             if ($storeid) {
3517                 $path = PVE::Storage::path($storecfg, $d->{file});
3518                 if (!defined($format)) {
3519                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
3520                     $format = qemu_img_format($scfg, $volname);
3521                 }
3522             } else {
3523                 $path = $d->{file};
3524                 die "efidisk format must be specified\n"
3525                     if !defined($format);
3526             }
3527         } else {
3528             warn "no efidisk configured! Using temporary efivars disk.\n";
3529             $path = "/tmp/$vmid-ovmf.fd";
3530             PVE::Tools::file_copy($ovmf_vars, $path, -s $ovmf_vars);
3531             $format = 'raw';
3532         }
3533
3534         push @$cmd, '-drive', "if=pflash,unit=0,format=raw,readonly,file=$ovmf_code";
3535         push @$cmd, '-drive', "if=pflash,unit=1,format=$format,id=drive-efidisk0,file=$path";
3536     }
3537
3538
3539     # add usb controllers
3540     my @usbcontrollers = PVE::QemuServer::USB::get_usb_controllers($conf, $bridges, $arch, $machine_type, $usbdesc->{format}, $MAX_USB_DEVICES);
3541     push @$devices, @usbcontrollers if @usbcontrollers;
3542     my $vga = parse_vga($conf->{vga});
3543
3544     my $qxlnum = vga_conf_has_spice($conf->{vga});
3545     $vga->{type} = 'qxl' if $qxlnum;
3546
3547     if (!$vga->{type}) {
3548         if ($arch eq 'aarch64') {
3549             $vga->{type} = 'virtio';
3550         } elsif (qemu_machine_feature_enabled($machine_type, $kvmver, 2, 9)) {
3551             $vga->{type} = (!$winversion || $winversion >= 6) ? 'std' : 'cirrus';
3552         } else {
3553             $vga->{type} = ($winversion >= 6) ? 'std' : 'cirrus';
3554         }
3555     }
3556
3557     # enable absolute mouse coordinates (needed by vnc)
3558     my $tablet;
3559     if (defined($conf->{tablet})) {
3560         $tablet = $conf->{tablet};
3561     } else {
3562         $tablet = $defaults->{tablet};
3563         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
3564         $tablet = 0 if $vga->{type} =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
3565     }
3566
3567     if ($tablet) {
3568         push @$devices, '-device', print_tabletdevice_full($conf, $arch) if $tablet;
3569         my $kbd = print_keyboarddevice_full($conf, $arch);
3570         push @$devices, '-device', $kbd if defined($kbd);
3571     }
3572
3573     my $kvm_off = 0;
3574     my $gpu_passthrough;
3575
3576     # host pci devices
3577     for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
3578         my $d = parse_hostpci($conf->{"hostpci$i"});
3579         next if !$d;
3580
3581         my $pcie = $d->{pcie};
3582         if($pcie){
3583             die "q35 machine model is not enabled" if !$q35;
3584             # win7 wants to have the pcie devices directly on the pcie bus
3585             # instead of in the root port
3586             if ($winversion == 7) {
3587                 $pciaddr = print_pcie_addr("hostpci${i}bus0");
3588             } else {
3589                 $pciaddr = print_pcie_addr("hostpci$i");
3590             }
3591         }else{
3592             $pciaddr = print_pci_addr("hostpci$i", $bridges, $arch, $machine_type);
3593         }
3594
3595         my $rombar = defined($d->{rombar}) && !$d->{rombar} ? ',rombar=0' : '';
3596         my $romfile = $d->{romfile};
3597
3598         my $xvga = '';
3599         if ($d->{'x-vga'}) {
3600             $xvga = ',x-vga=on';
3601             $kvm_off = 1;
3602             $vga->{type} = 'none' if !defined($conf->{vga});
3603             $gpu_passthrough = 1;
3604
3605             if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3606                 $xvga = "";
3607             }
3608         }
3609         my $pcidevices = $d->{pciid};
3610         my $multifunction = 1 if @$pcidevices > 1;
3611         my $sysfspath;
3612         if ($d->{mdev} && scalar(@$pcidevices) == 1) {
3613             my $id = $pcidevices->[0]->{id};
3614             my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $i);
3615             $sysfspath = "/sys/bus/pci/devices/0000:$id/$uuid";
3616         } elsif ($d->{mdev}) {
3617             warn "ignoring mediated device with multifunction device\n";
3618         }
3619
3620         my $j=0;
3621         foreach my $pcidevice (@$pcidevices) {
3622
3623             my $id = "hostpci$i";
3624             $id .= ".$j" if $multifunction;
3625             my $addr = $pciaddr;
3626             $addr .= ".$j" if $multifunction;
3627             my $devicestr = "vfio-pci";
3628             if ($sysfspath) {
3629                 $devicestr .= ",sysfsdev=$sysfspath";
3630             } else {
3631                 $devicestr .= ",host=$pcidevice->{id}";
3632             }
3633             $devicestr .= ",id=$id$addr";
3634
3635             if($j == 0){
3636                 $devicestr .= "$rombar$xvga";
3637                 $devicestr .= ",multifunction=on" if $multifunction;
3638                 $devicestr .= ",romfile=/usr/share/kvm/$romfile" if $romfile;
3639             }
3640
3641             push @$devices, '-device', $devicestr;
3642             $j++;
3643         }
3644     }
3645
3646     # usb devices
3647     my @usbdevices = PVE::QemuServer::USB::get_usb_devices($conf, $usbdesc->{format}, $MAX_USB_DEVICES);
3648     push @$devices, @usbdevices if @usbdevices;
3649     # serial devices
3650     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3651         if (my $path = $conf->{"serial$i"}) {
3652             if ($path eq 'socket') {
3653                 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
3654                 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server,nowait";
3655                 # On aarch64, serial0 is the UART device. Qemu only allows
3656                 # connecting UART devices via the '-serial' command line, as
3657                 # the device has a fixed slot on the hardware...
3658                 if ($arch eq 'aarch64' && $i == 0) {
3659                     push @$devices, '-serial', "chardev:serial$i";
3660                 } else {
3661                     push @$devices, '-device', "isa-serial,chardev=serial$i";
3662                 }
3663             } else {
3664                 die "no such serial device\n" if ! -c $path;
3665                 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
3666                 push @$devices, '-device', "isa-serial,chardev=serial$i";
3667             }
3668         }
3669     }
3670
3671     # parallel devices
3672     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
3673         if (my $path = $conf->{"parallel$i"}) {
3674             die "no such parallel device\n" if ! -c $path;
3675             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
3676             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
3677             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
3678         }
3679     }
3680
3681
3682     my $sockets = 1;
3683     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3684     $sockets = $conf->{sockets} if  $conf->{sockets};
3685
3686     my $cores = $conf->{cores} || 1;
3687
3688     my $maxcpus = $sockets * $cores;
3689
3690     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
3691
3692     my $allowed_vcpus = $cpuinfo->{cpus};
3693
3694     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n"
3695         if ($allowed_vcpus < $maxcpus);
3696
3697     if($hotplug_features->{cpu} && qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 7)) {
3698
3699         push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3700         for (my $i = 2; $i <= $vcpus; $i++)  {
3701             my $cpustr = print_cpu_device($conf,$i);
3702             push @$cmd, '-device', $cpustr;
3703         }
3704
3705     } else {
3706
3707         push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3708     }
3709     push @$cmd, '-nodefaults';
3710
3711     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
3712
3713     my $bootindex_hash = {};
3714     my $i = 1;
3715     foreach my $o (split(//, $bootorder)) {
3716         $bootindex_hash->{$o} = $i*100;
3717         $i++;
3718     }
3719
3720     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
3721
3722     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
3723
3724     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
3725
3726     if ($vga->{type} && $vga->{type} !~ m/^serial\d+$/ && $vga->{type} ne 'none'){
3727         push @$devices, '-device', print_vga_device($conf, $vga, $arch, $machine_type, undef, $qxlnum, $bridges);
3728         my $socket = vnc_socket($vmid);
3729         push @$cmd,  '-vnc', "unix:$socket,x509,password";
3730     } else {
3731         push @$cmd, '-vga', 'none' if $vga->{type} eq 'none';
3732         push @$cmd, '-nographic';
3733     }
3734
3735     # time drift fix
3736     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
3737
3738     my $useLocaltime = $conf->{localtime};
3739
3740     if ($winversion >= 5) { # windows
3741         $useLocaltime = 1 if !defined($conf->{localtime});
3742
3743         # use time drift fix when acpi is enabled
3744         if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
3745             $tdf = 1 if !defined($conf->{tdf});
3746         }
3747     }
3748
3749     if ($winversion >= 6) {
3750         push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3751         push @$cmd, '-no-hpet';
3752     }
3753
3754     push @$rtcFlags, 'driftfix=slew' if $tdf;
3755
3756     if (!$kvm) {
3757         push @$machineFlags, 'accel=tcg';
3758     }
3759
3760     if ($machine_type) {
3761         push @$machineFlags, "type=${machine_type}";
3762     }
3763
3764     if ($conf->{startdate}) {
3765         push @$rtcFlags, "base=$conf->{startdate}";
3766     } elsif ($useLocaltime) {
3767         push @$rtcFlags, 'base=localtime';
3768     }
3769
3770     push @$cmd, get_cpu_options($conf, $arch, $kvm, $machine_type, $kvm_off, $kvmver, $winversion, $gpu_passthrough);
3771
3772     PVE::QemuServer::Memory::config($conf, $vmid, $sockets, $cores, $defaults, $hotplug_features, $cmd);
3773
3774     push @$cmd, '-S' if $conf->{freeze};
3775
3776     push @$cmd, '-k', $conf->{keyboard} if defined($conf->{keyboard});
3777
3778     # enable sound
3779     #my $soundhw = $conf->{soundhw} || $defaults->{soundhw};
3780     #push @$cmd, '-soundhw', 'es1370';
3781     #push @$cmd, '-soundhw', $soundhw if $soundhw;
3782
3783     if (parse_guest_agent($conf)->{enabled}) {
3784         my $qgasocket = qmp_socket($vmid, 1);
3785         my $pciaddr = print_pci_addr("qga0", $bridges, $arch, $machine_type);
3786         push @$devices, '-chardev', "socket,path=$qgasocket,server,nowait,id=qga0";
3787         push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3788         push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3789     }
3790
3791     my $spice_port;
3792
3793     if ($qxlnum) {
3794         if ($qxlnum > 1) {
3795             if ($winversion){
3796                 for(my $i = 1; $i < $qxlnum; $i++){
3797                     push @$devices, '-device', print_vga_device($conf, $vga, $arch, $machine_type, $i, $qxlnum, $bridges);
3798                 }
3799             } else {
3800                 # assume other OS works like Linux
3801                 my ($ram, $vram) = ("134217728", "67108864");
3802                 if ($vga->{memory}) {
3803                     $ram = PVE::Tools::convert_size($qxlnum*4*$vga->{memory}, 'mb' => 'b');
3804                     $vram = PVE::Tools::convert_size($qxlnum*2*$vga->{memory}, 'mb' => 'b');
3805                 }
3806                 push @$cmd, '-global', "qxl-vga.ram_size=$ram";
3807                 push @$cmd, '-global', "qxl-vga.vram_size=$vram";
3808             }
3809         }
3810
3811         my $pciaddr = print_pci_addr("spice", $bridges, $arch, $machine_type);
3812
3813         my $nodename = PVE::INotify::nodename();
3814         my $pfamily = PVE::Tools::get_host_address_family($nodename);
3815         my @nodeaddrs = PVE::Tools::getaddrinfo_all('localhost', family => $pfamily);
3816         die "failed to get an ip address of type $pfamily for 'localhost'\n" if !@nodeaddrs;
3817         my $localhost = PVE::Network::addr_to_ip($nodeaddrs[0]->{addr});
3818         $spice_port = PVE::Tools::next_spice_port($pfamily, $localhost);
3819
3820         push @$devices, '-spice', "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";
3821
3822         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3823         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3824         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3825     }
3826
3827     # enable balloon by default, unless explicitly disabled
3828     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3829         $pciaddr = print_pci_addr("balloon0", $bridges, $arch, $machine_type);
3830         push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
3831     }
3832
3833     if ($conf->{watchdog}) {
3834         my $wdopts = parse_watchdog($conf->{watchdog});
3835         $pciaddr = print_pci_addr("watchdog", $bridges, $arch, $machine_type);
3836         my $watchdog = $wdopts->{model} || 'i6300esb';
3837         push @$devices, '-device', "$watchdog$pciaddr";
3838         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3839     }
3840
3841     my $vollist = [];
3842     my $scsicontroller = {};
3843     my $ahcicontroller = {};
3844     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3845
3846     # Add iscsi initiator name if available
3847     if (my $initiator = get_initiator_name()) {
3848         push @$devices, '-iscsi', "initiator-name=$initiator";
3849     }
3850
3851     foreach_drive($conf, sub {
3852         my ($ds, $drive) = @_;
3853
3854         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3855             push @$vollist, $drive->{file};
3856         }
3857
3858         # ignore efidisk here, already added in bios/fw handling code above
3859         return if $drive->{interface} eq 'efidisk';
3860
3861         $use_virtio = 1 if $ds =~ m/^virtio/;
3862
3863         if (drive_is_cdrom ($drive)) {
3864             if ($bootindex_hash->{d}) {
3865                 $drive->{bootindex} = $bootindex_hash->{d};
3866                 $bootindex_hash->{d} += 1;
3867             }
3868         } else {
3869             if ($bootindex_hash->{c}) {
3870                 $drive->{bootindex} = $bootindex_hash->{c} if $conf->{bootdisk} && ($conf->{bootdisk} eq $ds);
3871                 $bootindex_hash->{c} += 1;
3872             }
3873         }
3874
3875         if($drive->{interface} eq 'virtio'){
3876            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3877         }
3878
3879         if ($drive->{interface} eq 'scsi') {
3880
3881             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
3882
3883             $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges, $arch, $machine_type);
3884             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
3885
3886             my $iothread = '';
3887             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
3888                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3889                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3890             } elsif ($drive->{iothread}) {
3891                 warn "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n";
3892             }
3893
3894             my $queues = '';
3895             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
3896                 $queues = ",num_queues=$drive->{queues}";
3897             }
3898
3899             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues" if !$scsicontroller->{$controller};
3900             $scsicontroller->{$controller}=1;
3901         }
3902
3903         if ($drive->{interface} eq 'sata') {
3904            my $controller = int($drive->{index} / $MAX_SATA_DISKS);
3905            $pciaddr = print_pci_addr("ahci$controller", $bridges, $arch, $machine_type);
3906            push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr" if !$ahcicontroller->{$controller};
3907            $ahcicontroller->{$controller}=1;
3908         }
3909
3910         my $drive_cmd = print_drive_full($storecfg, $vmid, $drive);
3911         push @$devices, '-drive',$drive_cmd;
3912         push @$devices, '-device', print_drivedevice_full($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type);
3913     });
3914
3915     for (my $i = 0; $i < $MAX_NETS; $i++) {
3916          next if !$conf->{"net$i"};
3917          my $d = parse_net($conf->{"net$i"});
3918          next if !$d;
3919
3920          $use_virtio = 1 if $d->{model} eq 'virtio';
3921
3922          if ($bootindex_hash->{n}) {
3923             $d->{bootindex} = $bootindex_hash->{n};
3924             $bootindex_hash->{n} += 1;
3925          }
3926
3927          my $netdevfull = print_netdev_full($vmid, $conf, $arch, $d, "net$i");
3928          push @$devices, '-netdev', $netdevfull;
3929
3930          my $netdevicefull = print_netdevice_full($vmid, $conf, $d, "net$i", $bridges, $use_old_bios_files, $arch, $machine_type);
3931          push @$devices, '-device', $netdevicefull;
3932     }
3933
3934     if ($conf->{ivshmem}) {
3935         my $ivshmem = PVE::JSONSchema::parse_property_string($ivshmem_fmt, $conf->{ivshmem});
3936
3937         my $bus;
3938         if ($q35) {
3939             $bus = print_pcie_addr("ivshmem");
3940         } else {
3941             $bus = print_pci_addr("ivshmem", $bridges, $arch, $machine_type);
3942         }
3943
3944         my $ivshmem_name = $ivshmem->{name} // $vmid;
3945         my $path = '/dev/shm/pve-shm-' . $ivshmem_name;
3946
3947         push @$devices, '-device', "ivshmem-plain,memdev=ivshmem$bus,";
3948         push @$devices, '-object', "memory-backend-file,id=ivshmem,share=on,mem-path=$path,size=$ivshmem->{size}M";
3949     }
3950
3951     if (!$q35) {
3952         # add pci bridges
3953         if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3954            $bridges->{1} = 1;
3955            $bridges->{2} = 1;
3956         }
3957
3958         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
3959
3960         while (my ($k, $v) = each %$bridges) {
3961             $pciaddr = print_pci_addr("pci.$k", undef, $arch, $machine_type);
3962             unshift @$devices, '-device', "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr" if $k > 0;
3963         }
3964     }
3965
3966     push @$cmd, @$devices;
3967     push @$cmd, '-rtc', join(',', @$rtcFlags)
3968         if scalar(@$rtcFlags);
3969     push @$cmd, '-machine', join(',', @$machineFlags)
3970         if scalar(@$machineFlags);
3971     push @$cmd, '-global', join(',', @$globalFlags)
3972         if scalar(@$globalFlags);
3973
3974     # add custom args
3975     if ($conf->{args}) {
3976         my $aa = PVE::Tools::split_args($conf->{args});
3977         push @$cmd, @$aa;
3978     }
3979
3980     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
3981 }
3982
3983 sub vnc_socket {
3984     my ($vmid) = @_;
3985     return "${var_run_tmpdir}/$vmid.vnc";
3986 }
3987
3988 sub spice_port {
3989     my ($vmid) = @_;
3990
3991     my $res = vm_mon_cmd($vmid, 'query-spice');
3992
3993     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
3994 }
3995
3996 sub qmp_socket {
3997     my ($vmid, $qga, $name) = @_;
3998     my $sockettype = $qga ? 'qga' : 'qmp';
3999     my $ext = $name ? '-'.$name : '';
4000     return "${var_run_tmpdir}/$vmid$ext.$sockettype";
4001 }
4002
4003 sub pidfile_name {
4004     my ($vmid) = @_;
4005     return "${var_run_tmpdir}/$vmid.pid";
4006 }
4007
4008 sub vm_devices_list {
4009     my ($vmid) = @_;
4010
4011     my $res = vm_mon_cmd($vmid, 'query-pci');
4012     my $devices_to_check = [];
4013     my $devices = {};
4014     foreach my $pcibus (@$res) {
4015         push @$devices_to_check, @{$pcibus->{devices}},
4016     }
4017
4018     while (@$devices_to_check) {
4019         my $to_check = [];
4020         for my $d (@$devices_to_check) {
4021             $devices->{$d->{'qdev_id'}} = 1 if $d->{'qdev_id'};
4022             next if !$d->{'pci_bridge'};
4023
4024             $devices->{$d->{'qdev_id'}} += scalar(@{$d->{'pci_bridge'}->{devices}});
4025             push @$to_check, @{$d->{'pci_bridge'}->{devices}};
4026         }
4027         $devices_to_check = $to_check;
4028     }
4029
4030     my $resblock = vm_mon_cmd($vmid, 'query-block');
4031     foreach my $block (@$resblock) {
4032         if($block->{device} =~ m/^drive-(\S+)/){
4033                 $devices->{$1} = 1;
4034         }
4035     }
4036
4037     my $resmice = vm_mon_cmd($vmid, 'query-mice');
4038     foreach my $mice (@$resmice) {
4039         if ($mice->{name} eq 'QEMU HID Tablet') {
4040             $devices->{tablet} = 1;
4041             last;
4042         }
4043     }
4044
4045     # for usb devices there is no query-usb
4046     # but we can iterate over the entries in
4047     # qom-list path=/machine/peripheral
4048     my $resperipheral = vm_mon_cmd($vmid, 'qom-list', path => '/machine/peripheral');
4049     foreach my $per (@$resperipheral) {
4050         if ($per->{name} =~ m/^usb\d+$/) {
4051             $devices->{$per->{name}} = 1;
4052         }
4053     }
4054
4055     return $devices;
4056 }
4057
4058 sub vm_deviceplug {
4059     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4060
4061     my $q35 = machine_type_is_q35($conf);
4062
4063     my $devices_list = vm_devices_list($vmid);
4064     return 1 if defined($devices_list->{$deviceid});
4065
4066     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid, $arch, $machine_type); # add PCI bridge if we need it for the device
4067
4068     if ($deviceid eq 'tablet') {
4069
4070         qemu_deviceadd($vmid, print_tabletdevice_full($conf, $arch));
4071
4072     } elsif ($deviceid eq 'keyboard') {
4073
4074         qemu_deviceadd($vmid, print_keyboarddevice_full($conf, $arch));
4075
4076     } elsif ($deviceid =~ m/^usb(\d+)$/) {
4077
4078         die "usb hotplug currently not reliable\n";
4079         # since we can't reliably hot unplug all added usb devices
4080         # and usb passthrough disables live migration
4081         # we disable usb hotplugging for now
4082         qemu_deviceadd($vmid, PVE::QemuServer::USB::print_usbdevice_full($conf, $deviceid, $device));
4083
4084     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4085
4086         qemu_iothread_add($vmid, $deviceid, $device);
4087
4088         qemu_driveadd($storecfg, $vmid, $device);
4089         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, $arch, $machine_type);
4090
4091         qemu_deviceadd($vmid, $devicefull);
4092         eval { qemu_deviceaddverify($vmid, $deviceid); };
4093         if (my $err = $@) {
4094             eval { qemu_drivedel($vmid, $deviceid); };
4095             warn $@ if $@;
4096             die $err;
4097         }
4098
4099     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4100
4101
4102         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
4103         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
4104         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
4105
4106         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
4107
4108         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
4109             qemu_iothread_add($vmid, $deviceid, $device);
4110             $devicefull .= ",iothread=iothread-$deviceid";
4111         }
4112
4113         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
4114             $devicefull .= ",num_queues=$device->{queues}";
4115         }
4116
4117         qemu_deviceadd($vmid, $devicefull);
4118         qemu_deviceaddverify($vmid, $deviceid);
4119
4120     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4121
4122         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device, $arch, $machine_type);
4123         qemu_driveadd($storecfg, $vmid, $device);
4124
4125         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, $arch, $machine_type);
4126         eval { qemu_deviceadd($vmid, $devicefull); };
4127         if (my $err = $@) {
4128             eval { qemu_drivedel($vmid, $deviceid); };
4129             warn $@ if $@;
4130             die $err;
4131         }
4132
4133     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4134
4135         return undef if !qemu_netdevadd($vmid, $conf, $arch, $device, $deviceid);
4136
4137         my $machine_type = PVE::QemuServer::qemu_machine_pxe($vmid, $conf);
4138         my $use_old_bios_files = undef;
4139         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
4140
4141         my $netdevicefull = print_netdevice_full($vmid, $conf, $device, $deviceid, undef, $use_old_bios_files, $arch, $machine_type);
4142         qemu_deviceadd($vmid, $netdevicefull);
4143         eval { qemu_deviceaddverify($vmid, $deviceid); };
4144         if (my $err = $@) {
4145             eval { qemu_netdevdel($vmid, $deviceid); };
4146             warn $@ if $@;
4147             die $err;
4148         }
4149
4150     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
4151
4152         my $bridgeid = $2;
4153         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
4154         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
4155
4156         qemu_deviceadd($vmid, $devicefull);
4157         qemu_deviceaddverify($vmid, $deviceid);
4158
4159     } else {
4160         die "can't hotplug device '$deviceid'\n";
4161     }
4162
4163     return 1;
4164 }
4165
4166 # fixme: this should raise exceptions on error!
4167 sub vm_deviceunplug {
4168     my ($vmid, $conf, $deviceid) = @_;
4169
4170     my $devices_list = vm_devices_list($vmid);
4171     return 1 if !defined($devices_list->{$deviceid});
4172
4173     die "can't unplug bootdisk" if $conf->{bootdisk} && $conf->{bootdisk} eq $deviceid;
4174
4175     if ($deviceid eq 'tablet' || $deviceid eq 'keyboard') {
4176
4177         qemu_devicedel($vmid, $deviceid);
4178
4179     } elsif ($deviceid =~ m/^usb\d+$/) {
4180
4181         die "usb hotplug currently not reliable\n";
4182         # when unplugging usb devices this way,
4183         # there may be remaining usb controllers/hubs
4184         # so we disable it for now
4185         qemu_devicedel($vmid, $deviceid);
4186         qemu_devicedelverify($vmid, $deviceid);
4187
4188     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4189
4190         qemu_devicedel($vmid, $deviceid);
4191         qemu_devicedelverify($vmid, $deviceid);
4192         qemu_drivedel($vmid, $deviceid);
4193         qemu_iothread_del($conf, $vmid, $deviceid);
4194
4195     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4196
4197         qemu_devicedel($vmid, $deviceid);
4198         qemu_devicedelverify($vmid, $deviceid);
4199         qemu_iothread_del($conf, $vmid, $deviceid);
4200
4201     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4202
4203         qemu_devicedel($vmid, $deviceid);
4204         qemu_drivedel($vmid, $deviceid);
4205         qemu_deletescsihw($conf, $vmid, $deviceid);
4206
4207     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4208
4209         qemu_devicedel($vmid, $deviceid);
4210         qemu_devicedelverify($vmid, $deviceid);
4211         qemu_netdevdel($vmid, $deviceid);
4212
4213     } else {
4214         die "can't unplug device '$deviceid'\n";
4215     }
4216
4217     return 1;
4218 }
4219
4220 sub qemu_deviceadd {
4221     my ($vmid, $devicefull) = @_;
4222
4223     $devicefull = "driver=".$devicefull;
4224     my %options =  split(/[=,]/, $devicefull);
4225
4226     vm_mon_cmd($vmid, "device_add" , %options);
4227 }
4228
4229 sub qemu_devicedel {
4230     my ($vmid, $deviceid) = @_;
4231
4232     my $ret = vm_mon_cmd($vmid, "device_del", id => $deviceid);
4233 }
4234
4235 sub qemu_iothread_add {
4236     my($vmid, $deviceid, $device) = @_;
4237
4238     if ($device->{iothread}) {
4239         my $iothreads = vm_iothreads_list($vmid);
4240         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
4241     }
4242 }
4243
4244 sub qemu_iothread_del {
4245     my($conf, $vmid, $deviceid) = @_;
4246
4247     my $device = parse_drive($deviceid, $conf->{$deviceid});
4248     if ($device->{iothread}) {
4249         my $iothreads = vm_iothreads_list($vmid);
4250         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
4251     }
4252 }
4253
4254 sub qemu_objectadd {
4255     my($vmid, $objectid, $qomtype) = @_;
4256
4257     vm_mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
4258
4259     return 1;
4260 }
4261
4262 sub qemu_objectdel {
4263     my($vmid, $objectid) = @_;
4264
4265     vm_mon_cmd($vmid, "object-del", id => $objectid);
4266
4267     return 1;
4268 }
4269
4270 sub qemu_driveadd {
4271     my ($storecfg, $vmid, $device) = @_;
4272
4273     my $drive = print_drive_full($storecfg, $vmid, $device);
4274     $drive =~ s/\\/\\\\/g;
4275     my $ret = vm_human_monitor_command($vmid, "drive_add auto \"$drive\"");
4276
4277     # If the command succeeds qemu prints: "OK"
4278     return 1 if $ret =~ m/OK/s;
4279
4280     die "adding drive failed: $ret\n";
4281 }
4282
4283 sub qemu_drivedel {
4284     my($vmid, $deviceid) = @_;
4285
4286     my $ret = vm_human_monitor_command($vmid, "drive_del drive-$deviceid");
4287     $ret =~ s/^\s+//;
4288
4289     return 1 if $ret eq "";
4290
4291     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
4292     return 1 if $ret =~ m/Device \'.*?\' not found/s;
4293
4294     die "deleting drive $deviceid failed : $ret\n";
4295 }
4296
4297 sub qemu_deviceaddverify {
4298     my ($vmid, $deviceid) = @_;
4299
4300     for (my $i = 0; $i <= 5; $i++) {
4301          my $devices_list = vm_devices_list($vmid);
4302          return 1 if defined($devices_list->{$deviceid});
4303          sleep 1;
4304     }
4305
4306     die "error on hotplug device '$deviceid'\n";
4307 }
4308
4309
4310 sub qemu_devicedelverify {
4311     my ($vmid, $deviceid) = @_;
4312
4313     # need to verify that the device is correctly removed as device_del
4314     # is async and empty return is not reliable
4315
4316     for (my $i = 0; $i <= 5; $i++) {
4317          my $devices_list = vm_devices_list($vmid);
4318          return 1 if !defined($devices_list->{$deviceid});
4319          sleep 1;
4320     }
4321
4322     die "error on hot-unplugging device '$deviceid'\n";
4323 }
4324
4325 sub qemu_findorcreatescsihw {
4326     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4327
4328     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4329
4330     my $scsihwid="$controller_prefix$controller";
4331     my $devices_list = vm_devices_list($vmid);
4332
4333     if(!defined($devices_list->{$scsihwid})) {
4334         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device, $arch, $machine_type);
4335     }
4336
4337     return 1;
4338 }
4339
4340 sub qemu_deletescsihw {
4341     my ($conf, $vmid, $opt) = @_;
4342
4343     my $device = parse_drive($opt, $conf->{$opt});
4344
4345     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
4346         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
4347         return 1;
4348     }
4349
4350     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4351
4352     my $devices_list = vm_devices_list($vmid);
4353     foreach my $opt (keys %{$devices_list}) {
4354         if (PVE::QemuServer::is_valid_drivename($opt)) {
4355             my $drive = PVE::QemuServer::parse_drive($opt, $conf->{$opt});
4356             if($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
4357                 return 1;
4358             }
4359         }
4360     }
4361
4362     my $scsihwid="scsihw$controller";
4363
4364     vm_deviceunplug($vmid, $conf, $scsihwid);
4365
4366     return 1;
4367 }
4368
4369 sub qemu_add_pci_bridge {
4370     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4371
4372     my $bridges = {};
4373
4374     my $bridgeid;
4375
4376     print_pci_addr($device, $bridges, $arch, $machine_type);
4377
4378     while (my ($k, $v) = each %$bridges) {
4379         $bridgeid = $k;
4380     }
4381     return 1 if !defined($bridgeid) || $bridgeid < 1;
4382
4383     my $bridge = "pci.$bridgeid";
4384     my $devices_list = vm_devices_list($vmid);
4385
4386     if (!defined($devices_list->{$bridge})) {
4387         vm_deviceplug($storecfg, $conf, $vmid, $bridge, $arch, $machine_type);
4388     }
4389
4390     return 1;
4391 }
4392
4393 sub qemu_set_link_status {
4394     my ($vmid, $device, $up) = @_;
4395
4396     vm_mon_cmd($vmid, "set_link", name => $device,
4397                up => $up ? JSON::true : JSON::false);
4398 }
4399
4400 sub qemu_netdevadd {
4401     my ($vmid, $conf, $arch, $device, $deviceid) = @_;
4402
4403     my $netdev = print_netdev_full($vmid, $conf, $arch, $device, $deviceid, 1);
4404     my %options =  split(/[=,]/, $netdev);
4405
4406     vm_mon_cmd($vmid, "netdev_add",  %options);
4407     return 1;
4408 }
4409
4410 sub qemu_netdevdel {
4411     my ($vmid, $deviceid) = @_;
4412
4413     vm_mon_cmd($vmid, "netdev_del", id => $deviceid);
4414 }
4415
4416 sub qemu_usb_hotplug {
4417     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4418
4419     return if !$device;
4420
4421     # remove the old one first
4422     vm_deviceunplug($vmid, $conf, $deviceid);
4423
4424     # check if xhci controller is necessary and available
4425     if ($device->{usb3}) {
4426
4427         my $devicelist = vm_devices_list($vmid);
4428
4429         if (!$devicelist->{xhci}) {
4430             my $pciaddr = print_pci_addr("xhci", undef, $arch, $machine_type);
4431             qemu_deviceadd($vmid, "nec-usb-xhci,id=xhci$pciaddr");
4432         }
4433     }
4434     my $d = parse_usb_device($device->{host});
4435     $d->{usb3} = $device->{usb3};
4436
4437     # add the new one
4438     vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $d, $arch, $machine_type);
4439 }
4440
4441 sub qemu_cpu_hotplug {
4442     my ($vmid, $conf, $vcpus) = @_;
4443
4444     my $machine_type = PVE::QemuServer::get_current_qemu_machine($vmid);
4445
4446     my $sockets = 1;
4447     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
4448     $sockets = $conf->{sockets} if  $conf->{sockets};
4449     my $cores = $conf->{cores} || 1;
4450     my $maxcpus = $sockets * $cores;
4451
4452     $vcpus = $maxcpus if !$vcpus;
4453
4454     die "you can't add more vcpus than maxcpus\n"
4455         if $vcpus > $maxcpus;
4456
4457     my $currentvcpus = $conf->{vcpus} || $maxcpus;
4458
4459     if ($vcpus < $currentvcpus) {
4460
4461         if (qemu_machine_feature_enabled ($machine_type, undef, 2, 7)) {
4462
4463             for (my $i = $currentvcpus; $i > $vcpus; $i--) {
4464                 qemu_devicedel($vmid, "cpu$i");
4465                 my $retry = 0;
4466                 my $currentrunningvcpus = undef;
4467                 while (1) {
4468                     $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
4469                     last if scalar(@{$currentrunningvcpus}) == $i-1;
4470                     raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
4471                     $retry++;
4472                     sleep 1;
4473                 }
4474                 #update conf after each succesfull cpu unplug
4475                 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4476                 PVE::QemuConfig->write_config($vmid, $conf);
4477             }
4478         } else {
4479             die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
4480         }
4481
4482         return;
4483     }
4484
4485     my $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
4486     die "vcpus in running vm does not match its configuration\n"
4487         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
4488
4489     if (qemu_machine_feature_enabled ($machine_type, undef, 2, 7)) {
4490
4491         for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
4492             my $cpustr = print_cpu_device($conf, $i);
4493             qemu_deviceadd($vmid, $cpustr);
4494
4495             my $retry = 0;
4496             my $currentrunningvcpus = undef;
4497             while (1) {
4498                 $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
4499                 last if scalar(@{$currentrunningvcpus}) == $i;
4500                 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
4501                 sleep 1;
4502                 $retry++;
4503             }
4504             #update conf after each succesfull cpu hotplug
4505             $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4506             PVE::QemuConfig->write_config($vmid, $conf);
4507         }
4508     } else {
4509
4510         for (my $i = $currentvcpus; $i < $vcpus; $i++) {
4511             vm_mon_cmd($vmid, "cpu-add", id => int($i));
4512         }
4513     }
4514 }
4515
4516 sub qemu_block_set_io_throttle {
4517     my ($vmid, $deviceid,
4518         $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
4519         $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max,
4520         $bps_max_length, $bps_rd_max_length, $bps_wr_max_length,
4521         $iops_max_length, $iops_rd_max_length, $iops_wr_max_length) = @_;
4522
4523     return if !check_running($vmid) ;
4524
4525     vm_mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
4526         bps => int($bps),
4527         bps_rd => int($bps_rd),
4528         bps_wr => int($bps_wr),
4529         iops => int($iops),
4530         iops_rd => int($iops_rd),
4531         iops_wr => int($iops_wr),
4532         bps_max => int($bps_max),
4533         bps_rd_max => int($bps_rd_max),
4534         bps_wr_max => int($bps_wr_max),
4535         iops_max => int($iops_max),
4536         iops_rd_max => int($iops_rd_max),
4537         iops_wr_max => int($iops_wr_max),
4538         bps_max_length => int($bps_max_length),
4539         bps_rd_max_length => int($bps_rd_max_length),
4540         bps_wr_max_length => int($bps_wr_max_length),
4541         iops_max_length => int($iops_max_length),
4542         iops_rd_max_length => int($iops_rd_max_length),
4543         iops_wr_max_length => int($iops_wr_max_length),
4544     );
4545
4546 }
4547
4548 # old code, only used to shutdown old VM after update
4549 sub __read_avail {
4550     my ($fh, $timeout) = @_;
4551
4552     my $sel = new IO::Select;
4553     $sel->add($fh);
4554
4555     my $res = '';
4556     my $buf;
4557
4558     my @ready;
4559     while (scalar (@ready = $sel->can_read($timeout))) {
4560         my $count;
4561         if ($count = $fh->sysread($buf, 8192)) {
4562             if ($buf =~ /^(.*)\(qemu\) $/s) {
4563                 $res .= $1;
4564                 last;
4565             } else {
4566                 $res .= $buf;
4567             }
4568         } else {
4569             if (!defined($count)) {
4570                 die "$!\n";
4571             }
4572             last;
4573         }
4574     }
4575
4576     die "monitor read timeout\n" if !scalar(@ready);
4577
4578     return $res;
4579 }
4580
4581 sub qemu_block_resize {
4582     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4583
4584     my $running = check_running($vmid);
4585
4586     $size = 0 if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4587
4588     return if !$running;
4589
4590     vm_mon_cmd($vmid, "block_resize", device => $deviceid, size => int($size));
4591
4592 }
4593
4594 sub qemu_volume_snapshot {
4595     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4596
4597     my $running = check_running($vmid);
4598
4599     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4600         vm_mon_cmd($vmid, 'blockdev-snapshot-internal-sync', device => $deviceid, name => $snap);
4601     } else {
4602         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4603     }
4604 }
4605
4606 sub qemu_volume_snapshot_delete {
4607     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4608
4609     my $running = check_running($vmid);
4610
4611     if($running) {
4612
4613         $running = undef;
4614         my $conf = PVE::QemuConfig->load_config($vmid);
4615         foreach_drive($conf, sub {
4616             my ($ds, $drive) = @_;
4617             $running = 1 if $drive->{file} eq $volid;
4618         });
4619     }
4620
4621     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4622         vm_mon_cmd($vmid, 'blockdev-snapshot-delete-internal-sync', device => $deviceid, name => $snap);
4623     } else {
4624         PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4625     }
4626 }
4627
4628 sub set_migration_caps {
4629     my ($vmid) = @_;
4630
4631     my $cap_ref = [];
4632
4633     my $enabled_cap = {
4634         "auto-converge" => 1,
4635         "xbzrle" => 1,
4636         "x-rdma-pin-all" => 0,
4637         "zero-blocks" => 0,
4638         "compress" => 0
4639     };
4640
4641     my $supported_capabilities = vm_mon_cmd_nocheck($vmid, "query-migrate-capabilities");
4642
4643     for my $supported_capability (@$supported_capabilities) {
4644         push @$cap_ref, {
4645             capability => $supported_capability->{capability},
4646             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4647         };
4648     }
4649
4650     vm_mon_cmd_nocheck($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4651 }
4652
4653 my $fast_plug_option = {
4654     'lock' => 1,
4655     'name' => 1,
4656     'onboot' => 1,
4657     'shares' => 1,
4658     'startup' => 1,
4659     'description' => 1,
4660     'protection' => 1,
4661     'vmstatestorage' => 1,
4662     'hookscript' => 1,
4663 };
4664
4665 # hotplug changes in [PENDING]
4666 # $selection hash can be used to only apply specified options, for
4667 # example: { cores => 1 } (only apply changed 'cores')
4668 # $errors ref is used to return error messages
4669 sub vmconfig_hotplug_pending {
4670     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4671
4672     my $defaults = load_defaults();
4673     my ($arch, $machine_type) = get_basic_machine_info($conf, undef);
4674
4675     # commit values which do not have any impact on running VM first
4676     # Note: those option cannot raise errors, we we do not care about
4677     # $selection and always apply them.
4678
4679     my $add_error = sub {
4680         my ($opt, $msg) = @_;
4681         $errors->{$opt} = "hotplug problem - $msg";
4682     };
4683
4684     my $changes = 0;
4685     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4686         if ($fast_plug_option->{$opt}) {
4687             $conf->{$opt} = $conf->{pending}->{$opt};
4688             delete $conf->{pending}->{$opt};
4689             $changes = 1;
4690         }
4691     }
4692
4693     if ($changes) {
4694         PVE::QemuConfig->write_config($vmid, $conf);
4695         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4696     }
4697
4698     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4699
4700     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4701     while (my ($opt, $force) = each %$pending_delete_hash) {
4702         next if $selection && !$selection->{$opt};
4703         eval {
4704             if ($opt eq 'hotplug') {
4705                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
4706             } elsif ($opt eq 'tablet') {
4707                 die "skip\n" if !$hotplug_features->{usb};
4708                 if ($defaults->{tablet}) {
4709                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4710                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4711                         if $arch eq 'aarch64';
4712                 } else {
4713                     vm_deviceunplug($vmid, $conf, 'tablet');
4714                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4715                 }
4716             } elsif ($opt =~ m/^usb\d+/) {
4717                 die "skip\n";
4718                 # since we cannot reliably hot unplug usb devices
4719                 # we are disabling it
4720                 die "skip\n" if !$hotplug_features->{usb} || $conf->{$opt} =~ m/spice/i;
4721                 vm_deviceunplug($vmid, $conf, $opt);
4722             } elsif ($opt eq 'vcpus') {
4723                 die "skip\n" if !$hotplug_features->{cpu};
4724                 qemu_cpu_hotplug($vmid, $conf, undef);
4725             } elsif ($opt eq 'balloon') {
4726                 # enable balloon device is not hotpluggable
4727                 die "skip\n" if defined($conf->{balloon}) && $conf->{balloon} == 0;
4728                 # here we reset the ballooning value to memory
4729                 my $balloon = $conf->{memory} || $defaults->{memory};
4730                 vm_mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4731             } elsif ($fast_plug_option->{$opt}) {
4732                 # do nothing
4733             } elsif ($opt =~ m/^net(\d+)$/) {
4734                 die "skip\n" if !$hotplug_features->{network};
4735                 vm_deviceunplug($vmid, $conf, $opt);
4736             } elsif (is_valid_drivename($opt)) {
4737                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4738                 vm_deviceunplug($vmid, $conf, $opt);
4739                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4740             } elsif ($opt =~ m/^memory$/) {
4741                 die "skip\n" if !$hotplug_features->{memory};
4742                 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
4743             } elsif ($opt eq 'cpuunits') {
4744                 cgroups_write("cpu", $vmid, "cpu.shares", $defaults->{cpuunits});
4745             } elsif ($opt eq 'cpulimit') {
4746                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", -1);
4747             } else {
4748                 die "skip\n";
4749             }
4750         };
4751         if (my $err = $@) {
4752             &$add_error($opt, $err) if $err ne "skip\n";
4753         } else {
4754             # save new config if hotplug was successful
4755             delete $conf->{$opt};
4756             vmconfig_undelete_pending_option($conf, $opt);
4757             PVE::QemuConfig->write_config($vmid, $conf);
4758             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4759         }
4760     }
4761
4762     my $apply_pending_cloudinit;
4763     $apply_pending_cloudinit = sub {
4764         my ($key, $value) = @_;
4765         $apply_pending_cloudinit = sub {}; # once is enough
4766
4767         my @cloudinit_opts = keys %$confdesc_cloudinit;
4768         foreach my $opt (keys %{$conf->{pending}}) {
4769             next if !grep { $_ eq $opt } @cloudinit_opts;
4770             $conf->{$opt} = delete $conf->{pending}->{$opt};
4771         }
4772
4773         my $new_conf = { %$conf };
4774         $new_conf->{$key} = $value;
4775         PVE::QemuServer::Cloudinit::generate_cloudinitconfig($new_conf, $vmid);
4776     };
4777
4778     foreach my $opt (keys %{$conf->{pending}}) {
4779         next if $selection && !$selection->{$opt};
4780         my $value = $conf->{pending}->{$opt};
4781         eval {
4782             if ($opt eq 'hotplug') {
4783                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
4784             } elsif ($opt eq 'tablet') {
4785                 die "skip\n" if !$hotplug_features->{usb};
4786                 if ($value == 1) {
4787                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4788                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4789                         if $arch eq 'aarch64';
4790                 } elsif ($value == 0) {
4791                     vm_deviceunplug($vmid, $conf, 'tablet');
4792                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4793                 }
4794             } elsif ($opt =~ m/^usb\d+$/) {
4795                 die "skip\n";
4796                 # since we cannot reliably hot unplug usb devices
4797                 # we are disabling it
4798                 die "skip\n" if !$hotplug_features->{usb} || $value =~ m/spice/i;
4799                 my $d = eval { PVE::JSONSchema::parse_property_string($usbdesc->{format}, $value) };
4800                 die "skip\n" if !$d;
4801                 qemu_usb_hotplug($storecfg, $conf, $vmid, $opt, $d, $arch, $machine_type);
4802             } elsif ($opt eq 'vcpus') {
4803                 die "skip\n" if !$hotplug_features->{cpu};
4804                 qemu_cpu_hotplug($vmid, $conf, $value);
4805             } elsif ($opt eq 'balloon') {
4806                 # enable/disable balloning device is not hotpluggable
4807                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
4808                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
4809                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
4810
4811                 # allow manual ballooning if shares is set to zero
4812                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
4813                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
4814                     vm_mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4815                 }
4816             } elsif ($opt =~ m/^net(\d+)$/) {
4817                 # some changes can be done without hotplug
4818                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
4819                                     $vmid, $opt, $value, $arch, $machine_type);
4820             } elsif (is_valid_drivename($opt)) {
4821                 # some changes can be done without hotplug
4822                 my $drive = parse_drive($opt, $value);
4823                 if (drive_is_cloudinit($drive)) {
4824                     &$apply_pending_cloudinit($opt, $value);
4825                 }
4826                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
4827                                      $vmid, $opt, $value, 1, $arch, $machine_type);
4828             } elsif ($opt =~ m/^memory$/) { #dimms
4829                 die "skip\n" if !$hotplug_features->{memory};
4830                 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
4831             } elsif ($opt eq 'cpuunits') {
4832                 cgroups_write("cpu", $vmid, "cpu.shares", $conf->{pending}->{$opt});
4833             } elsif ($opt eq 'cpulimit') {
4834                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
4835                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", $cpulimit);
4836             } else {
4837                 die "skip\n";  # skip non-hot-pluggable options
4838             }
4839         };
4840         if (my $err = $@) {
4841             &$add_error($opt, $err) if $err ne "skip\n";
4842         } else {
4843             # save new config if hotplug was successful
4844             $conf->{$opt} = $value;
4845             delete $conf->{pending}->{$opt};
4846             PVE::QemuConfig->write_config($vmid, $conf);
4847             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4848         }
4849     }
4850 }
4851
4852 sub try_deallocate_drive {
4853     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
4854
4855     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
4856         my $volid = $drive->{file};
4857         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
4858             my $sid = PVE::Storage::parse_volume_id($volid);
4859             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
4860
4861             # check if the disk is really unused
4862             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
4863                 if is_volume_in_use($storecfg, $conf, $key, $volid);
4864             PVE::Storage::vdisk_free($storecfg, $volid);
4865             return 1;
4866         } else {
4867             # If vm is not owner of this disk remove from config
4868             return 1;
4869         }
4870     }
4871
4872     return undef;
4873 }
4874
4875 sub vmconfig_delete_or_detach_drive {
4876     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
4877
4878     my $drive = parse_drive($opt, $conf->{$opt});
4879
4880     my $rpcenv = PVE::RPCEnvironment::get();
4881     my $authuser = $rpcenv->get_user();
4882
4883     if ($force) {
4884         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
4885         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
4886     } else {
4887         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
4888     }
4889 }
4890
4891 sub vmconfig_apply_pending {
4892     my ($vmid, $conf, $storecfg) = @_;
4893
4894     # cold plug
4895
4896     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4897     while (my ($opt, $force) = each %$pending_delete_hash) {
4898         die "internal error" if $opt =~ m/^unused/;
4899         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4900         if (!defined($conf->{$opt})) {
4901             vmconfig_undelete_pending_option($conf, $opt);
4902             PVE::QemuConfig->write_config($vmid, $conf);
4903         } elsif (is_valid_drivename($opt)) {
4904             vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4905             vmconfig_undelete_pending_option($conf, $opt);
4906             delete $conf->{$opt};
4907             PVE::QemuConfig->write_config($vmid, $conf);
4908         } else {
4909             vmconfig_undelete_pending_option($conf, $opt);
4910             delete $conf->{$opt};
4911             PVE::QemuConfig->write_config($vmid, $conf);
4912         }
4913     }
4914
4915     $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4916
4917     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4918         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4919
4920         if (defined($conf->{$opt}) && ($conf->{$opt} eq $conf->{pending}->{$opt})) {
4921             # skip if nothing changed
4922         } elsif (is_valid_drivename($opt)) {
4923             vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
4924                 if defined($conf->{$opt});
4925             $conf->{$opt} = $conf->{pending}->{$opt};
4926         } else {
4927             $conf->{$opt} = $conf->{pending}->{$opt};
4928         }
4929
4930         delete $conf->{pending}->{$opt};
4931         PVE::QemuConfig->write_config($vmid, $conf);
4932     }
4933 }
4934
4935 my $safe_num_ne = sub {
4936     my ($a, $b) = @_;
4937
4938     return 0 if !defined($a) && !defined($b);
4939     return 1 if !defined($a);
4940     return 1 if !defined($b);
4941
4942     return $a != $b;
4943 };
4944
4945 my $safe_string_ne = sub {
4946     my ($a, $b) = @_;
4947
4948     return 0 if !defined($a) && !defined($b);
4949     return 1 if !defined($a);
4950     return 1 if !defined($b);
4951
4952     return $a ne $b;
4953 };
4954
4955 sub vmconfig_update_net {
4956     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
4957
4958     my $newnet = parse_net($value);
4959
4960     if ($conf->{$opt}) {
4961         my $oldnet = parse_net($conf->{$opt});
4962
4963         if (&$safe_string_ne($oldnet->{model}, $newnet->{model}) ||
4964             &$safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
4965             &$safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
4966             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
4967
4968             # for non online change, we try to hot-unplug
4969             die "skip\n" if !$hotplug;
4970             vm_deviceunplug($vmid, $conf, $opt);
4971         } else {
4972
4973             die "internal error" if $opt !~ m/net(\d+)/;
4974             my $iface = "tap${vmid}i$1";
4975
4976             if (&$safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
4977                 &$safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
4978                 &$safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
4979                 &$safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
4980                 PVE::Network::tap_unplug($iface);
4981                 PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
4982             } elsif (&$safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
4983                 # Rate can be applied on its own but any change above needs to
4984                 # include the rate in tap_plug since OVS resets everything.
4985                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
4986             }
4987
4988             if (&$safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
4989                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
4990             }
4991
4992             return 1;
4993         }
4994     }
4995
4996     if ($hotplug) {
4997         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet, $arch, $machine_type);
4998     } else {
4999         die "skip\n";
5000     }
5001 }
5002
5003 sub vmconfig_update_disk {
5004     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $force, $arch, $machine_type) = @_;
5005
5006     # fixme: do we need force?
5007
5008     my $drive = parse_drive($opt, $value);
5009
5010     if ($conf->{$opt}) {
5011
5012         if (my $old_drive = parse_drive($opt, $conf->{$opt}))  {
5013
5014             my $media = $drive->{media} || 'disk';
5015             my $oldmedia = $old_drive->{media} || 'disk';
5016             die "unable to change media type\n" if $media ne $oldmedia;
5017
5018             if (!drive_is_cdrom($old_drive)) {
5019
5020                 if ($drive->{file} ne $old_drive->{file}) {
5021
5022                     die "skip\n" if !$hotplug;
5023
5024                     # unplug and register as unused
5025                     vm_deviceunplug($vmid, $conf, $opt);
5026                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
5027
5028                 } else {
5029                     # update existing disk
5030
5031                     # skip non hotpluggable value
5032                     if (&$safe_string_ne($drive->{discard}, $old_drive->{discard}) ||
5033                         &$safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
5034                         &$safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
5035                         &$safe_string_ne($drive->{cache}, $old_drive->{cache})) {
5036                         die "skip\n";
5037                     }
5038
5039                     # apply throttle
5040                     if (&$safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
5041                         &$safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
5042                         &$safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
5043                         &$safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
5044                         &$safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
5045                         &$safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
5046                         &$safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
5047                         &$safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
5048                         &$safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
5049                         &$safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
5050                         &$safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
5051                         &$safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max}) ||
5052                         &$safe_num_ne($drive->{bps_max_length}, $old_drive->{bps_max_length}) ||
5053                         &$safe_num_ne($drive->{bps_rd_max_length}, $old_drive->{bps_rd_max_length}) ||
5054                         &$safe_num_ne($drive->{bps_wr_max_length}, $old_drive->{bps_wr_max_length}) ||
5055                         &$safe_num_ne($drive->{iops_max_length}, $old_drive->{iops_max_length}) ||
5056                         &$safe_num_ne($drive->{iops_rd_max_length}, $old_drive->{iops_rd_max_length}) ||
5057                         &$safe_num_ne($drive->{iops_wr_max_length}, $old_drive->{iops_wr_max_length})) {
5058
5059                         qemu_block_set_io_throttle($vmid,"drive-$opt",
5060                                                    ($drive->{mbps} || 0)*1024*1024,
5061                                                    ($drive->{mbps_rd} || 0)*1024*1024,
5062                                                    ($drive->{mbps_wr} || 0)*1024*1024,
5063                                                    $drive->{iops} || 0,
5064                                                    $drive->{iops_rd} || 0,
5065                                                    $drive->{iops_wr} || 0,
5066                                                    ($drive->{mbps_max} || 0)*1024*1024,
5067                                                    ($drive->{mbps_rd_max} || 0)*1024*1024,
5068                                                    ($drive->{mbps_wr_max} || 0)*1024*1024,
5069                                                    $drive->{iops_max} || 0,
5070                                                    $drive->{iops_rd_max} || 0,
5071                                                    $drive->{iops_wr_max} || 0,
5072                                                    $drive->{bps_max_length} || 1,
5073                                                    $drive->{bps_rd_max_length} || 1,
5074                                                    $drive->{bps_wr_max_length} || 1,
5075                                                    $drive->{iops_max_length} || 1,
5076                                                    $drive->{iops_rd_max_length} || 1,
5077                                                    $drive->{iops_wr_max_length} || 1);
5078
5079                     }
5080
5081                     return 1;
5082                 }
5083
5084             } else { # cdrom
5085
5086                 if ($drive->{file} eq 'none') {
5087                     vm_mon_cmd($vmid, "eject",force => JSON::true,device => "drive-$opt");
5088                     if (drive_is_cloudinit($old_drive)) {
5089                         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive);
5090                     }
5091                 } else {
5092                     my $path = get_iso_path($storecfg, $vmid, $drive->{file});
5093                     vm_mon_cmd($vmid, "eject", force => JSON::true,device => "drive-$opt"); # force eject if locked
5094                     vm_mon_cmd($vmid, "change", device => "drive-$opt",target => "$path") if $path;
5095                 }
5096
5097                 return 1;
5098             }
5099         }
5100     }
5101
5102     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
5103     # hotplug new disks
5104     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
5105     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive, $arch, $machine_type);
5106 }
5107
5108 sub vm_start {
5109     my ($storecfg, $vmid, $statefile, $skiplock, $migratedfrom, $paused,
5110         $forcemachine, $spice_ticket, $migration_network, $migration_type, $targetstorage) = @_;
5111
5112     PVE::QemuConfig->lock_config($vmid, sub {
5113         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
5114
5115         die "you can't start a vm if it's a template\n" if PVE::QemuConfig->is_template($conf);
5116
5117         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5118
5119         die "VM $vmid already running\n" if check_running($vmid, undef, $migratedfrom);
5120
5121         if (!$statefile && scalar(keys %{$conf->{pending}})) {
5122             vmconfig_apply_pending($vmid, $conf, $storecfg);
5123             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5124         }
5125
5126         PVE::QemuServer::Cloudinit::generate_cloudinitconfig($conf, $vmid);
5127
5128         my $defaults = load_defaults();
5129
5130         # set environment variable useful inside network script
5131         $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
5132
5133         my $local_volumes = {};
5134
5135         if ($targetstorage) {
5136             foreach_drive($conf, sub {
5137                 my ($ds, $drive) = @_;
5138
5139                 return if drive_is_cdrom($drive);
5140
5141                 my $volid = $drive->{file};
5142
5143                 return if !$volid;
5144
5145                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
5146
5147                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5148                 return if $scfg->{shared};
5149                 $local_volumes->{$ds} = [$volid, $storeid, $volname];
5150             });
5151
5152             my $format = undef;
5153
5154             foreach my $opt (sort keys %$local_volumes) {
5155
5156                 my ($volid, $storeid, $volname) = @{$local_volumes->{$opt}};
5157                 my $drive = parse_drive($opt, $conf->{$opt});
5158
5159                 #if remote storage is specified, use default format
5160                 if ($targetstorage && $targetstorage ne "1") {
5161                     $storeid = $targetstorage;
5162                     my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
5163                     $format = $defFormat;
5164                 } else {
5165                     #else we use same format than original
5166                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5167                     $format = qemu_img_format($scfg, $volid);
5168                 }
5169
5170                 my $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $format, undef, ($drive->{size}/1024));
5171                 my $newdrive = $drive;
5172                 $newdrive->{format} = $format;
5173                 $newdrive->{file} = $newvolid;
5174                 my $drivestr = PVE::QemuServer::print_drive($vmid, $newdrive);
5175                 $local_volumes->{$opt} = $drivestr;
5176                 #pass drive to conf for command line
5177                 $conf->{$opt} = $drivestr;
5178             }
5179         }
5180
5181         PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-start', 1);
5182
5183         my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine);
5184
5185         my $migrate_port = 0;
5186         my $migrate_uri;
5187         if ($statefile) {
5188             if ($statefile eq 'tcp') {
5189                 my $localip = "localhost";
5190                 my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
5191                 my $nodename = PVE::INotify::nodename();
5192
5193                 if (!defined($migration_type)) {
5194                     if (defined($datacenterconf->{migration}->{type})) {
5195                         $migration_type = $datacenterconf->{migration}->{type};
5196                     } else {
5197                         $migration_type = 'secure';
5198                     }
5199                 }
5200
5201                 if ($migration_type eq 'insecure') {
5202                     my $migrate_network_addr = PVE::Cluster::get_local_migration_ip($migration_network);
5203                     if ($migrate_network_addr) {
5204                         $localip = $migrate_network_addr;
5205                     } else {
5206                         $localip = PVE::Cluster::remote_node_ip($nodename, 1);
5207                     }
5208
5209                     $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5210                 }
5211
5212                 my $pfamily = PVE::Tools::get_host_address_family($nodename);
5213                 $migrate_port = PVE::Tools::next_migrate_port($pfamily);
5214                 $migrate_uri = "tcp:${localip}:${migrate_port}";
5215                 push @$cmd, '-incoming', $migrate_uri;
5216                 push @$cmd, '-S';
5217
5218             } elsif ($statefile eq 'unix') {
5219                 # should be default for secure migrations as a ssh TCP forward
5220                 # tunnel is not deterministic reliable ready and fails regurarly
5221                 # to set up in time, so use UNIX socket forwards
5222                 my $socket_addr = "/run/qemu-server/$vmid.migrate";
5223                 unlink $socket_addr;
5224
5225                 $migrate_uri = "unix:$socket_addr";
5226
5227                 push @$cmd, '-incoming', $migrate_uri;
5228                 push @$cmd, '-S';
5229
5230             } else {
5231                 push @$cmd, '-loadstate', $statefile;
5232             }
5233         } elsif ($paused) {
5234             push @$cmd, '-S';
5235         }
5236
5237         # host pci devices
5238         for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
5239           my $d = parse_hostpci($conf->{"hostpci$i"});
5240           next if !$d;
5241           my $pcidevices = $d->{pciid};
5242           foreach my $pcidevice (@$pcidevices) {
5243                 my $pciid = $pcidevice->{id};
5244
5245                 my $info = PVE::SysFSTools::pci_device_info("0000:$pciid");
5246                 die "IOMMU not present\n" if !PVE::SysFSTools::check_iommu_support();
5247                 die "no pci device info for device '$pciid'\n" if !$info;
5248
5249                 if ($d->{mdev}) {
5250                     my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $i);
5251                     PVE::SysFSTools::pci_create_mdev_device($pciid, $uuid, $d->{mdev});
5252                 } else {
5253                     die "can't unbind/bind pci group to vfio '$pciid'\n"
5254                         if !PVE::SysFSTools::pci_dev_group_bind_to_vfio($pciid);
5255                     die "can't reset pci device '$pciid'\n"
5256                         if $info->{has_fl_reset} and !PVE::SysFSTools::pci_dev_reset($info);
5257                 }
5258           }
5259         }
5260
5261         PVE::Storage::activate_volumes($storecfg, $vollist);
5262
5263         if (-d "/sys/fs/cgroup/systemd/qemu.slice/$vmid.scope") {
5264             eval {
5265                 run_command(['/bin/systemctl', 'stop', "$vmid.scope"],
5266                     outfunc => sub {}, errfunc => sub {});
5267             };
5268         }
5269
5270         my $cpuunits = defined($conf->{cpuunits}) ? $conf->{cpuunits}
5271                                                   : $defaults->{cpuunits};
5272
5273         my $start_timeout = $conf->{hugepages} ? 300 : 30;
5274         my %run_params = (timeout => $statefile ? undef : $start_timeout, umask => 0077);
5275
5276         my %properties = (
5277             Slice => 'qemu.slice',
5278             KillMode => 'none',
5279             CPUShares => $cpuunits
5280         );
5281
5282         if (my $cpulimit = $conf->{cpulimit}) {
5283             $properties{CPUQuota} = int($cpulimit * 100);
5284         }
5285         $properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
5286
5287         my $run_qemu = sub {
5288             PVE::Tools::run_fork sub {
5289                 PVE::Systemd::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
5290                 run_command($cmd, %run_params);
5291             };
5292         };
5293
5294         if ($conf->{hugepages}) {
5295
5296             my $code = sub {
5297                 my $hugepages_topology = PVE::QemuServer::Memory::hugepages_topology($conf);
5298                 my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
5299
5300                 PVE::QemuServer::Memory::hugepages_mount();
5301                 PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
5302
5303                 eval { $run_qemu->() };
5304                 if (my $err = $@) {
5305                     PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology);
5306                     die $err;
5307                 }
5308
5309                 PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology);
5310             };
5311             eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
5312
5313         } else {
5314             eval { $run_qemu->() };
5315         }
5316
5317         if (my $err = $@) {
5318             # deactivate volumes if start fails
5319             eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
5320             die "start failed: $err";
5321         }
5322
5323         print "migration listens on $migrate_uri\n" if $migrate_uri;
5324
5325         if ($statefile && $statefile ne 'tcp' && $statefile ne 'unix')  {
5326             eval { vm_mon_cmd_nocheck($vmid, "cont"); };
5327             warn $@ if $@;
5328         }
5329
5330         #start nbd server for storage migration
5331         if ($targetstorage) {
5332             my $nodename = PVE::INotify::nodename();
5333             my $migrate_network_addr = PVE::Cluster::get_local_migration_ip($migration_network);
5334             my $localip = $migrate_network_addr ? $migrate_network_addr : PVE::Cluster::remote_node_ip($nodename, 1);
5335             my $pfamily = PVE::Tools::get_host_address_family($nodename);
5336             $migrate_port = PVE::Tools::next_migrate_port($pfamily);
5337
5338             vm_mon_cmd_nocheck($vmid, "nbd-server-start", addr => { type => 'inet', data => { host => "${localip}", port => "${migrate_port}" } } );
5339
5340             $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5341
5342             foreach my $opt (sort keys %$local_volumes) {
5343                 my $volid = $local_volumes->{$opt};
5344                 vm_mon_cmd_nocheck($vmid, "nbd-server-add", device => "drive-$opt", writable => JSON::true );
5345                 my $migrate_storage_uri = "nbd:${localip}:${migrate_port}:exportname=drive-$opt";
5346                 print "storage migration listens on $migrate_storage_uri volume:$volid\n";
5347             }
5348         }
5349
5350         if ($migratedfrom) {
5351             eval {
5352                 set_migration_caps($vmid);
5353             };
5354             warn $@ if $@;
5355
5356             if ($spice_port) {
5357                 print "spice listens on port $spice_port\n";
5358                 if ($spice_ticket) {
5359                     vm_mon_cmd_nocheck($vmid, "set_password", protocol => 'spice', password => $spice_ticket);
5360                     vm_mon_cmd_nocheck($vmid, "expire_password", protocol => 'spice', time => "+30");
5361                 }
5362             }
5363
5364         } else {
5365             vm_mon_cmd_nocheck($vmid, "balloon", value => $conf->{balloon}*1024*1024)
5366                 if !$statefile && $conf->{balloon};
5367
5368             foreach my $opt (keys %$conf) {
5369                 next if $opt !~  m/^net\d+$/;
5370                 my $nicconf = parse_net($conf->{$opt});
5371                 qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
5372             }
5373         }
5374
5375         vm_mon_cmd_nocheck($vmid, 'qom-set',
5376                     path => "machine/peripheral/balloon0",
5377                     property => "guest-stats-polling-interval",
5378                     value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
5379
5380         PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'post-start');
5381     });
5382 }
5383
5384 sub vm_mon_cmd {
5385     my ($vmid, $execute, %params) = @_;
5386
5387     my $cmd = { execute => $execute, arguments => \%params };
5388     vm_qmp_command($vmid, $cmd);
5389 }
5390
5391 sub vm_mon_cmd_nocheck {
5392     my ($vmid, $execute, %params) = @_;
5393
5394     my $cmd = { execute => $execute, arguments => \%params };
5395     vm_qmp_command($vmid, $cmd, 1);
5396 }
5397
5398 sub vm_qmp_command {
5399     my ($vmid, $cmd, $nocheck) = @_;
5400
5401     my $res;
5402
5403     my $timeout;
5404     if ($cmd->{arguments} && $cmd->{arguments}->{timeout}) {
5405         $timeout = $cmd->{arguments}->{timeout};
5406         delete $cmd->{arguments}->{timeout};
5407     }
5408
5409     eval {
5410         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
5411         my $sname = qmp_socket($vmid);
5412         if (-e $sname) { # test if VM is reasonambe new and supports qmp/qga
5413             my $qmpclient = PVE::QMPClient->new();
5414
5415             $res = $qmpclient->cmd($vmid, $cmd, $timeout);
5416         } else {
5417             die "unable to open monitor socket\n";
5418         }
5419     };
5420     if (my $err = $@) {
5421         syslog("err", "VM $vmid qmp command failed - $err");
5422         die $err;
5423     }
5424
5425     return $res;
5426 }
5427
5428 sub vm_human_monitor_command {
5429     my ($vmid, $cmdline) = @_;
5430
5431     my $res;
5432
5433     my $cmd = {
5434         execute => 'human-monitor-command',
5435         arguments => { 'command-line' => $cmdline},
5436     };
5437
5438     return vm_qmp_command($vmid, $cmd);
5439 }
5440
5441 sub vm_commandline {
5442     my ($storecfg, $vmid, $snapname) = @_;
5443
5444     my $conf = PVE::QemuConfig->load_config($vmid);
5445
5446     if ($snapname) {
5447         my $snapshot = $conf->{snapshots}->{$snapname};
5448         die "snapshot '$snapname' does not exist\n" if !defined($snapshot);
5449
5450         $snapshot->{digest} = $conf->{digest}; # keep file digest for API
5451
5452         $conf = $snapshot;
5453     }
5454
5455     my $defaults = load_defaults();
5456
5457     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults);
5458
5459     return PVE::Tools::cmd2string($cmd);
5460 }
5461
5462 sub vm_reset {
5463     my ($vmid, $skiplock) = @_;
5464
5465     PVE::QemuConfig->lock_config($vmid, sub {
5466
5467         my $conf = PVE::QemuConfig->load_config($vmid);
5468
5469         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5470
5471         vm_mon_cmd($vmid, "system_reset");
5472     });
5473 }
5474
5475 sub get_vm_volumes {
5476     my ($conf) = @_;
5477
5478     my $vollist = [];
5479     foreach_volid($conf, sub {
5480         my ($volid, $attr) = @_;
5481
5482         return if $volid =~ m|^/|;
5483
5484         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5485         return if !$sid;
5486
5487         push @$vollist, $volid;
5488     });
5489
5490     return $vollist;
5491 }
5492
5493 sub vm_stop_cleanup {
5494     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
5495
5496     eval {
5497
5498         if (!$keepActive) {
5499             my $vollist = get_vm_volumes($conf);
5500             PVE::Storage::deactivate_volumes($storecfg, $vollist);
5501         }
5502
5503         foreach my $ext (qw(mon qmp pid vnc qga)) {
5504             unlink "/var/run/qemu-server/${vmid}.$ext";
5505         }
5506
5507         if ($conf->{ivshmem}) {
5508             my $ivshmem = PVE::JSONSchema::parse_property_string($ivshmem_fmt, $conf->{ivshmem});
5509             # just delete it for now, VMs which have this already open do not
5510             # are affected, but new VMs will get a separated one. If this
5511             # becomes an issue we either add some sort of ref-counting or just
5512             # add a "don't delete on stop" flag to the ivshmem format.
5513             unlink '/dev/shm/pve-shm-' . ($ivshmem->{name} // $vmid);
5514         }
5515
5516         foreach my $key (keys %$conf) {
5517             next if $key !~ m/^hostpci(\d+)$/;
5518             my $hostpciindex = $1;
5519             my $d = parse_hostpci($conf->{$key});
5520             my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $hostpciindex);
5521
5522             foreach my $pci (@{$d->{pciid}}) {
5523                 my $pciid = $pci->{id};
5524                 PVE::SysFSTools::pci_cleanup_mdev_device($pciid, $uuid);
5525             }
5526         }
5527
5528         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
5529     };
5530     warn $@ if $@; # avoid errors - just warn
5531 }
5532
5533 # Note: use $nockeck to skip tests if VM configuration file exists.
5534 # We need that when migration VMs to other nodes (files already moved)
5535 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
5536 sub vm_stop {
5537     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
5538
5539     $force = 1 if !defined($force) && !$shutdown;
5540
5541     if ($migratedfrom){
5542         my $pid = check_running($vmid, $nocheck, $migratedfrom);
5543         kill 15, $pid if $pid;
5544         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
5545         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
5546         return;
5547     }
5548
5549     PVE::QemuConfig->lock_config($vmid, sub {
5550
5551         my $pid = check_running($vmid, $nocheck);
5552         return if !$pid;
5553
5554         my $conf;
5555         if (!$nocheck) {
5556             $conf = PVE::QemuConfig->load_config($vmid);
5557             PVE::QemuConfig->check_lock($conf) if !$skiplock;
5558             if (!defined($timeout) && $shutdown && $conf->{startup}) {
5559                 my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
5560                 $timeout = $opts->{down} if $opts->{down};
5561             }
5562             PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-stop');
5563         }
5564
5565         $timeout = 60 if !defined($timeout);
5566
5567         eval {
5568             if ($shutdown) {
5569                 if (defined($conf) && parse_guest_agent($conf)->{enabled}) {
5570                     vm_qmp_command($vmid, { execute => "guest-shutdown" }, $nocheck);
5571                 } else {
5572                     vm_qmp_command($vmid, { execute => "system_powerdown" }, $nocheck);
5573                 }
5574             } else {
5575                 vm_qmp_command($vmid, { execute => "quit" }, $nocheck);
5576             }
5577         };
5578         my $err = $@;
5579
5580         if (!$err) {
5581             my $count = 0;
5582             while (($count < $timeout) && check_running($vmid, $nocheck)) {
5583                 $count++;
5584                 sleep 1;
5585             }
5586
5587             if ($count >= $timeout) {
5588                 if ($force) {
5589                     warn "VM still running - terminating now with SIGTERM\n";
5590                     kill 15, $pid;
5591                 } else {
5592                     die "VM quit/powerdown failed - got timeout\n";
5593                 }
5594             } else {
5595                 vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5596                 return;
5597             }
5598         } else {
5599             if ($force) {
5600                 warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
5601                 kill 15, $pid;
5602             } else {
5603                 die "VM quit/powerdown failed\n";
5604             }
5605         }
5606
5607         # wait again
5608         $timeout = 10;
5609
5610         my $count = 0;
5611         while (($count < $timeout) && check_running($vmid, $nocheck)) {
5612             $count++;
5613             sleep 1;
5614         }
5615
5616         if ($count >= $timeout) {
5617             warn "VM still running - terminating now with SIGKILL\n";
5618             kill 9, $pid;
5619             sleep 1;
5620         }
5621
5622         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5623    });
5624 }
5625
5626 sub vm_suspend {
5627     my ($vmid, $skiplock) = @_;
5628
5629     PVE::QemuConfig->lock_config($vmid, sub {
5630
5631         my $conf = PVE::QemuConfig->load_config($vmid);
5632
5633         PVE::QemuConfig->check_lock($conf)
5634             if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
5635
5636         vm_mon_cmd($vmid, "stop");
5637     });
5638 }
5639
5640 sub vm_resume {
5641     my ($vmid, $skiplock, $nocheck) = @_;
5642
5643     PVE::QemuConfig->lock_config($vmid, sub {
5644
5645         my $res = vm_mon_cmd($vmid, 'query-status');
5646         my $resume_cmd = 'cont';
5647
5648         if ($res->{status} && $res->{status} eq 'suspended') {
5649             $resume_cmd = 'system_wakeup';
5650         }
5651
5652         if (!$nocheck) {
5653
5654             my $conf = PVE::QemuConfig->load_config($vmid);
5655
5656             PVE::QemuConfig->check_lock($conf)
5657                 if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
5658
5659             vm_mon_cmd($vmid, $resume_cmd);
5660
5661         } else {
5662             vm_mon_cmd_nocheck($vmid, $resume_cmd);
5663         }
5664     });
5665 }
5666
5667 sub vm_sendkey {
5668     my ($vmid, $skiplock, $key) = @_;
5669
5670     PVE::QemuConfig->lock_config($vmid, sub {
5671
5672         my $conf = PVE::QemuConfig->load_config($vmid);
5673
5674         # there is no qmp command, so we use the human monitor command
5675         vm_human_monitor_command($vmid, "sendkey $key");
5676     });
5677 }
5678
5679 sub vm_destroy {
5680     my ($storecfg, $vmid, $skiplock) = @_;
5681
5682     PVE::QemuConfig->lock_config($vmid, sub {
5683
5684         my $conf = PVE::QemuConfig->load_config($vmid);
5685
5686         if (!check_running($vmid)) {
5687             destroy_vm($storecfg, $vmid, undef, $skiplock);
5688         } else {
5689             die "VM $vmid is running - destroy failed\n";
5690         }
5691     });
5692 }
5693
5694 # vzdump restore implementaion
5695
5696 sub tar_archive_read_firstfile {
5697     my $archive = shift;
5698
5699     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
5700
5701     # try to detect archive type first
5702     my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
5703         die "unable to open file '$archive'\n";
5704     my $firstfile = <$fh>;
5705     kill 15, $pid;
5706     close $fh;
5707
5708     die "ERROR: archive contaions no data\n" if !$firstfile;
5709     chomp $firstfile;
5710
5711     return $firstfile;
5712 }
5713
5714 sub tar_restore_cleanup {
5715     my ($storecfg, $statfile) = @_;
5716
5717     print STDERR "starting cleanup\n";
5718
5719     if (my $fd = IO::File->new($statfile, "r")) {
5720         while (defined(my $line = <$fd>)) {
5721             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5722                 my $volid = $2;
5723                 eval {
5724                     if ($volid =~ m|^/|) {
5725                         unlink $volid || die 'unlink failed\n';
5726                     } else {
5727                         PVE::Storage::vdisk_free($storecfg, $volid);
5728                     }
5729                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
5730                 };
5731                 print STDERR "unable to cleanup '$volid' - $@" if $@;
5732             } else {
5733                 print STDERR "unable to parse line in statfile - $line";
5734             }
5735         }
5736         $fd->close();
5737     }
5738 }
5739
5740 sub restore_archive {
5741     my ($archive, $vmid, $user, $opts) = @_;
5742
5743     my $format = $opts->{format};
5744     my $comp;
5745
5746     if ($archive =~ m/\.tgz$/ || $archive =~ m/\.tar\.gz$/) {
5747         $format = 'tar' if !$format;
5748         $comp = 'gzip';
5749     } elsif ($archive =~ m/\.tar$/) {
5750         $format = 'tar' if !$format;
5751     } elsif ($archive =~ m/.tar.lzo$/) {
5752         $format = 'tar' if !$format;
5753         $comp = 'lzop';
5754     } elsif ($archive =~ m/\.vma$/) {
5755         $format = 'vma' if !$format;
5756     } elsif ($archive =~ m/\.vma\.gz$/) {
5757         $format = 'vma' if !$format;
5758         $comp = 'gzip';
5759     } elsif ($archive =~ m/\.vma\.lzo$/) {
5760         $format = 'vma' if !$format;
5761         $comp = 'lzop';
5762     } else {
5763         $format = 'vma' if !$format; # default
5764     }
5765
5766     # try to detect archive format
5767     if ($format eq 'tar') {
5768         return restore_tar_archive($archive, $vmid, $user, $opts);
5769     } else {
5770         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
5771     }
5772 }
5773
5774 sub restore_update_config_line {
5775     my ($outfd, $cookie, $vmid, $map, $line, $unique) = @_;
5776
5777     return if $line =~ m/^\#qmdump\#/;
5778     return if $line =~ m/^\#vzdump\#/;
5779     return if $line =~ m/^lock:/;
5780     return if $line =~ m/^unused\d+:/;
5781     return if $line =~ m/^parent:/;
5782     return if $line =~ m/^template:/; # restored VM is never a template
5783
5784     my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
5785     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
5786         # try to convert old 1.X settings
5787         my ($id, $ind, $ethcfg) = ($1, $2, $3);
5788         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
5789             my ($model, $macaddr) = split(/\=/, $devconfig);
5790             $macaddr = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if !$macaddr || $unique;
5791             my $net = {
5792                 model => $model,
5793                 bridge => "vmbr$ind",
5794                 macaddr => $macaddr,
5795             };
5796             my $netstr = print_net($net);
5797
5798             print $outfd "net$cookie->{netcount}: $netstr\n";
5799             $cookie->{netcount}++;
5800         }
5801     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
5802         my ($id, $netstr) = ($1, $2);
5803         my $net = parse_net($netstr);
5804         $net->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if $net->{macaddr};
5805         $netstr = print_net($net);
5806         print $outfd "$id: $netstr\n";
5807     } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk)\d+):\s*(\S+)\s*$/) {
5808         my $virtdev = $1;
5809         my $value = $3;
5810         my $di = parse_drive($virtdev, $value);
5811         if (defined($di->{backup}) && !$di->{backup}) {
5812             print $outfd "#$line";
5813         } elsif ($map->{$virtdev}) {
5814             delete $di->{format}; # format can change on restore
5815             $di->{file} = $map->{$virtdev};
5816             $value = print_drive($vmid, $di);
5817             print $outfd "$virtdev: $value\n";
5818         } else {
5819             print $outfd $line;
5820         }
5821     } elsif (($line =~ m/^vmgenid: (.*)/)) {
5822         my $vmgenid = $1;
5823         if ($vmgenid ne '0') {
5824             # always generate a new vmgenid if there was a valid one setup
5825             $vmgenid = generate_uuid();
5826         }
5827         print $outfd "vmgenid: $vmgenid\n";
5828     } elsif (($line =~ m/^(smbios1: )(.*)/) && $unique) {
5829         my ($uuid, $uuid_str);
5830         UUID::generate($uuid);
5831         UUID::unparse($uuid, $uuid_str);
5832         my $smbios1 = parse_smbios1($2);
5833         $smbios1->{uuid} = $uuid_str;
5834         print $outfd $1.print_smbios1($smbios1)."\n";
5835     } else {
5836         print $outfd $line;
5837     }
5838 }
5839
5840 sub scan_volids {
5841     my ($cfg, $vmid) = @_;
5842
5843     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid);
5844
5845     my $volid_hash = {};
5846     foreach my $storeid (keys %$info) {
5847         foreach my $item (@{$info->{$storeid}}) {
5848             next if !($item->{volid} && $item->{size});
5849             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
5850             $volid_hash->{$item->{volid}} = $item;
5851         }
5852     }
5853
5854     return $volid_hash;
5855 }
5856
5857 sub is_volume_in_use {
5858     my ($storecfg, $conf, $skip_drive, $volid) = @_;
5859
5860     my $path = PVE::Storage::path($storecfg, $volid);
5861
5862     my $scan_config = sub {
5863         my ($cref, $snapname) = @_;
5864
5865         foreach my $key (keys %$cref) {
5866             my $value = $cref->{$key};
5867             if (is_valid_drivename($key)) {
5868                 next if $skip_drive && $key eq $skip_drive;
5869                 my $drive = parse_drive($key, $value);
5870                 next if !$drive || !$drive->{file} || drive_is_cdrom($drive);
5871                 return 1 if $volid eq $drive->{file};
5872                 if ($drive->{file} =~ m!^/!) {
5873                     return 1 if $drive->{file} eq $path;
5874                 } else {
5875                     my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file}, 1);
5876                     next if !$storeid;
5877                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid, 1);
5878                     next if !$scfg;
5879                     return 1 if $path eq PVE::Storage::path($storecfg, $drive->{file}, $snapname);
5880                 }
5881             }
5882         }
5883
5884         return 0;
5885     };
5886
5887     return 1 if &$scan_config($conf);
5888
5889     undef $skip_drive;
5890
5891     foreach my $snapname (keys %{$conf->{snapshots}}) {
5892         return 1 if &$scan_config($conf->{snapshots}->{$snapname}, $snapname);
5893     }
5894
5895     return 0;
5896 }
5897
5898 sub update_disksize {
5899     my ($vmid, $conf, $volid_hash) = @_;
5900
5901     my $changes;
5902     my $prefix = "VM $vmid:";
5903
5904     # used and unused disks
5905     my $referenced = {};
5906
5907     # Note: it is allowed to define multiple storages with same path (alias), so
5908     # we need to check both 'volid' and real 'path' (two different volid can point
5909     # to the same path).
5910
5911     my $referencedpath = {};
5912
5913     # update size info
5914     foreach my $opt (keys %$conf) {
5915         if (is_valid_drivename($opt)) {
5916             my $drive = parse_drive($opt, $conf->{$opt});
5917             my $volid = $drive->{file};
5918             next if !$volid;
5919
5920             $referenced->{$volid} = 1;
5921             if ($volid_hash->{$volid} &&
5922                 (my $path = $volid_hash->{$volid}->{path})) {
5923                 $referencedpath->{$path} = 1;
5924             }
5925
5926             next if drive_is_cdrom($drive);
5927             next if !$volid_hash->{$volid};
5928
5929             $drive->{size} = $volid_hash->{$volid}->{size};
5930             my $new = print_drive($vmid, $drive);
5931             if ($new ne $conf->{$opt}) {
5932                 $changes = 1;
5933                 $conf->{$opt} = $new;
5934                 print "$prefix update disk '$opt' information.\n";
5935             }
5936         }
5937     }
5938
5939     # remove 'unusedX' entry if volume is used
5940     foreach my $opt (keys %$conf) {
5941         next if $opt !~ m/^unused\d+$/;
5942         my $volid = $conf->{$opt};
5943         my $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
5944         if ($referenced->{$volid} || ($path && $referencedpath->{$path})) {
5945             print "$prefix remove entry '$opt', its volume '$volid' is in use.\n";
5946             $changes = 1;
5947             delete $conf->{$opt};
5948         }
5949
5950         $referenced->{$volid} = 1;
5951         $referencedpath->{$path} = 1 if $path;
5952     }
5953
5954     foreach my $volid (sort keys %$volid_hash) {
5955         next if $volid =~ m/vm-$vmid-state-/;
5956         next if $referenced->{$volid};
5957         my $path = $volid_hash->{$volid}->{path};
5958         next if !$path; # just to be sure
5959         next if $referencedpath->{$path};
5960         $changes = 1;
5961         my $key = PVE::QemuConfig->add_unused_volume($conf, $volid);
5962         print "$prefix add unreferenced volume '$volid' as '$key' to config.\n";
5963         $referencedpath->{$path} = 1; # avoid to add more than once (aliases)
5964     }
5965
5966     return $changes;
5967 }
5968
5969 sub rescan {
5970     my ($vmid, $nolock, $dryrun) = @_;
5971
5972     my $cfg = PVE::Storage::config();
5973
5974     # FIXME: Remove once our RBD plugin can handle CT and VM on a single storage
5975     # see: https://pve.proxmox.com/pipermail/pve-devel/2018-July/032900.html
5976     foreach my $stor (keys %{$cfg->{ids}}) {
5977         delete($cfg->{ids}->{$stor}) if ! $cfg->{ids}->{$stor}->{content}->{images};
5978     }
5979
5980     print "rescan volumes...\n";
5981     my $volid_hash = scan_volids($cfg, $vmid);
5982
5983     my $updatefn =  sub {
5984         my ($vmid) = @_;
5985
5986         my $conf = PVE::QemuConfig->load_config($vmid);
5987
5988         PVE::QemuConfig->check_lock($conf);
5989
5990         my $vm_volids = {};
5991         foreach my $volid (keys %$volid_hash) {
5992             my $info = $volid_hash->{$volid};
5993             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
5994         }
5995
5996         my $changes = update_disksize($vmid, $conf, $vm_volids);
5997
5998         PVE::QemuConfig->write_config($vmid, $conf) if $changes && !$dryrun;
5999     };
6000
6001     if (defined($vmid)) {
6002         if ($nolock) {
6003             &$updatefn($vmid);
6004         } else {
6005             PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6006         }
6007     } else {
6008         my $vmlist = config_list();
6009         foreach my $vmid (keys %$vmlist) {
6010             if ($nolock) {
6011                 &$updatefn($vmid);
6012             } else {
6013                 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6014             }
6015         }
6016     }
6017 }
6018
6019 sub restore_vma_archive {
6020     my ($archive, $vmid, $user, $opts, $comp) = @_;
6021
6022     my $readfrom = $archive;
6023
6024     my $cfg = PVE::Storage::config();
6025     my $commands = [];
6026     my $bwlimit = $opts->{bwlimit};
6027
6028     my $dbg_cmdstring = '';
6029     my $add_pipe = sub {
6030         my ($cmd) = @_;
6031         push @$commands, $cmd;
6032         $dbg_cmdstring .= ' | ' if length($dbg_cmdstring);
6033         $dbg_cmdstring .= PVE::Tools::cmd2string($cmd);
6034         $readfrom = '-';
6035     };
6036
6037     my $input = undef;
6038     if ($archive eq '-') {
6039         $input = '<&STDIN';
6040     } else {
6041         # If we use a backup from a PVE defined storage we also consider that
6042         # storage's rate limit:
6043         my (undef, $volid) = PVE::Storage::path_to_volume_id($cfg, $archive);
6044         if (defined($volid)) {
6045             my ($sid, undef) = PVE::Storage::parse_volume_id($volid);
6046             my $readlimit = PVE::Storage::get_bandwidth_limit('restore', [$sid], $bwlimit);
6047             if ($readlimit) {
6048                 print STDERR "applying read rate limit: $readlimit\n";
6049                 my $cstream = ['cstream', '-t', $readlimit*1024, '--', $readfrom];
6050                 $add_pipe->($cstream);
6051             }
6052         }
6053     }
6054
6055     if ($comp) {
6056         my $cmd;
6057         if ($comp eq 'gzip') {
6058             $cmd = ['zcat', $readfrom];
6059         } elsif ($comp eq 'lzop') {
6060             $cmd = ['lzop', '-d', '-c', $readfrom];
6061         } else {
6062             die "unknown compression method '$comp'\n";
6063         }
6064         $add_pipe->($cmd);
6065     }
6066
6067     my $tmpdir = "/var/tmp/vzdumptmp$$";
6068     rmtree $tmpdir;
6069
6070     # disable interrupts (always do cleanups)
6071     local $SIG{INT} =
6072         local $SIG{TERM} =
6073         local $SIG{QUIT} =
6074         local $SIG{HUP} = sub { warn "got interrupt - ignored\n"; };
6075
6076     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
6077     POSIX::mkfifo($mapfifo, 0600);
6078     my $fifofh;
6079
6080     my $openfifo = sub {
6081         open($fifofh, '>', $mapfifo) || die $!;
6082     };
6083
6084     $add_pipe->(['vma', 'extract', '-v', '-r', $mapfifo, $readfrom, $tmpdir]);
6085
6086     my $oldtimeout;
6087     my $timeout = 5;
6088
6089     my $devinfo = {};
6090
6091     my $rpcenv = PVE::RPCEnvironment::get();
6092
6093     my $conffile = PVE::QemuConfig->config_file($vmid);
6094     my $tmpfn = "$conffile.$$.tmp";
6095
6096     # Note: $oldconf is undef if VM does not exists
6097     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
6098     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
6099
6100     my %storage_limits;
6101
6102     my $print_devmap = sub {
6103         my $virtdev_hash = {};
6104
6105         my $cfgfn = "$tmpdir/qemu-server.conf";
6106
6107         # we can read the config - that is already extracted
6108         my $fh = IO::File->new($cfgfn, "r") ||
6109             "unable to read qemu-server.conf - $!\n";
6110
6111         my $fwcfgfn = "$tmpdir/qemu-server.fw";
6112         if (-f $fwcfgfn) {
6113             my $pve_firewall_dir = '/etc/pve/firewall';
6114             mkdir $pve_firewall_dir; # make sure the dir exists
6115             PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
6116         }
6117
6118         while (defined(my $line = <$fh>)) {
6119             if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
6120                 my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
6121                 die "archive does not contain data for drive '$virtdev'\n"
6122                     if !$devinfo->{$devname};
6123                 if (defined($opts->{storage})) {
6124                     $storeid = $opts->{storage} || 'local';
6125                 } elsif (!$storeid) {
6126                     $storeid = 'local';
6127                 }
6128                 $format = 'raw' if !$format;
6129                 $devinfo->{$devname}->{devname} = $devname;
6130                 $devinfo->{$devname}->{virtdev} = $virtdev;
6131                 $devinfo->{$devname}->{format} = $format;
6132                 $devinfo->{$devname}->{storeid} = $storeid;
6133
6134                 # check permission on storage
6135                 my $pool = $opts->{pool}; # todo: do we need that?
6136                 if ($user ne 'root@pam') {
6137                     $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
6138                 }
6139
6140                 $storage_limits{$storeid} = $bwlimit;
6141
6142                 $virtdev_hash->{$virtdev} = $devinfo->{$devname};
6143             }
6144         }
6145
6146         foreach my $key (keys %storage_limits) {
6147             my $limit = PVE::Storage::get_bandwidth_limit('restore', [$key], $bwlimit);
6148             next if !$limit;
6149             print STDERR "rate limit for storage $key: $limit KiB/s\n";
6150             $storage_limits{$key} = $limit * 1024;
6151         }
6152
6153         foreach my $devname (keys %$devinfo) {
6154             die "found no device mapping information for device '$devname'\n"
6155                 if !$devinfo->{$devname}->{virtdev};
6156         }
6157
6158         # create empty/temp config
6159         if ($oldconf) {
6160             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
6161             foreach_drive($oldconf, sub {
6162                 my ($ds, $drive) = @_;
6163
6164                 return if drive_is_cdrom($drive);
6165
6166                 my $volid = $drive->{file};
6167
6168                 return if !$volid || $volid =~ m|^/|;
6169
6170                 my ($path, $owner) = PVE::Storage::path($cfg, $volid);
6171                 return if !$path || !$owner || ($owner != $vmid);
6172
6173                 # Note: only delete disk we want to restore
6174                 # other volumes will become unused
6175                 if ($virtdev_hash->{$ds}) {
6176                     eval { PVE::Storage::vdisk_free($cfg, $volid); };
6177                     if (my $err = $@) {
6178                         warn $err;
6179                     }
6180                 }
6181             });
6182
6183             # delete vmstate files
6184             # since after the restore we have no snapshots anymore
6185             foreach my $snapname (keys %{$oldconf->{snapshots}}) {
6186                 my $snap = $oldconf->{snapshots}->{$snapname};
6187                 if ($snap->{vmstate}) {
6188                     eval { PVE::Storage::vdisk_free($cfg, $snap->{vmstate}); };
6189                     if (my $err = $@) {
6190                         warn $err;
6191                     }
6192                 }
6193             }
6194         }
6195
6196         my $map = {};
6197         foreach my $virtdev (sort keys %$virtdev_hash) {
6198             my $d = $virtdev_hash->{$virtdev};
6199             my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
6200             my $storeid = $d->{storeid};
6201             my $scfg = PVE::Storage::storage_config($cfg, $storeid);
6202
6203             my $map_opts = '';
6204             if (my $limit = $storage_limits{$storeid}) {
6205                 $map_opts .= "throttling.bps=$limit:throttling.group=$storeid:";
6206             }
6207
6208             # test if requested format is supported
6209             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($cfg, $storeid);
6210             my $supported = grep { $_ eq $d->{format} } @$validFormats;
6211             $d->{format} = $defFormat if !$supported;
6212
6213             my $volid = PVE::Storage::vdisk_alloc($cfg, $storeid, $vmid,
6214                                                   $d->{format}, undef, $alloc_size);
6215             print STDERR "new volume ID is '$volid'\n";
6216             $d->{volid} = $volid;
6217             my $path = PVE::Storage::path($cfg, $volid);
6218
6219             PVE::Storage::activate_volumes($cfg,[$volid]);
6220
6221             my $write_zeros = 1;
6222             if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
6223                 $write_zeros = 0;
6224             }
6225
6226             print $fifofh "${map_opts}format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
6227
6228             print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
6229             $map->{$virtdev} = $volid;
6230         }
6231
6232         $fh->seek(0, 0) || die "seek failed - $!\n";
6233
6234         my $outfd = new IO::File ($tmpfn, "w") ||
6235             die "unable to write config for VM $vmid\n";
6236
6237         my $cookie = { netcount => 0 };
6238         while (defined(my $line = <$fh>)) {
6239             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
6240         }
6241
6242         $fh->close();
6243         $outfd->close();
6244     };
6245
6246     eval {
6247         # enable interrupts
6248         local $SIG{INT} =
6249             local $SIG{TERM} =
6250             local $SIG{QUIT} =
6251             local $SIG{HUP} =
6252             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6253         local $SIG{ALRM} = sub { die "got timeout\n"; };
6254
6255         $oldtimeout = alarm($timeout);
6256
6257         my $parser = sub {
6258             my $line = shift;
6259
6260             print "$line\n";
6261
6262             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
6263                 my ($dev_id, $size, $devname) = ($1, $2, $3);
6264                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
6265             } elsif ($line =~ m/^CTIME: /) {
6266                 # we correctly received the vma config, so we can disable
6267                 # the timeout now for disk allocation (set to 10 minutes, so
6268                 # that we always timeout if something goes wrong)
6269                 alarm(600);
6270                 &$print_devmap();
6271                 print $fifofh "done\n";
6272                 my $tmp = $oldtimeout || 0;
6273                 $oldtimeout = undef;
6274                 alarm($tmp);
6275                 close($fifofh);
6276             }
6277         };
6278
6279         print "restore vma archive: $dbg_cmdstring\n";
6280         run_command($commands, input => $input, outfunc => $parser, afterfork => $openfifo);
6281     };
6282     my $err = $@;
6283
6284     alarm($oldtimeout) if $oldtimeout;
6285
6286     my $vollist = [];
6287     foreach my $devname (keys %$devinfo) {
6288         my $volid = $devinfo->{$devname}->{volid};
6289         push @$vollist, $volid if $volid;
6290     }
6291
6292     PVE::Storage::deactivate_volumes($cfg, $vollist);
6293
6294     unlink $mapfifo;
6295
6296     if ($err) {
6297         rmtree $tmpdir;
6298         unlink $tmpfn;
6299
6300         foreach my $devname (keys %$devinfo) {
6301             my $volid = $devinfo->{$devname}->{volid};
6302             next if !$volid;
6303             eval {
6304                 if ($volid =~ m|^/|) {
6305                     unlink $volid || die 'unlink failed\n';
6306                 } else {
6307                     PVE::Storage::vdisk_free($cfg, $volid);
6308                 }
6309                 print STDERR "temporary volume '$volid' sucessfuly removed\n";
6310             };
6311             print STDERR "unable to cleanup '$volid' - $@" if $@;
6312         }
6313         die $err;
6314     }
6315
6316     rmtree $tmpdir;
6317
6318     rename($tmpfn, $conffile) ||
6319         die "unable to commit configuration file '$conffile'\n";
6320
6321     PVE::Cluster::cfs_update(); # make sure we read new file
6322
6323     eval { rescan($vmid, 1); };
6324     warn $@ if $@;
6325 }
6326
6327 sub restore_tar_archive {
6328     my ($archive, $vmid, $user, $opts) = @_;
6329
6330     if ($archive ne '-') {
6331         my $firstfile = tar_archive_read_firstfile($archive);
6332         die "ERROR: file '$archive' dos not lock like a QemuServer vzdump backup\n"
6333             if $firstfile ne 'qemu-server.conf';
6334     }
6335
6336     my $storecfg = PVE::Storage::config();
6337
6338     # destroy existing data - keep empty config
6339     my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
6340     destroy_vm($storecfg, $vmid, 1) if -f $vmcfgfn;
6341
6342     my $tocmd = "/usr/lib/qemu-server/qmextract";
6343
6344     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
6345     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
6346     $tocmd .= ' --prealloc' if $opts->{prealloc};
6347     $tocmd .= ' --info' if $opts->{info};
6348
6349     # tar option "xf" does not autodetect compression when read from STDIN,
6350     # so we pipe to zcat
6351     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
6352         PVE::Tools::shellquote("--to-command=$tocmd");
6353
6354     my $tmpdir = "/var/tmp/vzdumptmp$$";
6355     mkpath $tmpdir;
6356
6357     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
6358     local $ENV{VZDUMP_VMID} = $vmid;
6359     local $ENV{VZDUMP_USER} = $user;
6360
6361     my $conffile = PVE::QemuConfig->config_file($vmid);
6362     my $tmpfn = "$conffile.$$.tmp";
6363
6364     # disable interrupts (always do cleanups)
6365     local $SIG{INT} =
6366         local $SIG{TERM} =
6367         local $SIG{QUIT} =
6368         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
6369
6370     eval {
6371         # enable interrupts
6372         local $SIG{INT} =
6373             local $SIG{TERM} =
6374             local $SIG{QUIT} =
6375             local $SIG{HUP} =
6376             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6377
6378         if ($archive eq '-') {
6379             print "extracting archive from STDIN\n";
6380             run_command($cmd, input => "<&STDIN");
6381         } else {
6382             print "extracting archive '$archive'\n";
6383             run_command($cmd);
6384         }
6385
6386         return if $opts->{info};
6387
6388         # read new mapping
6389         my $map = {};
6390         my $statfile = "$tmpdir/qmrestore.stat";
6391         if (my $fd = IO::File->new($statfile, "r")) {
6392             while (defined (my $line = <$fd>)) {
6393                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
6394                     $map->{$1} = $2 if $1;
6395                 } else {
6396                     print STDERR "unable to parse line in statfile - $line\n";
6397                 }
6398             }
6399             $fd->close();
6400         }
6401
6402         my $confsrc = "$tmpdir/qemu-server.conf";
6403
6404         my $srcfd = new IO::File($confsrc, "r") ||
6405             die "unable to open file '$confsrc'\n";
6406
6407         my $outfd = new IO::File ($tmpfn, "w") ||
6408             die "unable to write config for VM $vmid\n";
6409
6410         my $cookie = { netcount => 0 };
6411         while (defined (my $line = <$srcfd>)) {
6412             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
6413         }
6414
6415         $srcfd->close();
6416         $outfd->close();
6417     };
6418     my $err = $@;
6419
6420     if ($err) {
6421
6422         unlink $tmpfn;
6423
6424         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
6425
6426         die $err;
6427     }
6428
6429     rmtree $tmpdir;
6430
6431     rename $tmpfn, $conffile ||
6432         die "unable to commit configuration file '$conffile'\n";
6433
6434     PVE::Cluster::cfs_update(); # make sure we read new file
6435
6436     eval { rescan($vmid, 1); };
6437     warn $@ if $@;
6438 };
6439
6440 sub foreach_storage_used_by_vm {
6441     my ($conf, $func) = @_;
6442
6443     my $sidhash = {};
6444
6445     foreach_drive($conf, sub {
6446         my ($ds, $drive) = @_;
6447         return if drive_is_cdrom($drive);
6448
6449         my $volid = $drive->{file};
6450
6451         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
6452         $sidhash->{$sid} = $sid if $sid;
6453     });
6454
6455     foreach my $sid (sort keys %$sidhash) {
6456         &$func($sid);
6457     }
6458 }
6459
6460 sub do_snapshots_with_qemu {
6461     my ($storecfg, $volid) = @_;
6462
6463     my $storage_name = PVE::Storage::parse_volume_id($volid);
6464
6465     if ($qemu_snap_storage->{$storecfg->{ids}->{$storage_name}->{type}}
6466         && !$storecfg->{ids}->{$storage_name}->{krbd}){
6467         return 1;
6468     }
6469
6470     if ($volid =~ m/\.(qcow2|qed)$/){
6471         return 1;
6472     }
6473
6474     return undef;
6475 }
6476
6477 sub qga_check_running {
6478     my ($vmid, $nowarn) = @_;
6479
6480     eval { vm_mon_cmd($vmid, "guest-ping", timeout => 3); };
6481     if ($@) {
6482         warn "Qemu Guest Agent is not running - $@" if !$nowarn;
6483         return 0;
6484     }
6485     return 1;
6486 }
6487
6488 sub template_create {
6489     my ($vmid, $conf, $disk) = @_;
6490
6491     my $storecfg = PVE::Storage::config();
6492
6493     foreach_drive($conf, sub {
6494         my ($ds, $drive) = @_;
6495
6496         return if drive_is_cdrom($drive);
6497         return if $disk && $ds ne $disk;
6498
6499         my $volid = $drive->{file};
6500         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
6501
6502         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
6503         $drive->{file} = $voliddst;
6504         $conf->{$ds} = print_drive($vmid, $drive);
6505         PVE::QemuConfig->write_config($vmid, $conf);
6506     });
6507 }
6508
6509 sub qemu_img_convert {
6510     my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized) = @_;
6511
6512     my $storecfg = PVE::Storage::config();
6513     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
6514     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
6515
6516     if ($src_storeid && $dst_storeid) {
6517
6518         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
6519
6520         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
6521         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6522
6523         my $src_format = qemu_img_format($src_scfg, $src_volname);
6524         my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
6525
6526         my $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
6527         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6528
6529         my $cmd = [];
6530         push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
6531         push @$cmd, '-l', "snapshot.name=$snapname" if($snapname && $src_format eq "qcow2");
6532         push @$cmd, '-t', 'none' if $dst_scfg->{type} eq 'zfspool';
6533         push @$cmd, '-T', 'none' if $src_scfg->{type} eq 'zfspool';
6534         push @$cmd, '-f', $src_format, '-O', $dst_format, $src_path;
6535         if ($is_zero_initialized) {
6536             push @$cmd, "zeroinit:$dst_path";
6537         } else {
6538             push @$cmd, $dst_path;
6539         }
6540
6541         my $parser = sub {
6542             my $line = shift;
6543             if($line =~ m/\((\S+)\/100\%\)/){
6544                 my $percent = $1;
6545                 my $transferred = int($size * $percent / 100);
6546                 my $remaining = $size - $transferred;
6547
6548                 print "transferred: $transferred bytes remaining: $remaining bytes total: $size bytes progression: $percent %\n";
6549             }
6550
6551         };
6552
6553         eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
6554         my $err = $@;
6555         die "copy failed: $err" if $err;
6556     }
6557 }
6558
6559 sub qemu_img_format {
6560     my ($scfg, $volname) = @_;
6561
6562     if ($scfg->{path} && $volname =~ m/\.($QEMU_FORMAT_RE)$/) {
6563         return $1;
6564     } else {
6565         return "raw";
6566     }
6567 }
6568
6569 sub qemu_drive_mirror {
6570     my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $skipcomplete, $qga) = @_;
6571
6572     $jobs = {} if !$jobs;
6573
6574     my $qemu_target;
6575     my $format;
6576     $jobs->{"drive-$drive"} = {};
6577
6578     if ($dst_volid =~ /^nbd:/) {
6579         $qemu_target = $dst_volid;
6580         $format = "nbd";
6581     } else {
6582         my $storecfg = PVE::Storage::config();
6583         my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
6584
6585         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6586
6587         $format = qemu_img_format($dst_scfg, $dst_volname);
6588
6589         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6590
6591         $qemu_target = $is_zero_initialized ? "zeroinit:$dst_path" : $dst_path;
6592     }
6593
6594     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $qemu_target };
6595     $opts->{format} = $format if $format;
6596
6597     print "drive mirror is starting for drive-$drive\n";
6598
6599     eval { vm_mon_cmd($vmid, "drive-mirror", %$opts); }; #if a job already run for this device,it's throw an error
6600
6601     if (my $err = $@) {
6602         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6603         die "mirroring error: $err";
6604     }
6605
6606     qemu_drive_mirror_monitor ($vmid, $vmiddst, $jobs, $skipcomplete, $qga);
6607 }
6608
6609 sub qemu_drive_mirror_monitor {
6610     my ($vmid, $vmiddst, $jobs, $skipcomplete, $qga) = @_;
6611
6612     eval {
6613         my $err_complete = 0;
6614
6615         while (1) {
6616             die "storage migration timed out\n" if $err_complete > 300;
6617
6618             my $stats = vm_mon_cmd($vmid, "query-block-jobs");
6619
6620             my $running_mirror_jobs = {};
6621             foreach my $stat (@$stats) {
6622                 next if $stat->{type} ne 'mirror';
6623                 $running_mirror_jobs->{$stat->{device}} = $stat;
6624             }
6625
6626             my $readycounter = 0;
6627
6628             foreach my $job (keys %$jobs) {
6629
6630                 if(defined($jobs->{$job}->{complete}) && !defined($running_mirror_jobs->{$job})) {
6631                     print "$job : finished\n";
6632                     delete $jobs->{$job};
6633                     next;
6634                 }
6635
6636                 die "$job: mirroring has been cancelled\n" if !defined($running_mirror_jobs->{$job});
6637
6638                 my $busy = $running_mirror_jobs->{$job}->{busy};
6639                 my $ready = $running_mirror_jobs->{$job}->{ready};
6640                 if (my $total = $running_mirror_jobs->{$job}->{len}) {
6641                     my $transferred = $running_mirror_jobs->{$job}->{offset} || 0;
6642                     my $remaining = $total - $transferred;
6643                     my $percent = sprintf "%.2f", ($transferred * 100 / $total);
6644
6645                     print "$job: transferred: $transferred bytes remaining: $remaining bytes total: $total bytes progression: $percent % busy: $busy ready: $ready \n";
6646                 }
6647
6648                 $readycounter++ if $running_mirror_jobs->{$job}->{ready};
6649             }
6650
6651             last if scalar(keys %$jobs) == 0;
6652
6653             if ($readycounter == scalar(keys %$jobs)) {
6654                 print "all mirroring jobs are ready \n";
6655                 last if $skipcomplete; #do the complete later
6656
6657                 if ($vmiddst && $vmiddst != $vmid) {
6658                     my $agent_running = $qga && qga_check_running($vmid);
6659                     if ($agent_running) {
6660                         print "freeze filesystem\n";
6661                         eval { PVE::QemuServer::vm_mon_cmd($vmid, "guest-fsfreeze-freeze"); };
6662                     } else {
6663                         print "suspend vm\n";
6664                         eval { PVE::QemuServer::vm_suspend($vmid, 1); };
6665                     }
6666
6667                     # if we clone a disk for a new target vm, we don't switch the disk
6668                     PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs);
6669
6670                     if ($agent_running) {
6671                         print "unfreeze filesystem\n";
6672                         eval { PVE::QemuServer::vm_mon_cmd($vmid, "guest-fsfreeze-thaw"); };
6673                     } else {
6674                         print "resume vm\n";
6675                         eval {  PVE::QemuServer::vm_resume($vmid, 1, 1); };
6676                     }
6677
6678                     last;
6679                 } else {
6680
6681                     foreach my $job (keys %$jobs) {
6682                         # try to switch the disk if source and destination are on the same guest
6683                         print "$job: Completing block job...\n";
6684
6685                         eval { vm_mon_cmd($vmid, "block-job-complete", device => $job) };
6686                         if ($@ =~ m/cannot be completed/) {
6687                             print "$job: Block job cannot be completed, try again.\n";
6688                             $err_complete++;
6689                         }else {
6690                             print "$job: Completed successfully.\n";
6691                             $jobs->{$job}->{complete} = 1;
6692                         }
6693                     }
6694                 }
6695             }
6696             sleep 1;
6697         }
6698     };
6699     my $err = $@;
6700
6701     if ($err) {
6702         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6703         die "mirroring error: $err";
6704     }
6705
6706 }
6707
6708 sub qemu_blockjobs_cancel {
6709     my ($vmid, $jobs) = @_;
6710
6711     foreach my $job (keys %$jobs) {
6712         print "$job: Cancelling block job\n";
6713         eval { vm_mon_cmd($vmid, "block-job-cancel", device => $job); };
6714         $jobs->{$job}->{cancel} = 1;
6715     }
6716
6717     while (1) {
6718         my $stats = vm_mon_cmd($vmid, "query-block-jobs");
6719
6720         my $running_jobs = {};
6721         foreach my $stat (@$stats) {
6722             $running_jobs->{$stat->{device}} = $stat;
6723         }
6724
6725         foreach my $job (keys %$jobs) {
6726
6727             if (defined($jobs->{$job}->{cancel}) && !defined($running_jobs->{$job})) {
6728                 print "$job: Done.\n";
6729                 delete $jobs->{$job};
6730             }
6731         }
6732
6733         last if scalar(keys %$jobs) == 0;
6734
6735         sleep 1;
6736     }
6737 }
6738
6739 sub clone_disk {
6740     my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
6741         $newvmid, $storage, $format, $full, $newvollist, $jobs, $skipcomplete, $qga) = @_;
6742
6743     my $newvolid;
6744
6745     if (!$full) {
6746         print "create linked clone of drive $drivename ($drive->{file})\n";
6747         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
6748         push @$newvollist, $newvolid;
6749     } else {
6750
6751         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6752         $storeid = $storage if $storage;
6753
6754         my $dst_format = resolve_dst_disk_format($storecfg, $storeid, $volname, $format);
6755         my ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 3);
6756
6757         print "create full clone of drive $drivename ($drive->{file})\n";
6758         my $name = undef;
6759         if (drive_is_cloudinit($drive)) {
6760             $name = "vm-$newvmid-cloudinit";
6761             $snapname = undef;
6762             # cloudinit only supports raw and qcow2 atm:
6763             if ($dst_format eq 'qcow2') {
6764                 $name .= '.qcow2';
6765             } elsif ($dst_format ne 'raw') {
6766                 die "clone: unhandled format for cloudinit image\n";
6767             }
6768         }
6769         $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $newvmid, $dst_format, $name, ($size/1024));
6770         push @$newvollist, $newvolid;
6771
6772         PVE::Storage::activate_volumes($storecfg, [$newvolid]);
6773
6774         my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
6775         if (!$running || $snapname) {
6776             qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit);
6777         } else {
6778
6779             my $kvmver = get_running_qemu_version ($vmid);
6780             if (!qemu_machine_feature_enabled (undef, $kvmver, 2, 7)) {
6781                 die "drive-mirror with iothread requires qemu version 2.7 or higher\n"
6782                     if $drive->{iothread};
6783             }
6784
6785             qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid, $sparseinit, $jobs, $skipcomplete, $qga);
6786         }
6787     }
6788
6789     my ($size) = PVE::Storage::volume_size_info($storecfg, $newvolid, 3);
6790
6791     my $disk = $drive;
6792     $disk->{format} = undef;
6793     $disk->{file} = $newvolid;
6794     $disk->{size} = $size;
6795
6796     return $disk;
6797 }
6798
6799 # this only works if VM is running
6800 sub get_current_qemu_machine {
6801     my ($vmid) = @_;
6802
6803     my $cmd = { execute => 'query-machines', arguments => {} };
6804     my $res = vm_qmp_command($vmid, $cmd);
6805
6806     my ($current, $default);
6807     foreach my $e (@$res) {
6808         $default = $e->{name} if $e->{'is-default'};
6809         $current = $e->{name} if $e->{'is-current'};
6810     }
6811
6812     # fallback to the default machine if current is not supported by qemu
6813     return $current || $default || 'pc';
6814 }
6815
6816 sub get_running_qemu_version {
6817     my ($vmid) = @_;
6818     my $cmd = { execute => 'query-version', arguments => {} };
6819     my $res = vm_qmp_command($vmid, $cmd);
6820     return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
6821 }
6822
6823 sub qemu_machine_feature_enabled {
6824     my ($machine, $kvmver, $version_major, $version_minor) = @_;
6825
6826     my $current_major;
6827     my $current_minor;
6828
6829     if ($machine && $machine =~ m/^((?:pc(-i440fx|-q35)?|virt)-(\d+)\.(\d+))/) {
6830
6831         $current_major = $3;
6832         $current_minor = $4;
6833
6834     } elsif ($kvmver =~ m/^(\d+)\.(\d+)/) {
6835
6836         $current_major = $1;
6837         $current_minor = $2;
6838     }
6839
6840     return 1 if $current_major > $version_major ||
6841                 ($current_major == $version_major &&
6842                  $current_minor >= $version_minor);
6843 }
6844
6845 sub qemu_machine_pxe {
6846     my ($vmid, $conf, $machine) = @_;
6847
6848     $machine =  PVE::QemuServer::get_current_qemu_machine($vmid) if !$machine;
6849
6850     if ($conf->{machine} && $conf->{machine} =~ m/\.pxe$/) {
6851         $machine .= '.pxe';
6852     }
6853
6854     return $machine;
6855 }
6856
6857 sub qemu_use_old_bios_files {
6858     my ($machine_type) = @_;
6859
6860     return if !$machine_type;
6861
6862     my $use_old_bios_files = undef;
6863
6864     if ($machine_type =~ m/^(\S+)\.pxe$/) {
6865         $machine_type = $1;
6866         $use_old_bios_files = 1;
6867     } else {
6868         my $kvmver = kvm_user_version();
6869         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
6870         # load new efi bios files on migration. So this hack is required to allow
6871         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
6872         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
6873         $use_old_bios_files = !qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 4);
6874     }
6875
6876     return ($use_old_bios_files, $machine_type);
6877 }
6878
6879 sub create_efidisk($$$$$) {
6880     my ($storecfg, $storeid, $vmid, $fmt, $arch) = @_;
6881
6882     my (undef, $ovmf_vars) = get_ovmf_files($arch);
6883     die "EFI vars default image not found\n" if ! -f $ovmf_vars;
6884
6885     my $vars_size = PVE::Tools::convert_size(-s $ovmf_vars, 'b' => 'kb');
6886     my $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $fmt, undef, $vars_size);
6887     PVE::Storage::activate_volumes($storecfg, [$volid]);
6888
6889     my $path = PVE::Storage::path($storecfg, $volid);
6890     eval {
6891         run_command(['/usr/bin/qemu-img', 'convert', '-n', '-f', 'raw', '-O', $fmt, $ovmf_vars, $path]);
6892     };
6893     die "Copying EFI vars image failed: $@" if $@;
6894
6895     return ($volid, $vars_size);
6896 }
6897
6898 sub vm_iothreads_list {
6899     my ($vmid) = @_;
6900
6901     my $res = vm_mon_cmd($vmid, 'query-iothreads');
6902
6903     my $iothreads = {};
6904     foreach my $iothread (@$res) {
6905         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
6906     }
6907
6908     return $iothreads;
6909 }
6910
6911 sub scsihw_infos {
6912     my ($conf, $drive) = @_;
6913
6914     my $maxdev = 0;
6915
6916     if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)) {
6917         $maxdev = 7;
6918     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
6919         $maxdev = 1;
6920     } else {
6921         $maxdev = 256;
6922     }
6923
6924     my $controller = int($drive->{index} / $maxdev);
6925     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single') ? "virtioscsi" : "scsihw";
6926
6927     return ($maxdev, $controller, $controller_prefix);
6928 }
6929
6930 sub add_hyperv_enlightenments {
6931     my ($cpuFlags, $winversion, $machine_type, $kvmver, $bios, $gpu_passthrough, $hv_vendor_id) = @_;
6932
6933     return if $winversion < 6;
6934     return if $bios && $bios eq 'ovmf' && $winversion < 8;
6935
6936     if ($gpu_passthrough || defined($hv_vendor_id)) {
6937         $hv_vendor_id //= 'proxmox';
6938         push @$cpuFlags , "hv_vendor_id=$hv_vendor_id";
6939     }
6940
6941     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
6942         push @$cpuFlags , 'hv_spinlocks=0x1fff';
6943         push @$cpuFlags , 'hv_vapic';
6944         push @$cpuFlags , 'hv_time';
6945     } else {
6946         push @$cpuFlags , 'hv_spinlocks=0xffff';
6947     }
6948
6949     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 6)) {
6950         push @$cpuFlags , 'hv_reset';
6951         push @$cpuFlags , 'hv_vpindex';
6952         push @$cpuFlags , 'hv_runtime';
6953     }
6954
6955     if ($winversion >= 7) {
6956         push @$cpuFlags , 'hv_relaxed';
6957
6958         if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 12)) {
6959             push @$cpuFlags , 'hv_synic';
6960             push @$cpuFlags , 'hv_stimer';
6961         }
6962     }
6963 }
6964
6965 sub windows_version {
6966     my ($ostype) = @_;
6967
6968     return 0 if !$ostype;
6969
6970     my $winversion = 0;
6971
6972     if($ostype eq 'wxp' || $ostype eq 'w2k3' || $ostype eq 'w2k') {
6973         $winversion = 5;
6974     } elsif($ostype eq 'w2k8' || $ostype eq 'wvista') {
6975         $winversion = 6;
6976     } elsif ($ostype =~ m/^win(\d+)$/) {
6977         $winversion = $1;
6978     }
6979
6980     return $winversion;
6981 }
6982
6983 sub resolve_dst_disk_format {
6984         my ($storecfg, $storeid, $src_volname, $format) = @_;
6985         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
6986
6987         if (!$format) {
6988             # if no target format is specified, use the source disk format as hint
6989             if ($src_volname) {
6990                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6991                 $format = qemu_img_format($scfg, $src_volname);
6992             } else {
6993                 return $defFormat;
6994             }
6995         }
6996
6997         # test if requested format is supported - else use default
6998         my $supported = grep { $_ eq $format } @$validFormats;
6999         $format = $defFormat if !$supported;
7000         return $format;
7001 }
7002
7003 sub resolve_first_disk {
7004     my $conf = shift;
7005     my @disks = PVE::QemuServer::valid_drive_names();
7006     my $firstdisk;
7007     foreach my $ds (reverse @disks) {
7008         next if !$conf->{$ds};
7009         my $disk = PVE::QemuServer::parse_drive($ds, $conf->{$ds});
7010         next if PVE::QemuServer::drive_is_cdrom($disk);
7011         $firstdisk = $ds;
7012     }
7013     return $firstdisk;
7014 }
7015
7016 sub generate_uuid {
7017     my ($uuid, $uuid_str);
7018     UUID::generate($uuid);
7019     UUID::unparse($uuid, $uuid_str);
7020     return $uuid_str;
7021 }
7022
7023 sub generate_smbios1_uuid {
7024     return "uuid=".generate_uuid();
7025 }
7026
7027 sub nbd_stop {
7028     my ($vmid) = @_;
7029
7030     vm_mon_cmd($vmid, 'nbd-server-stop');
7031 }
7032
7033 # bash completion helper
7034
7035 sub complete_backup_archives {
7036     my ($cmdname, $pname, $cvalue) = @_;
7037
7038     my $cfg = PVE::Storage::config();
7039
7040     my $storeid;
7041
7042     if ($cvalue =~ m/^([^:]+):/) {
7043         $storeid = $1;
7044     }
7045
7046     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
7047
7048     my $res = [];
7049     foreach my $id (keys %$data) {
7050         foreach my $item (@{$data->{$id}}) {
7051             next if $item->{format} !~ m/^vma\.(gz|lzo)$/;
7052             push @$res, $item->{volid} if defined($item->{volid});
7053         }
7054     }
7055
7056     return $res;
7057 }
7058
7059 my $complete_vmid_full = sub {
7060     my ($running) = @_;
7061
7062     my $idlist = vmstatus();
7063
7064     my $res = [];
7065
7066     foreach my $id (keys %$idlist) {
7067         my $d = $idlist->{$id};
7068         if (defined($running)) {
7069             next if $d->{template};
7070             next if $running && $d->{status} ne 'running';
7071             next if !$running && $d->{status} eq 'running';
7072         }
7073         push @$res, $id;
7074
7075     }
7076     return $res;
7077 };
7078
7079 sub complete_vmid {
7080     return &$complete_vmid_full();
7081 }
7082
7083 sub complete_vmid_stopped {
7084     return &$complete_vmid_full(0);
7085 }
7086
7087 sub complete_vmid_running {
7088     return &$complete_vmid_full(1);
7089 }
7090
7091 sub complete_storage {
7092
7093     my $cfg = PVE::Storage::config();
7094     my $ids = $cfg->{ids};
7095
7096     my $res = [];
7097     foreach my $sid (keys %$ids) {
7098         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
7099         next if !$ids->{$sid}->{content}->{images};
7100         push @$res, $sid;
7101     }
7102
7103     return $res;
7104 }
7105
7106 1;