PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5 use POSIX;
   6 use IO::Handle;
   7 use IO::Select;
   8 use IO::File;
   9 use IO::Dir;
  10 use IO::Socket::UNIX;
  11 use File::Basename;
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use Digest::SHA;
  16 use Fcntl ':flock';
  17 use Cwd 'abs_path';
  18 use IPC::Open3;
  19 use JSON;
  20 use Fcntl;
  21 use PVE::SafeSyslog;
  22 use Storable qw(dclone);
  23 use PVE::Exception qw(raise raise_param_exc);
  24 use PVE::Storage;
  25 use PVE::Tools qw(run_command lock_file lock_file_full file_read_firstline dir_glob_foreach $IPV6RE);
  26 use PVE::JSONSchema qw(get_standard_option);
  27 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file cfs_lock_file);
  28 use PVE::INotify;
  29 use PVE::ProcFSTools;
  30 use PVE::QemuConfig;
  31 use PVE::QMPClient;
  32 use PVE::RPCEnvironment;
  33 use PVE::GuestHelpers;
  34 use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr);
  35 use PVE::QemuServer::Memory;
  36 use PVE::QemuServer::USB qw(parse_usb_device);
  37 use PVE::QemuServer::Cloudinit;
  38 use PVE::SysFSTools;
  39 use PVE::Systemd;
  40 use Time::HiRes qw(gettimeofday);
  41 use File::Copy qw(copy);
  42 use URI::Escape;
  43
  44 my $EDK2_FW_BASE = '/usr/share/pve-edk2-firmware/';
  45 my $OVMF = {
  46     x86_64 => [
  47         "$EDK2_FW_BASE/OVMF_CODE.fd",
  48         "$EDK2_FW_BASE/OVMF_VARS.fd"
  49     ],
  50     aarch64 => [
  51         "$EDK2_FW_BASE/AAVMF_CODE.fd",
  52         "$EDK2_FW_BASE/AAVMF_VARS.fd"
  53     ],
  54 };
  55
  56 my $qemu_snap_storage = {rbd => 1, sheepdog => 1};
  57
  58 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  59
  60 my $QEMU_FORMAT_RE = qr/raw|cow|qcow|qcow2|qed|vmdk|cloop/;
  61
  62 # Note about locking: we use flock on the config file protect
  63 # against concurent actions.
  64 # Aditionaly, we have a 'lock' setting in the config file. This
  65 # can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
  66 # allowed when such lock is set. But you can ignore this kind of
  67 # lock with the --skiplock flag.
  68
  69 cfs_register_file('/qemu-server/',
  70                   \&parse_vm_config,
  71                   \&write_vm_config);
  72
  73 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
  74     description => "Some command save/restore state from this location.",
  75     type => 'string',
  76     maxLength => 128,
  77     optional => 1,
  78 });
  79
  80 PVE::JSONSchema::register_standard_option('pve-snapshot-name', {
  81     description => "The name of the snapshot.",
  82     type => 'string', format => 'pve-configid',
  83     maxLength => 40,
  84 });
  85
  86 PVE::JSONSchema::register_standard_option('pve-qm-image-format', {
  87     type => 'string',
  88     enum => [qw(raw cow qcow qed qcow2 vmdk cloop)],
  89     description => "The drive's backing file's data format.",
  90     optional => 1,
  91 });
  92
  93 PVE::JSONSchema::register_standard_option('pve-qemu-machine', {
  94         description => "Specifies the Qemu machine type.",
  95         type => 'string',
  96         pattern => '(pc|pc(-i440fx)?-\d+\.\d+(\.pxe)?|q35|pc-q35-\d+\.\d+(\.pxe)?|virt(?:-\d+\.\d+)?)',
  97         maxLength => 40,
  98         optional => 1,
  99 });
 100
 101 #no warnings 'redefine';
 102
 103 sub cgroups_write {
 104    my ($controller, $vmid, $option, $value) = @_;
 105
 106    my $path = "/sys/fs/cgroup/$controller/qemu.slice/$vmid.scope/$option";
 107    PVE::ProcFSTools::write_proc_entry($path, $value);
 108
 109 }
 110
 111 my $nodename = PVE::INotify::nodename();
 112
 113 mkdir "/etc/pve/nodes/$nodename";
 114 my $confdir = "/etc/pve/nodes/$nodename/qemu-server";
 115 mkdir $confdir;
 116
 117 my $var_run_tmpdir = "/var/run/qemu-server";
 118 mkdir $var_run_tmpdir;
 119
 120 my $lock_dir = "/var/lock/qemu-server";
 121 mkdir $lock_dir;
 122
 123 my $cpu_vendor_list = {
 124     # Intel CPUs
 125     486 => 'GenuineIntel',
 126     pentium => 'GenuineIntel',
 127     pentium2  => 'GenuineIntel',
 128     pentium3  => 'GenuineIntel',
 129     coreduo => 'GenuineIntel',
 130     core2duo => 'GenuineIntel',
 131     Conroe  => 'GenuineIntel',
 132     Penryn  => 'GenuineIntel',
 133     Nehalem  => 'GenuineIntel',
 134     'Nehalem-IBRS'  => 'GenuineIntel',
 135     Westmere => 'GenuineIntel',
 136     'Westmere-IBRS' => 'GenuineIntel',
 137     SandyBridge => 'GenuineIntel',
 138     'SandyBridge-IBRS' => 'GenuineIntel',
 139     IvyBridge => 'GenuineIntel',
 140     'IvyBridge-IBRS' => 'GenuineIntel',
 141     Haswell => 'GenuineIntel',
 142     'Haswell-IBRS' => 'GenuineIntel',
 143     'Haswell-noTSX' => 'GenuineIntel',
 144     'Haswell-noTSX-IBRS' => 'GenuineIntel',
 145     Broadwell => 'GenuineIntel',
 146     'Broadwell-IBRS' => 'GenuineIntel',
 147     'Broadwell-noTSX' => 'GenuineIntel',
 148     'Broadwell-noTSX-IBRS' => 'GenuineIntel',
 149     'Skylake-Client' => 'GenuineIntel',
 150     'Skylake-Client-IBRS' => 'GenuineIntel',
 151     'Skylake-Server' => 'GenuineIntel',
 152     'Skylake-Server-IBRS' => 'GenuineIntel',
 153
 154     # AMD CPUs
 155     athlon => 'AuthenticAMD',
 156     phenom  => 'AuthenticAMD',
 157     Opteron_G1  => 'AuthenticAMD',
 158     Opteron_G2  => 'AuthenticAMD',
 159     Opteron_G3  => 'AuthenticAMD',
 160     Opteron_G4  => 'AuthenticAMD',
 161     Opteron_G5  => 'AuthenticAMD',
 162     EPYC => 'AuthenticAMD',
 163     'EPYC-IBPB' => 'AuthenticAMD',
 164
 165     # generic types, use vendor from host node
 166     host => 'default',
 167     kvm32 => 'default',
 168     kvm64 => 'default',
 169     qemu32 => 'default',
 170     qemu64 => 'default',
 171     max => 'default',
 172 };
 173
 174 my $cpu_flag = qr/[+-](pcid|spec-ctrl|ibpb|ssbd|virt-ssbd|amd-ssbd|amd-no-ssb|pdpe1gb)/;
 175
 176 my $cpu_fmt = {
 177     cputype => {
 178         description => "Emulated CPU type.",
 179         type => 'string',
 180         enum => [ sort { "\L$a" cmp "\L$b" } keys %$cpu_vendor_list ],
 181         default => 'kvm64',
 182         default_key => 1,
 183     },
 184     hidden => {
 185         description => "Do not identify as a KVM virtual machine.",
 186         type => 'boolean',
 187         optional => 1,
 188         default => 0
 189     },
 190     'hv-vendor-id' => {
 191         type => 'string',
 192         pattern => qr/[a-zA-Z0-9]{1,12}/,
 193         format_description => 'vendor-id',
 194         description => 'The Hyper-V vendor ID. Some drivers or programs inside Windows guests need a specific ID.',
 195         optional => 1,
 196     },
 197     flags => {
 198         description => "List of additional CPU flags separated by ';'."
 199                      . " Use '+FLAG' to enable, '-FLAG' to disable a flag."
 200                      . " Currently supported flags: 'pcid', 'spec-ctrl', 'ibpb', 'ssbd', 'virt-ssbd', 'amd-ssbd', 'amd-no-ssb', 'pdpe1gb'.",
 201         format_description => '+FLAG[;-FLAG...]',
 202         type => 'string',
 203         pattern => qr/$cpu_flag(;$cpu_flag)*/,
 204         optional => 1,
 205     },
 206 };
 207
 208 my $watchdog_fmt = {
 209     model => {
 210         default_key => 1,
 211         type => 'string',
 212         enum => [qw(i6300esb ib700)],
 213         description => "Watchdog type to emulate.",
 214         default => 'i6300esb',
 215         optional => 1,
 216     },
 217     action => {
 218         type => 'string',
 219         enum => [qw(reset shutdown poweroff pause debug none)],
 220         description => "The action to perform if after activation the guest fails to poll the watchdog in time.",
 221         optional => 1,
 222     },
 223 };
 224 PVE::JSONSchema::register_format('pve-qm-watchdog', $watchdog_fmt);
 225
 226 my $agent_fmt = {
 227     enabled => {
 228         description => "Enable/disable Qemu GuestAgent.",
 229         type => 'boolean',
 230         default => 0,
 231         default_key => 1,
 232     },
 233     fstrim_cloned_disks => {
 234         description => "Run fstrim after cloning/moving a disk.",
 235         type => 'boolean',
 236         optional => 1,
 237         default => 0
 238     },
 239 };
 240
 241 my $vga_fmt = {
 242     type => {
 243         description => "Select the VGA type.",
 244         type => 'string',
 245         default => 'std',
 246         optional => 1,
 247         default_key => 1,
 248         enum => [qw(cirrus qxl qxl2 qxl3 qxl4 none serial0 serial1 serial2 serial3 std virtio vmware)],
 249     },
 250     memory => {
 251         description => "Sets the VGA memory (in MiB). Has no effect with serial display.",
 252         type => 'integer',
 253         optional => 1,
 254         minimum => 4,
 255         maximum => 512,
 256     },
 257 };
 258
 259 my $ivshmem_fmt = {
 260     size => {
 261         type => 'integer',
 262         minimum => 1,
 263         description => "The size of the file in MB.",
 264     },
 265     name => {
 266         type => 'string',
 267         pattern => '[a-zA-Z0-9\-]+',
 268         optional => 1,
 269         format_description => 'string',
 270         description => "The name of the file. Will be prefixed with 'pve-shm-'. Default is the VMID. Will be deleted when the VM is stopped.",
 271     },
 272 };
 273
 274 my $confdesc = {
 275     onboot => {
 276         optional => 1,
 277         type => 'boolean',
 278         description => "Specifies whether a VM will be started during system bootup.",
 279         default => 0,
 280     },
 281     autostart => {
 282         optional => 1,
 283         type => 'boolean',
 284         description => "Automatic restart after crash (currently ignored).",
 285         default => 0,
 286     },
 287     hotplug => {
 288         optional => 1,
 289         type => 'string', format => 'pve-hotplug-features',
 290         description => "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
 291         default => 'network,disk,usb',
 292     },
 293     reboot => {
 294         optional => 1,
 295         type => 'boolean',
 296         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 297         default => 1,
 298     },
 299     lock => {
 300         optional => 1,
 301         type => 'string',
 302         description => "Lock/unlock the VM.",
 303         enum => [qw(backup clone create migrate rollback snapshot snapshot-delete)],
 304     },
 305     cpulimit => {
 306         optional => 1,
 307         type => 'number',
 308         description => "Limit of CPU usage.",
 309         verbose_description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
 310         minimum => 0,
 311         maximum => 128,
 312         default => 0,
 313     },
 314     cpuunits => {
 315         optional => 1,
 316         type => 'integer',
 317         description => "CPU weight for a VM.",
 318         verbose_description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.",
 319         minimum => 2,
 320         maximum => 262144,
 321         default => 1024,
 322     },
 323     memory => {
 324         optional => 1,
 325         type => 'integer',
 326         description => "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
 327         minimum => 16,
 328         default => 512,
 329     },
 330     balloon => {
 331         optional => 1,
 332         type => 'integer',
 333         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 334         minimum => 0,
 335     },
 336     shares => {
 337         optional => 1,
 338         type => 'integer',
 339         description => "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning. Auto-ballooning is done by pvestatd.",
 340         minimum => 0,
 341         maximum => 50000,
 342         default => 1000,
 343     },
 344     keyboard => {
 345         optional => 1,
 346         type => 'string',
 347         description => "Keybord layout for vnc server. Default is read from the '/etc/pve/datacenter.cfg' configuration file.".
 348                        "It should not be necessary to set it.",
 349         enum => PVE::Tools::kvmkeymaplist(),
 350         default => undef,
 351     },
 352     name => {
 353         optional => 1,
 354         type => 'string', format => 'dns-name',
 355         description => "Set a name for the VM. Only used on the configuration web interface.",
 356     },
 357     scsihw => {
 358         optional => 1,
 359         type => 'string',
 360         description => "SCSI controller model",
 361         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 362         default => 'lsi',
 363     },
 364     description => {
 365         optional => 1,
 366         type => 'string',
 367         description => "Description for the VM. Only used on the configuration web interface. This is saved as comment inside the configuration file.",
 368     },
 369     ostype => {
 370         optional => 1,
 371         type => 'string',
 372         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 l24 l26 solaris)],
 373         description => "Specify guest operating system.",
 374         verbose_description => <<EODESC,
 375 Specify guest operating system. This is used to enable special
 376 optimization/features for specific operating systems:
 377
 378 [horizontal]
 379 other;; unspecified OS
 380 wxp;; Microsoft Windows XP
 381 w2k;; Microsoft Windows 2000
 382 w2k3;; Microsoft Windows 2003
 383 w2k8;; Microsoft Windows 2008
 384 wvista;; Microsoft Windows Vista
 385 win7;; Microsoft Windows 7
 386 win8;; Microsoft Windows 8/2012/2012r2
 387 win10;; Microsoft Windows 10/2016
 388 l24;; Linux 2.4 Kernel
 389 l26;; Linux 2.6/3.X Kernel
 390 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
 391 EODESC
 392     },
 393     boot => {
 394         optional => 1,
 395         type => 'string',
 396         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n).",
 397         pattern => '[acdn]{1,4}',
 398         default => 'cdn',
 399     },
 400     bootdisk => {
 401         optional => 1,
 402         type => 'string', format => 'pve-qm-bootdisk',
 403         description => "Enable booting from specified disk.",
 404         pattern => '(ide|sata|scsi|virtio)\d+',
 405     },
 406     smp => {
 407         optional => 1,
 408         type => 'integer',
 409         description => "The number of CPUs. Please use option -sockets instead.",
 410         minimum => 1,
 411         default => 1,
 412     },
 413     sockets => {
 414         optional => 1,
 415         type => 'integer',
 416         description => "The number of CPU sockets.",
 417         minimum => 1,
 418         default => 1,
 419     },
 420     cores => {
 421         optional => 1,
 422         type => 'integer',
 423         description => "The number of cores per socket.",
 424         minimum => 1,
 425         default => 1,
 426     },
 427     numa => {
 428         optional => 1,
 429         type => 'boolean',
 430         description => "Enable/disable NUMA.",
 431         default => 0,
 432     },
 433     hugepages => {
 434         optional => 1,
 435         type => 'string',
 436         description => "Enable/disable hugepages memory.",
 437         enum => [qw(any 2 1024)],
 438     },
 439     vcpus => {
 440         optional => 1,
 441         type => 'integer',
 442         description => "Number of hotplugged vcpus.",
 443         minimum => 1,
 444         default => 0,
 445     },
 446     acpi => {
 447         optional => 1,
 448         type => 'boolean',
 449         description => "Enable/disable ACPI.",
 450         default => 1,
 451     },
 452     agent => {
 453         optional => 1,
 454         description => "Enable/disable Qemu GuestAgent and its properties.",
 455         type => 'string',
 456         format => $agent_fmt,
 457     },
 458     kvm => {
 459         optional => 1,
 460         type => 'boolean',
 461         description => "Enable/disable KVM hardware virtualization.",
 462         default => 1,
 463     },
 464     tdf => {
 465         optional => 1,
 466         type => 'boolean',
 467         description => "Enable/disable time drift fix.",
 468         default => 0,
 469     },
 470     localtime => {
 471         optional => 1,
 472         type => 'boolean',
 473         description => "Set the real time clock to local time. This is enabled by default if ostype indicates a Microsoft OS.",
 474     },
 475     freeze => {
 476         optional => 1,
 477         type => 'boolean',
 478         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 479     },
 480     vga => {
 481         optional => 1,
 482         type => 'string', format => $vga_fmt,
 483         description => "Configure the VGA hardware.",
 484         verbose_description => "Configure the VGA Hardware. If you want to use ".
 485             "high resolution modes (>= 1280x1024x16) you may need to increase " .
 486             "the vga memory option. Since QEMU 2.9 the default VGA display type " .
 487             "is 'std' for all OS types besides some Windows versions (XP and " .
 488             "older) which use 'cirrus'. The 'qxl' option enables the SPICE " .
 489             "display server. For win* OS you can select how many independent " .
 490             "displays you want, Linux guests can add displays them self.\n".
 491             "You can also run without any graphic card, using a serial device as terminal.",
 492     },
 493     watchdog => {
 494         optional => 1,
 495         type => 'string', format => 'pve-qm-watchdog',
 496         description => "Create a virtual hardware watchdog device.",
 497         verbose_description => "Create a virtual hardware watchdog device. Once enabled" .
 498             " (by a guest action), the watchdog must be periodically polled " .
 499             "by an agent inside the guest or else the watchdog will reset " .
 500             "the guest (or execute the respective action specified)",
 501     },
 502     startdate => {
 503         optional => 1,
 504         type => 'string',
 505         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 506         description => "Set the initial date of the real time clock. Valid format for date are: 'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 507         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 508         default => 'now',
 509     },
 510     startup =>  get_standard_option('pve-startup-order'),
 511     template => {
 512         optional => 1,
 513         type => 'boolean',
 514         description => "Enable/disable Template.",
 515         default => 0,
 516     },
 517     args => {
 518         optional => 1,
 519         type => 'string',
 520         description => "Arbitrary arguments passed to kvm.",
 521         verbose_description => <<EODESCR,
 522 Arbitrary arguments passed to kvm, for example:
 523
 524 args: -no-reboot -no-hpet
 525
 526 NOTE: this option is for experts only.
 527 EODESCR
 528     },
 529     tablet => {
 530         optional => 1,
 531         type => 'boolean',
 532         default => 1,
 533         description => "Enable/disable the USB tablet device.",
 534         verbose_description => "Enable/disable the USB tablet device. This device is " .
 535             "usually needed to allow absolute mouse positioning with VNC. " .
 536             "Else the mouse runs out of sync with normal VNC clients. " .
 537             "If you're running lots of console-only guests on one host, " .
 538             "you may consider disabling this to save some context switches. " .
 539             "This is turned off by default if you use spice (-vga=qxl).",
 540     },
 541     migrate_speed => {
 542         optional => 1,
 543         type => 'integer',
 544         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 545         minimum => 0,
 546         default => 0,
 547     },
 548     migrate_downtime => {
 549         optional => 1,
 550         type => 'number',
 551         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 552         minimum => 0,
 553         default => 0.1,
 554     },
 555     cdrom => {
 556         optional => 1,
 557         type => 'string', format => 'pve-qm-ide',
 558         typetext => '<volume>',
 559         description => "This is an alias for option -ide2",
 560     },
 561     cpu => {
 562         optional => 1,
 563         description => "Emulated CPU type.",
 564         type => 'string',
 565         format => $cpu_fmt,
 566     },
 567     parent => get_standard_option('pve-snapshot-name', {
 568         optional => 1,
 569         description => "Parent snapshot name. This is used internally, and should not be modified.",
 570     }),
 571     snaptime => {
 572         optional => 1,
 573         description => "Timestamp for snapshots.",
 574         type => 'integer',
 575         minimum => 0,
 576     },
 577     vmstate => {
 578         optional => 1,
 579         type => 'string', format => 'pve-volume-id',
 580         description => "Reference to a volume which stores the VM state. This is used internally for snapshots.",
 581     },
 582     vmstatestorage => get_standard_option('pve-storage-id', {
 583         description => "Default storage for VM state volumes/files.",
 584         optional => 1,
 585     }),
 586     runningmachine => get_standard_option('pve-qemu-machine', {
 587         description => "Specifies the Qemu machine type of the running vm. This is used internally for snapshots.",
 588     }),
 589     machine => get_standard_option('pve-qemu-machine'),
 590     arch => {
 591         description => "Virtual processor architecture. Defaults to the host.",
 592         optional => 1,
 593         type => 'string',
 594         enum => [qw(x86_64 aarch64)],
 595     },
 596     smbios1 => {
 597         description => "Specify SMBIOS type 1 fields.",
 598         type => 'string', format => 'pve-qm-smbios1',
 599         maxLength => 256,
 600         optional => 1,
 601     },
 602     protection => {
 603         optional => 1,
 604         type => 'boolean',
 605         description => "Sets the protection flag of the VM. This will disable the remove VM and remove disk operations.",
 606         default => 0,
 607     },
 608     bios => {
 609         optional => 1,
 610         type => 'string',
 611         enum => [ qw(seabios ovmf) ],
 612         description => "Select BIOS implementation.",
 613         default => 'seabios',
 614     },
 615     vmgenid => {
 616         type => 'string',
 617         pattern => '(?:[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}|[01])',
 618         format_description => 'UUID',
 619         description => "Set VM Generation ID. Use '1' to autogenerate on create or update, pass '0' to disable explicitly.",
 620         verbose_description => "The VM generation ID (vmgenid) device exposes a".
 621             " 128-bit integer value identifier to the guest OS. This allows to".
 622             " notify the guest operating system when the virtual machine is".
 623             " executed with a different configuration (e.g. snapshot execution".
 624             " or creation from a template). The guest operating system notices".
 625             " the change, and is then able to react as appropriate by marking".
 626             " its copies of distributed databases as dirty, re-initializing its".
 627             " random number generator, etc.\n".
 628             "Note that auto-creation only works when done throug API/CLI create".
 629             " or update methods, but not when manually editing the config file.",
 630         default => "1 (autogenerated)",
 631         optional => 1,
 632     },
 633     hookscript => {
 634         type => 'string',
 635         format => 'pve-volume-id',
 636         optional => 1,
 637         description => "Script that will be executed during various steps in the vms lifetime.",
 638     },
 639     ivshmem => {
 640         type => 'string',
 641         format => $ivshmem_fmt,
 642         description => "Inter-VM shared memory. Useful for direct communication between VMs, or to the host.",
 643         optional => 1,
 644     }
 645 };
 646
 647 my $confdesc_cloudinit = {
 648     citype => {
 649         optional => 1,
 650         type => 'string',
 651         description => 'Specifies the cloud-init configuration format. The default depends on the configured operating system type (`ostype`. We use the `nocloud` format for Linux, and `configdrive2` for windows.',
 652         enum => ['configdrive2', 'nocloud'],
 653     },
 654     ciuser => {
 655         optional => 1,
 656         type => 'string',
 657         description => "cloud-init: User name to change ssh keys and password for instead of the image's configured default user.",
 658     },
 659     cipassword => {
 660         optional => 1,
 661         type => 'string',
 662         description => 'cloud-init: Password to assign the user. Using this is generally not recommended. Use ssh keys instead. Also note that older cloud-init versions do not support hashed passwords.',
 663     },
 664     searchdomain => {
 665         optional => 1,
 666         type => 'string',
 667         description => "cloud-init: Sets DNS search domains for a container. Create will automatically use the setting from the host if neither searchdomain nor nameserver are set.",
 668     },
 669     nameserver => {
 670         optional => 1,
 671         type => 'string', format => 'address-list',
 672         description => "cloud-init: Sets DNS server IP address for a container. Create will automatically use the setting from the host if neither searchdomain nor nameserver are set.",
 673     },
 674     sshkeys => {
 675         optional => 1,
 676         type => 'string',
 677         format => 'urlencoded',
 678         description => "cloud-init: Setup public SSH keys (one key per line, OpenSSH format).",
 679     },
 680 };
 681
 682 # what about other qemu settings ?
 683 #cpu => 'string',
 684 #machine => 'string',
 685 #fda => 'file',
 686 #fdb => 'file',
 687 #mtdblock => 'file',
 688 #sd => 'file',
 689 #pflash => 'file',
 690 #snapshot => 'bool',
 691 #bootp => 'file',
 692 ##tftp => 'dir',
 693 ##smb => 'dir',
 694 #kernel => 'file',
 695 #append => 'string',
 696 #initrd => 'file',
 697 ##soundhw => 'string',
 698
 699 while (my ($k, $v) = each %$confdesc) {
 700     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 701 }
 702
 703 my $MAX_IDE_DISKS = 4;
 704 my $MAX_SCSI_DISKS = 14;
 705 my $MAX_VIRTIO_DISKS = 16;
 706 my $MAX_SATA_DISKS = 6;
 707 my $MAX_USB_DEVICES = 5;
 708 my $MAX_NETS = 32;
 709 my $MAX_UNUSED_DISKS = 256;
 710 my $MAX_HOSTPCI_DEVICES = 4;
 711 my $MAX_SERIAL_PORTS = 4;
 712 my $MAX_PARALLEL_PORTS = 3;
 713 my $MAX_NUMA = 8;
 714
 715 my $numa_fmt = {
 716     cpus => {
 717         type => "string",
 718         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 719         description => "CPUs accessing this NUMA node.",
 720         format_description => "id[-id];...",
 721     },
 722     memory => {
 723         type => "number",
 724         description => "Amount of memory this NUMA node provides.",
 725         optional => 1,
 726     },
 727     hostnodes => {
 728         type => "string",
 729         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 730         description => "Host NUMA nodes to use.",
 731         format_description => "id[-id];...",
 732         optional => 1,
 733     },
 734     policy => {
 735         type => 'string',
 736         enum => [qw(preferred bind interleave)],
 737         description => "NUMA allocation policy.",
 738         optional => 1,
 739     },
 740 };
 741 PVE::JSONSchema::register_format('pve-qm-numanode', $numa_fmt);
 742 my $numadesc = {
 743     optional => 1,
 744     type => 'string', format => $numa_fmt,
 745     description => "NUMA topology.",
 746 };
 747 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 748
 749 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 750     $confdesc->{"numa$i"} = $numadesc;
 751 }
 752
 753 my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000',  'pcnet',  'virtio',
 754                       'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3',
 755                       'e1000-82540em', 'e1000-82544gc', 'e1000-82545em'];
 756 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 757
 758 my $net_fmt_bridge_descr = <<__EOD__;
 759 Bridge to attach the network device to. The Proxmox VE standard bridge
 760 is called 'vmbr0'.
 761
 762 If you do not specify a bridge, we create a kvm user (NATed) network
 763 device, which provides DHCP and DNS services. The following addresses
 764 are used:
 765
 766  10.0.2.2   Gateway
 767  10.0.2.3   DNS Server
 768  10.0.2.4   SMB Server
 769
 770 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 771 __EOD__
 772
 773 my $net_fmt = {
 774     macaddr => {
 775         type => 'string',
 776         pattern => qr/[0-9a-f]{2}(?::[0-9a-f]{2}){5}/i,
 777         description => "MAC address. That address must be unique withing your network. This is automatically generated if not specified.",
 778         format_description => "XX:XX:XX:XX:XX:XX",
 779         optional => 1,
 780     },
 781     model => {
 782         type => 'string',
 783         description => "Network Card Model. The 'virtio' model provides the best performance with very low CPU overhead. If your guest does not support this driver, it is usually best to use 'e1000'.",
 784         enum => $nic_model_list,
 785         default_key => 1,
 786     },
 787     (map { $_ => { keyAlias => 'model', alias => 'macaddr' }} @$nic_model_list),
 788     bridge => {
 789         type => 'string',
 790         description => $net_fmt_bridge_descr,
 791         format_description => 'bridge',
 792         optional => 1,
 793     },
 794     queues => {
 795         type => 'integer',
 796         minimum => 0, maximum => 16,
 797         description => 'Number of packet queues to be used on the device.',
 798         optional => 1,
 799     },
 800     rate => {
 801         type => 'number',
 802         minimum => 0,
 803         description => "Rate limit in mbps (megabytes per second) as floating point number.",
 804         optional => 1,
 805     },
 806     tag => {
 807         type => 'integer',
 808         minimum => 1, maximum => 4094,
 809         description => 'VLAN tag to apply to packets on this interface.',
 810         optional => 1,
 811     },
 812     trunks => {
 813         type => 'string',
 814         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 815         description => 'VLAN trunks to pass through this interface.',
 816         format_description => 'vlanid[;vlanid...]',
 817         optional => 1,
 818     },
 819     firewall => {
 820         type => 'boolean',
 821         description => 'Whether this interface should be protected by the firewall.',
 822         optional => 1,
 823     },
 824     link_down => {
 825         type => 'boolean',
 826         description => 'Whether this interface should be disconnected (like pulling the plug).',
 827         optional => 1,
 828     },
 829 };
 830
 831 my $netdesc = {
 832     optional => 1,
 833     type => 'string', format => $net_fmt,
 834     description => "Specify network devices.",
 835 };
 836
 837 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 838
 839 my $ipconfig_fmt = {
 840     ip => {
 841         type => 'string',
 842         format => 'pve-ipv4-config',
 843         format_description => 'IPv4Format/CIDR',
 844         description => 'IPv4 address in CIDR format.',
 845         optional => 1,
 846         default => 'dhcp',
 847     },
 848     gw => {
 849         type => 'string',
 850         format => 'ipv4',
 851         format_description => 'GatewayIPv4',
 852         description => 'Default gateway for IPv4 traffic.',
 853         optional => 1,
 854         requires => 'ip',
 855     },
 856     ip6 => {
 857         type => 'string',
 858         format => 'pve-ipv6-config',
 859         format_description => 'IPv6Format/CIDR',
 860         description => 'IPv6 address in CIDR format.',
 861         optional => 1,
 862         default => 'dhcp',
 863     },
 864     gw6 => {
 865         type => 'string',
 866         format => 'ipv6',
 867         format_description => 'GatewayIPv6',
 868         description => 'Default gateway for IPv6 traffic.',
 869         optional => 1,
 870         requires => 'ip6',
 871     },
 872 };
 873 PVE::JSONSchema::register_format('pve-qm-ipconfig', $ipconfig_fmt);
 874 my $ipconfigdesc = {
 875     optional => 1,
 876     type => 'string', format => 'pve-qm-ipconfig',
 877     description => <<'EODESCR',
 878 cloud-init: Specify IP addresses and gateways for the corresponding interface.
 879
 880 IP addresses use CIDR notation, gateways are optional but need an IP of the same type specified.
 881
 882 The special string 'dhcp' can be used for IP addresses to use DHCP, in which case no explicit gateway should be provided.
 883 For IPv6 the special string 'auto' can be used to use stateless autoconfiguration.
 884
 885 If cloud-init is enabled and neither an IPv4 nor an IPv6 address is specified, it defaults to using dhcp on IPv4.
 886 EODESCR
 887 };
 888 PVE::JSONSchema::register_standard_option("pve-qm-ipconfig", $netdesc);
 889
 890 for (my $i = 0; $i < $MAX_NETS; $i++)  {
 891     $confdesc->{"net$i"} = $netdesc;
 892     $confdesc_cloudinit->{"ipconfig$i"} = $ipconfigdesc;
 893 }
 894
 895 foreach my $key (keys %$confdesc_cloudinit) {
 896     $confdesc->{$key} = $confdesc_cloudinit->{$key};
 897 }
 898
 899 PVE::JSONSchema::register_format('pve-volume-id-or-qm-path', \&verify_volume_id_or_qm_path);
 900 sub verify_volume_id_or_qm_path {
 901     my ($volid, $noerr) = @_;
 902
 903     if ($volid eq 'none' || $volid eq 'cdrom' || $volid =~ m|^/|) {
 904         return $volid;
 905     }
 906
 907     # if its neither 'none' nor 'cdrom' nor a path, check if its a volume-id
 908     $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') };
 909     if ($@) {
 910         return undef if $noerr;
 911         die $@;
 912     }
 913     return $volid;
 914 }
 915
 916 my $drivename_hash;
 917
 918 my %drivedesc_base = (
 919     volume => { alias => 'file' },
 920     file => {
 921         type => 'string',
 922         format => 'pve-volume-id-or-qm-path',
 923         default_key => 1,
 924         format_description => 'volume',
 925         description => "The drive's backing volume.",
 926     },
 927     media => {
 928         type => 'string',
 929         enum => [qw(cdrom disk)],
 930         description => "The drive's media type.",
 931         default => 'disk',
 932         optional => 1
 933     },
 934     cyls => {
 935         type => 'integer',
 936         description => "Force the drive's physical geometry to have a specific cylinder count.",
 937         optional => 1
 938     },
 939     heads => {
 940         type => 'integer',
 941         description => "Force the drive's physical geometry to have a specific head count.",
 942         optional => 1
 943     },
 944     secs => {
 945         type => 'integer',
 946         description => "Force the drive's physical geometry to have a specific sector count.",
 947         optional => 1
 948     },
 949     trans => {
 950         type => 'string',
 951         enum => [qw(none lba auto)],
 952         description => "Force disk geometry bios translation mode.",
 953         optional => 1,
 954     },
 955     snapshot => {
 956         type => 'boolean',
 957         description => "Controls qemu's snapshot mode feature."
 958             . " If activated, changes made to the disk are temporary and will"
 959             . " be discarded when the VM is shutdown.",
 960         optional => 1,
 961     },
 962     cache => {
 963         type => 'string',
 964         enum => [qw(none writethrough writeback unsafe directsync)],
 965         description => "The drive's cache mode",
 966         optional => 1,
 967     },
 968     format => get_standard_option('pve-qm-image-format'),
 969     size => {
 970         type => 'string',
 971         format => 'disk-size',
 972         format_description => 'DiskSize',
 973         description => "Disk size. This is purely informational and has no effect.",
 974         optional => 1,
 975     },
 976     backup => {
 977         type => 'boolean',
 978         description => "Whether the drive should be included when making backups.",
 979         optional => 1,
 980     },
 981     replicate => {
 982         type => 'boolean',
 983         description => 'Whether the drive should considered for replication jobs.',
 984         optional => 1,
 985         default => 1,
 986     },
 987     rerror => {
 988         type => 'string',
 989         enum => [qw(ignore report stop)],
 990         description => 'Read error action.',
 991         optional => 1,
 992     },
 993     werror => {
 994         type => 'string',
 995         enum => [qw(enospc ignore report stop)],
 996         description => 'Write error action.',
 997         optional => 1,
 998     },
 999     aio => {
1000         type => 'string',
1001         enum => [qw(native threads)],
1002         description => 'AIO type to use.',
1003         optional => 1,
1004     },
1005     discard => {
1006         type => 'string',
1007         enum => [qw(ignore on)],
1008         description => 'Controls whether to pass discard/trim requests to the underlying storage.',
1009         optional => 1,
1010     },
1011     detect_zeroes => {
1012         type => 'boolean',
1013         description => 'Controls whether to detect and try to optimize writes of zeroes.',
1014         optional => 1,
1015     },
1016     serial => {
1017         type => 'string',
1018         format => 'urlencoded',
1019         format_description => 'serial',
1020         maxLength => 20*3, # *3 since it's %xx url enoded
1021         description => "The drive's reported serial number, url-encoded, up to 20 bytes long.",
1022         optional => 1,
1023     },
1024     shared => {
1025         type => 'boolean',
1026         description => 'Mark this locally-managed volume as available on all nodes',
1027         verbose_description => "Mark this locally-managed volume as available on all nodes.\n\nWARNING: This option does not share the volume automatically, it assumes it is shared already!",
1028         optional => 1,
1029         default => 0,
1030     }
1031 );
1032
1033 my %iothread_fmt = ( iothread => {
1034         type => 'boolean',
1035         description => "Whether to use iothreads for this drive",
1036         optional => 1,
1037 });
1038
1039 my %model_fmt = (
1040     model => {
1041         type => 'string',
1042         format => 'urlencoded',
1043         format_description => 'model',
1044         maxLength => 40*3, # *3 since it's %xx url enoded
1045         description => "The drive's reported model name, url-encoded, up to 40 bytes long.",
1046         optional => 1,
1047     },
1048 );
1049
1050 my %queues_fmt = (
1051     queues => {
1052         type => 'integer',
1053         description => "Number of queues.",
1054         minimum => 2,
1055         optional => 1
1056     }
1057 );
1058
1059 my %scsiblock_fmt = (
1060     scsiblock => {
1061         type => 'boolean',
1062         description => "whether to use scsi-block for full passthrough of host block device\n\nWARNING: can lead to I/O errors in combination with low memory or high memory fragmentation on host",
1063         optional => 1,
1064         default => 0,
1065     },
1066 );
1067
1068 my %ssd_fmt = (
1069     ssd => {
1070         type => 'boolean',
1071         description => "Whether to expose this drive as an SSD, rather than a rotational hard disk.",
1072         optional => 1,
1073     },
1074 );
1075
1076 my %wwn_fmt = (
1077     wwn => {
1078         type => 'string',
1079         pattern => qr/^(0x)[0-9a-fA-F]{16}/,
1080         format_description => 'wwn',
1081         description => "The drive's worldwide name, encoded as 16 bytes hex string, prefixed by '0x'.",
1082         optional => 1,
1083     },
1084 );
1085
1086 my $add_throttle_desc = sub {
1087     my ($key, $type, $what, $unit, $longunit, $minimum) = @_;
1088     my $d = {
1089         type => $type,
1090         format_description => $unit,
1091         description => "Maximum $what in $longunit.",
1092         optional => 1,
1093     };
1094     $d->{minimum} = $minimum if defined($minimum);
1095     $drivedesc_base{$key} = $d;
1096 };
1097 # throughput: (leaky bucket)
1098 $add_throttle_desc->('bps',     'integer', 'r/w speed',   'bps',  'bytes per second');
1099 $add_throttle_desc->('bps_rd',  'integer', 'read speed',  'bps',  'bytes per second');
1100 $add_throttle_desc->('bps_wr',  'integer', 'write speed', 'bps',  'bytes per second');
1101 $add_throttle_desc->('mbps',    'number',  'r/w speed',   'mbps', 'megabytes per second');
1102 $add_throttle_desc->('mbps_rd', 'number',  'read speed',  'mbps', 'megabytes per second');
1103 $add_throttle_desc->('mbps_wr', 'number',  'write speed', 'mbps', 'megabytes per second');
1104 $add_throttle_desc->('iops',    'integer', 'r/w I/O',     'iops', 'operations per second');
1105 $add_throttle_desc->('iops_rd', 'integer', 'read I/O',    'iops', 'operations per second');
1106 $add_throttle_desc->('iops_wr', 'integer', 'write I/O',   'iops', 'operations per second');
1107
1108 # pools: (pool of IO before throttling starts taking effect)
1109 $add_throttle_desc->('mbps_max',    'number',  'unthrottled r/w pool',       'mbps', 'megabytes per second');
1110 $add_throttle_desc->('mbps_rd_max', 'number',  'unthrottled read pool',      'mbps', 'megabytes per second');
1111 $add_throttle_desc->('mbps_wr_max', 'number',  'unthrottled write pool',     'mbps', 'megabytes per second');
1112 $add_throttle_desc->('iops_max',    'integer', 'unthrottled r/w I/O pool',   'iops', 'operations per second');
1113 $add_throttle_desc->('iops_rd_max', 'integer', 'unthrottled read I/O pool',  'iops', 'operations per second');
1114 $add_throttle_desc->('iops_wr_max', 'integer', 'unthrottled write I/O pool', 'iops', 'operations per second');
1115
1116 # burst lengths
1117 $add_throttle_desc->('bps_max_length',     'integer', 'length of I/O bursts',       'seconds', 'seconds', 1);
1118 $add_throttle_desc->('bps_rd_max_length',  'integer', 'length of read I/O bursts',  'seconds', 'seconds', 1);
1119 $add_throttle_desc->('bps_wr_max_length',  'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
1120 $add_throttle_desc->('iops_max_length',    'integer', 'length of I/O bursts',       'seconds', 'seconds', 1);
1121 $add_throttle_desc->('iops_rd_max_length', 'integer', 'length of read I/O bursts',  'seconds', 'seconds', 1);
1122 $add_throttle_desc->('iops_wr_max_length', 'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
1123
1124 # legacy support
1125 $drivedesc_base{'bps_rd_length'} = { alias => 'bps_rd_max_length' };
1126 $drivedesc_base{'bps_wr_length'} = { alias => 'bps_wr_max_length' };
1127 $drivedesc_base{'iops_rd_length'} = { alias => 'iops_rd_max_length' };
1128 $drivedesc_base{'iops_wr_length'} = { alias => 'iops_wr_max_length' };
1129
1130 my $ide_fmt = {
1131     %drivedesc_base,
1132     %model_fmt,
1133     %ssd_fmt,
1134     %wwn_fmt,
1135 };
1136 PVE::JSONSchema::register_format("pve-qm-ide", $ide_fmt);
1137
1138 my $idedesc = {
1139     optional => 1,
1140     type => 'string', format => $ide_fmt,
1141     description => "Use volume as IDE hard disk or CD-ROM (n is 0 to " .($MAX_IDE_DISKS -1) . ").",
1142 };
1143 PVE::JSONSchema::register_standard_option("pve-qm-ide", $idedesc);
1144
1145 my $scsi_fmt = {
1146     %drivedesc_base,
1147     %iothread_fmt,
1148     %queues_fmt,
1149     %scsiblock_fmt,
1150     %ssd_fmt,
1151     %wwn_fmt,
1152 };
1153 my $scsidesc = {
1154     optional => 1,
1155     type => 'string', format => $scsi_fmt,
1156     description => "Use volume as SCSI hard disk or CD-ROM (n is 0 to " . ($MAX_SCSI_DISKS - 1) . ").",
1157 };
1158 PVE::JSONSchema::register_standard_option("pve-qm-scsi", $scsidesc);
1159
1160 my $sata_fmt = {
1161     %drivedesc_base,
1162     %ssd_fmt,
1163     %wwn_fmt,
1164 };
1165 my $satadesc = {
1166     optional => 1,
1167     type => 'string', format => $sata_fmt,
1168     description => "Use volume as SATA hard disk or CD-ROM (n is 0 to " . ($MAX_SATA_DISKS - 1). ").",
1169 };
1170 PVE::JSONSchema::register_standard_option("pve-qm-sata", $satadesc);
1171
1172 my $virtio_fmt = {
1173     %drivedesc_base,
1174     %iothread_fmt,
1175 };
1176 my $virtiodesc = {
1177     optional => 1,
1178     type => 'string', format => $virtio_fmt,
1179     description => "Use volume as VIRTIO hard disk (n is 0 to " . ($MAX_VIRTIO_DISKS - 1) . ").",
1180 };
1181 PVE::JSONSchema::register_standard_option("pve-qm-virtio", $virtiodesc);
1182
1183 my $alldrive_fmt = {
1184     %drivedesc_base,
1185     %iothread_fmt,
1186     %model_fmt,
1187     %queues_fmt,
1188     %scsiblock_fmt,
1189     %ssd_fmt,
1190     %wwn_fmt,
1191 };
1192
1193 my $efidisk_fmt = {
1194     volume => { alias => 'file' },
1195     file => {
1196         type => 'string',
1197         format => 'pve-volume-id-or-qm-path',
1198         default_key => 1,
1199         format_description => 'volume',
1200         description => "The drive's backing volume.",
1201     },
1202     format => get_standard_option('pve-qm-image-format'),
1203     size => {
1204         type => 'string',
1205         format => 'disk-size',
1206         format_description => 'DiskSize',
1207         description => "Disk size. This is purely informational and has no effect.",
1208         optional => 1,
1209     },
1210 };
1211
1212 my $efidisk_desc = {
1213     optional => 1,
1214     type => 'string', format => $efidisk_fmt,
1215     description => "Configure a Disk for storing EFI vars",
1216 };
1217
1218 PVE::JSONSchema::register_standard_option("pve-qm-efidisk", $efidisk_desc);
1219
1220 my $usb_fmt = {
1221     host => {
1222         default_key => 1,
1223         type => 'string', format => 'pve-qm-usb-device',
1224         format_description => 'HOSTUSBDEVICE|spice',
1225         description => <<EODESCR,
1226 The Host USB device or port or the value 'spice'. HOSTUSBDEVICE syntax is:
1227
1228  'bus-port(.port)*' (decimal numbers) or
1229  'vendor_id:product_id' (hexadeciaml numbers) or
1230  'spice'
1231
1232 You can use the 'lsusb -t' command to list existing usb devices.
1233
1234 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1235
1236 The value 'spice' can be used to add a usb redirection devices for spice.
1237 EODESCR
1238     },
1239     usb3 => {
1240         optional => 1,
1241         type => 'boolean',
1242         description => "Specifies whether if given host option is a USB3 device or port (this does currently not work reliably with spice redirection and is then ignored).",
1243         default => 0,
1244     },
1245 };
1246
1247 my $usbdesc = {
1248     optional => 1,
1249     type => 'string', format => $usb_fmt,
1250     description => "Configure an USB device (n is 0 to 4).",
1251 };
1252 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
1253
1254 my $PCIRE = qr/[a-f0-9]{2}:[a-f0-9]{2}(?:\.[a-f0-9])?/;
1255 my $hostpci_fmt = {
1256     host => {
1257         default_key => 1,
1258         type => 'string',
1259         pattern => qr/$PCIRE(;$PCIRE)*/,
1260         format_description => 'HOSTPCIID[;HOSTPCIID2...]',
1261         description => <<EODESCR,
1262 Host PCI device pass through. The PCI ID of a host's PCI device or a list
1263 of PCI virtual functions of the host. HOSTPCIID syntax is:
1264
1265 'bus:dev.func' (hexadecimal numbers)
1266
1267 You can us the 'lspci' command to list existing PCI devices.
1268 EODESCR
1269     },
1270     rombar => {
1271         type => 'boolean',
1272         description =>  "Specify whether or not the device's ROM will be visible in the guest's memory map.",
1273         optional => 1,
1274         default => 1,
1275     },
1276     romfile => {
1277         type => 'string',
1278         pattern => '[^,;]+',
1279         format_description => 'string',
1280         description => "Custom pci device rom filename (must be located in /usr/share/kvm/).",
1281         optional => 1,
1282     },
1283     pcie => {
1284         type => 'boolean',
1285         description =>  "Choose the PCI-express bus (needs the 'q35' machine model).",
1286         optional => 1,
1287         default => 0,
1288     },
1289     'x-vga' => {
1290         type => 'boolean',
1291         description =>  "Enable vfio-vga device support.",
1292         optional => 1,
1293         default => 0,
1294     },
1295     'mdev' => {
1296         type => 'string',
1297         format_description => 'string',
1298         pattern => '[^/\.:]+',
1299         optional => 1,
1300         description => <<EODESCR
1301 The type of mediated device to use.
1302 An instance of this type will be created on startup of the VM and
1303 will be cleaned up when the VM stops.
1304 EODESCR
1305     }
1306 };
1307 PVE::JSONSchema::register_format('pve-qm-hostpci', $hostpci_fmt);
1308
1309 my $hostpcidesc = {
1310         optional => 1,
1311         type => 'string', format => 'pve-qm-hostpci',
1312         description => "Map host PCI devices into guest.",
1313         verbose_description =>  <<EODESCR,
1314 Map host PCI devices into guest.
1315
1316 NOTE: This option allows direct access to host hardware. So it is no longer
1317 possible to migrate such machines - use with special care.
1318
1319 CAUTION: Experimental! User reported problems with this option.
1320 EODESCR
1321 };
1322 PVE::JSONSchema::register_standard_option("pve-qm-hostpci", $hostpcidesc);
1323
1324 my $serialdesc = {
1325         optional => 1,
1326         type => 'string',
1327         pattern => '(/dev/.+|socket)',
1328         description =>  "Create a serial device inside the VM (n is 0 to 3)",
1329         verbose_description =>  <<EODESCR,
1330 Create a serial device inside the VM (n is 0 to 3), and pass through a
1331 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1332 host side (use 'qm terminal' to open a terminal connection).
1333
1334 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines - use with special care.
1335
1336 CAUTION: Experimental! User reported problems with this option.
1337 EODESCR
1338 };
1339
1340 my $paralleldesc= {
1341         optional => 1,
1342         type => 'string',
1343         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
1344         description =>  "Map host parallel devices (n is 0 to 2).",
1345         verbose_description =>  <<EODESCR,
1346 Map host parallel devices (n is 0 to 2).
1347
1348 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1349
1350 CAUTION: Experimental! User reported problems with this option.
1351 EODESCR
1352 };
1353
1354 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
1355     $confdesc->{"parallel$i"} = $paralleldesc;
1356 }
1357
1358 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
1359     $confdesc->{"serial$i"} = $serialdesc;
1360 }
1361
1362 for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
1363     $confdesc->{"hostpci$i"} = $hostpcidesc;
1364 }
1365
1366 for (my $i = 0; $i < $MAX_IDE_DISKS; $i++)  {
1367     $drivename_hash->{"ide$i"} = 1;
1368     $confdesc->{"ide$i"} = $idedesc;
1369 }
1370
1371 for (my $i = 0; $i < $MAX_SATA_DISKS; $i++)  {
1372     $drivename_hash->{"sata$i"} = 1;
1373     $confdesc->{"sata$i"} = $satadesc;
1374 }
1375
1376 for (my $i = 0; $i < $MAX_SCSI_DISKS; $i++)  {
1377     $drivename_hash->{"scsi$i"} = 1;
1378     $confdesc->{"scsi$i"} = $scsidesc ;
1379 }
1380
1381 for (my $i = 0; $i < $MAX_VIRTIO_DISKS; $i++)  {
1382     $drivename_hash->{"virtio$i"} = 1;
1383     $confdesc->{"virtio$i"} = $virtiodesc;
1384 }
1385
1386 $drivename_hash->{efidisk0} = 1;
1387 $confdesc->{efidisk0} = $efidisk_desc;
1388
1389 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
1390     $confdesc->{"usb$i"} = $usbdesc;
1391 }
1392
1393 my $unuseddesc = {
1394     optional => 1,
1395     type => 'string', format => 'pve-volume-id',
1396     description => "Reference to unused volumes. This is used internally, and should not be modified manually.",
1397 };
1398
1399 for (my $i = 0; $i < $MAX_UNUSED_DISKS; $i++)  {
1400     $confdesc->{"unused$i"} = $unuseddesc;
1401 }
1402
1403 my $kvm_api_version = 0;
1404
1405 sub kvm_version {
1406     return $kvm_api_version if $kvm_api_version;
1407
1408     open my $fh, '<', '/dev/kvm'
1409         or return undef;
1410
1411     # 0xae00 => KVM_GET_API_VERSION
1412     $kvm_api_version = ioctl($fh, 0xae00, 0);
1413
1414     return $kvm_api_version;
1415 }
1416
1417 my $kvm_user_version;
1418
1419 sub kvm_user_version {
1420
1421     return $kvm_user_version if $kvm_user_version;
1422
1423     $kvm_user_version = 'unknown';
1424
1425     my $code = sub {
1426         my $line = shift;
1427         if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1428             $kvm_user_version = $2;
1429         }
1430     };
1431
1432     eval { run_command("kvm -version", outfunc => $code); };
1433     warn $@ if $@;
1434
1435     return $kvm_user_version;
1436
1437 }
1438
1439 sub kernel_has_vhost_net {
1440     return -c '/dev/vhost-net';
1441 }
1442
1443 sub valid_drive_names {
1444     # order is important - used to autoselect boot disk
1445     return ((map { "ide$_" } (0 .. ($MAX_IDE_DISKS - 1))),
1446             (map { "scsi$_" } (0 .. ($MAX_SCSI_DISKS - 1))),
1447             (map { "virtio$_" } (0 .. ($MAX_VIRTIO_DISKS - 1))),
1448             (map { "sata$_" } (0 .. ($MAX_SATA_DISKS - 1))),
1449             'efidisk0');
1450 }
1451
1452 sub is_valid_drivename {
1453     my $dev = shift;
1454
1455     return defined($drivename_hash->{$dev});
1456 }
1457
1458 sub option_exists {
1459     my $key = shift;
1460     return defined($confdesc->{$key});
1461 }
1462
1463 sub nic_models {
1464     return $nic_model_list;
1465 }
1466
1467 sub os_list_description {
1468
1469     return {
1470         other => 'Other',
1471         wxp => 'Windows XP',
1472         w2k => 'Windows 2000',
1473         w2k3 =>, 'Windows 2003',
1474         w2k8 => 'Windows 2008',
1475         wvista => 'Windows Vista',
1476         win7 => 'Windows 7',
1477         win8 => 'Windows 8/2012',
1478         win10 => 'Windows 10/2016',
1479         l24 => 'Linux 2.4',
1480         l26 => 'Linux 2.6',
1481     };
1482 }
1483
1484 my $cdrom_path;
1485
1486 sub get_cdrom_path {
1487
1488     return  $cdrom_path if $cdrom_path;
1489
1490     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
1491     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
1492     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
1493 }
1494
1495 sub get_iso_path {
1496     my ($storecfg, $vmid, $cdrom) = @_;
1497
1498     if ($cdrom eq 'cdrom') {
1499         return get_cdrom_path();
1500     } elsif ($cdrom eq 'none') {
1501         return '';
1502     } elsif ($cdrom =~ m|^/|) {
1503         return $cdrom;
1504     } else {
1505         return PVE::Storage::path($storecfg, $cdrom);
1506     }
1507 }
1508
1509 # try to convert old style file names to volume IDs
1510 sub filename_to_volume_id {
1511     my ($vmid, $file, $media) = @_;
1512
1513      if (!($file eq 'none' || $file eq 'cdrom' ||
1514           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1515
1516         return undef if $file =~ m|/|;
1517
1518         if ($media && $media eq 'cdrom') {
1519             $file = "local:iso/$file";
1520         } else {
1521             $file = "local:$vmid/$file";
1522         }
1523     }
1524
1525     return $file;
1526 }
1527
1528 sub verify_media_type {
1529     my ($opt, $vtype, $media) = @_;
1530
1531     return if !$media;
1532
1533     my $etype;
1534     if ($media eq 'disk') {
1535         $etype = 'images';
1536     } elsif ($media eq 'cdrom') {
1537         $etype = 'iso';
1538     } else {
1539         die "internal error";
1540     }
1541
1542     return if ($vtype eq $etype);
1543
1544     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1545 }
1546
1547 sub cleanup_drive_path {
1548     my ($opt, $storecfg, $drive) = @_;
1549
1550     # try to convert filesystem paths to volume IDs
1551
1552     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1553         ($drive->{file} !~ m|^/dev/.+|) &&
1554         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1555         ($drive->{file} !~ m/^\d+$/)) {
1556         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1557         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"}) if !$vtype;
1558         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1559         verify_media_type($opt, $vtype, $drive->{media});
1560         $drive->{file} = $volid;
1561     }
1562
1563     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1564 }
1565
1566 sub parse_hotplug_features {
1567     my ($data) = @_;
1568
1569     my $res = {};
1570
1571     return $res if $data eq '0';
1572
1573     $data = $confdesc->{hotplug}->{default} if $data eq '1';
1574
1575     foreach my $feature (PVE::Tools::split_list($data)) {
1576         if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
1577             $res->{$1} = 1;
1578         } else {
1579             die "invalid hotplug feature '$feature'\n";
1580         }
1581     }
1582     return $res;
1583 }
1584
1585 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1586 sub pve_verify_hotplug_features {
1587     my ($value, $noerr) = @_;
1588
1589     return $value if parse_hotplug_features($value);
1590
1591     return undef if $noerr;
1592
1593     die "unable to parse hotplug option\n";
1594 }
1595
1596 # ideX = [volume=]volume-id[,media=d][,cyls=c,heads=h,secs=s[,trans=t]]
1597 #        [,snapshot=on|off][,cache=on|off][,format=f][,backup=yes|no]
1598 #        [,rerror=ignore|report|stop][,werror=enospc|ignore|report|stop]
1599 #        [,aio=native|threads][,discard=ignore|on][,detect_zeroes=on|off]
1600 #        [,iothread=on][,serial=serial][,model=model]
1601
1602 sub parse_drive {
1603     my ($key, $data) = @_;
1604
1605     my ($interface, $index);
1606
1607     if ($key =~ m/^([^\d]+)(\d+)$/) {
1608         $interface = $1;
1609         $index = $2;
1610     } else {
1611         return undef;
1612     }
1613
1614     my $desc = $key =~ /^unused\d+$/ ? $alldrive_fmt
1615                                      : $confdesc->{$key}->{format};
1616     if (!$desc) {
1617         warn "invalid drive key: $key\n";
1618         return undef;
1619     }
1620     my $res = eval { PVE::JSONSchema::parse_property_string($desc, $data) };
1621     return undef if !$res;
1622     $res->{interface} = $interface;
1623     $res->{index} = $index;
1624
1625     my $error = 0;
1626     foreach my $opt (qw(bps bps_rd bps_wr)) {
1627         if (my $bps = defined(delete $res->{$opt})) {
1628             if (defined($res->{"m$opt"})) {
1629                 warn "both $opt and m$opt specified\n";
1630                 ++$error;
1631                 next;
1632             }
1633             $res->{"m$opt"} = sprintf("%.3f", $bps / (1024*1024.0));
1634         }
1635     }
1636
1637     # can't use the schema's 'requires' because of the mbps* => bps* "transforming aliases"
1638     for my $requirement (
1639         [mbps_max => 'mbps'],
1640         [mbps_rd_max => 'mbps_rd'],
1641         [mbps_wr_max => 'mbps_wr'],
1642         [miops_max => 'miops'],
1643         [miops_rd_max => 'miops_rd'],
1644         [miops_wr_max => 'miops_wr'],
1645         [bps_max_length => 'mbps_max'],
1646         [bps_rd_max_length => 'mbps_rd_max'],
1647         [bps_wr_max_length => 'mbps_wr_max'],
1648         [iops_max_length => 'iops_max'],
1649         [iops_rd_max_length => 'iops_rd_max'],
1650         [iops_wr_max_length => 'iops_wr_max']) {
1651         my ($option, $requires) = @$requirement;
1652         if ($res->{$option} && !$res->{$requires}) {
1653             warn "$option requires $requires\n";
1654             ++$error;
1655         }
1656     }
1657
1658     return undef if $error;
1659
1660     return undef if $res->{mbps_rd} && $res->{mbps};
1661     return undef if $res->{mbps_wr} && $res->{mbps};
1662     return undef if $res->{iops_rd} && $res->{iops};
1663     return undef if $res->{iops_wr} && $res->{iops};
1664
1665     if ($res->{media} && ($res->{media} eq 'cdrom')) {
1666         return undef if $res->{snapshot} || $res->{trans} || $res->{format};
1667         return undef if $res->{heads} || $res->{secs} || $res->{cyls};
1668         return undef if $res->{interface} eq 'virtio';
1669     }
1670
1671     if (my $size = $res->{size}) {
1672         return undef if !defined($res->{size} = PVE::JSONSchema::parse_size($size));
1673     }
1674
1675     return $res;
1676 }
1677
1678 sub print_drive {
1679     my ($vmid, $drive) = @_;
1680     my $data = { %$drive };
1681     delete $data->{$_} for qw(index interface);
1682     return PVE::JSONSchema::print_property_string($data, $alldrive_fmt);
1683 }
1684
1685 sub scsi_inquiry {
1686     my($fh, $noerr) = @_;
1687
1688     my $SG_IO = 0x2285;
1689     my $SG_GET_VERSION_NUM = 0x2282;
1690
1691     my $versionbuf = "\x00" x 8;
1692     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1693     if (!$ret) {
1694         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1695         return undef;
1696     }
1697     my $version = unpack("I", $versionbuf);
1698     if ($version < 30000) {
1699         die "scsi generic interface too old\n"  if !$noerr;
1700         return undef;
1701     }
1702
1703     my $buf = "\x00" x 36;
1704     my $sensebuf = "\x00" x 8;
1705     my $cmd = pack("C x3 C x1", 0x12, 36);
1706
1707     # see /usr/include/scsi/sg.h
1708     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1709
1710     my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1711                       length($sensebuf), 0, length($buf), $buf,
1712                       $cmd, $sensebuf, 6000);
1713
1714     $ret = ioctl($fh, $SG_IO, $packet);
1715     if (!$ret) {
1716         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1717         return undef;
1718     }
1719
1720     my @res = unpack($sg_io_hdr_t, $packet);
1721     if ($res[17] || $res[18]) {
1722         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1723         return undef;
1724     }
1725
1726     my $res = {};
1727     (my $byte0, my $byte1, $res->{vendor},
1728      $res->{product}, $res->{revision}) = unpack("C C x6 A8 A16 A4", $buf);
1729
1730     $res->{removable} = $byte1 & 128 ? 1 : 0;
1731     $res->{type} = $byte0 & 31;
1732
1733     return $res;
1734 }
1735
1736 sub path_is_scsi {
1737     my ($path) = @_;
1738
1739     my $fh = IO::File->new("+<$path") || return undef;
1740     my $res = scsi_inquiry($fh, 1);
1741     close($fh);
1742
1743     return $res;
1744 }
1745
1746 sub machine_type_is_q35 {
1747     my ($conf) = @_;
1748
1749     return $conf->{machine} && ($conf->{machine} =~ m/q35/) ? 1 : 0;
1750 }
1751
1752 sub print_tabletdevice_full {
1753     my ($conf, $arch) = @_;
1754
1755     my $q35 = machine_type_is_q35($conf);
1756
1757     # we use uhci for old VMs because tablet driver was buggy in older qemu
1758     my $usbbus;
1759     if (machine_type_is_q35($conf) || $arch eq 'aarch64') {
1760         $usbbus = 'ehci';
1761     } else {
1762         $usbbus = 'uhci';
1763     }
1764
1765     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1766 }
1767
1768 sub print_keyboarddevice_full {
1769     my ($conf, $arch, $machine) = @_;
1770
1771     return undef if $arch ne 'aarch64';
1772
1773     return "usb-kbd,id=keyboard,bus=ehci.0,port=2";
1774 }
1775
1776 sub print_drivedevice_full {
1777     my ($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type) = @_;
1778
1779     my $device = '';
1780     my $maxdev = 0;
1781
1782     if ($drive->{interface} eq 'virtio') {
1783         my $pciaddr = print_pci_addr("$drive->{interface}$drive->{index}", $bridges, $arch, $machine_type);
1784         $device = "virtio-blk-pci,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}$pciaddr";
1785         $device .= ",iothread=iothread-$drive->{interface}$drive->{index}" if $drive->{iothread};
1786     } elsif ($drive->{interface} eq 'scsi') {
1787
1788         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1789         my $unit = $drive->{index} % $maxdev;
1790         my $devicetype = 'hd';
1791         my $path = '';
1792         if (drive_is_cdrom($drive)) {
1793             $devicetype = 'cd';
1794         } else {
1795             if ($drive->{file} =~ m|^/|) {
1796                 $path = $drive->{file};
1797                 if (my $info = path_is_scsi($path)) {
1798                     if ($info->{type} == 0 && $drive->{scsiblock}) {
1799                         $devicetype = 'block';
1800                     } elsif ($info->{type} == 1) { # tape
1801                         $devicetype = 'generic';
1802                     }
1803                 }
1804             } else {
1805                  $path = PVE::Storage::path($storecfg, $drive->{file});
1806             }
1807
1808             if($path =~ m/^iscsi\:\/\//){
1809                 $devicetype = 'generic';
1810             }
1811         }
1812
1813         if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)){
1814             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1815         } else {
1816             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0,lun=$drive->{index},drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1817         }
1818
1819         if ($drive->{ssd} && ($devicetype eq 'block' || $devicetype eq 'hd')) {
1820             $device .= ",rotation_rate=1";
1821         }
1822         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1823
1824     } elsif ($drive->{interface} eq 'ide' || $drive->{interface} eq 'sata') {
1825         my $maxdev = ($drive->{interface} eq 'sata') ? $MAX_SATA_DISKS : 2;
1826         my $controller = int($drive->{index} / $maxdev);
1827         my $unit = $drive->{index} % $maxdev;
1828         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1829
1830         $device = "ide-$devicetype";
1831         if ($drive->{interface} eq 'ide') {
1832             $device .= ",bus=ide.$controller,unit=$unit";
1833         } else {
1834             $device .= ",bus=ahci$controller.$unit";
1835         }
1836         $device .= ",drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1837
1838         if ($devicetype eq 'hd') {
1839             if (my $model = $drive->{model}) {
1840                 $model = URI::Escape::uri_unescape($model);
1841                 $device .= ",model=$model";
1842             }
1843             if ($drive->{ssd}) {
1844                 $device .= ",rotation_rate=1";
1845             }
1846         }
1847         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1848     } elsif ($drive->{interface} eq 'usb') {
1849         die "implement me";
1850         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1851     } else {
1852         die "unsupported interface type";
1853     }
1854
1855     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1856
1857     if (my $serial = $drive->{serial}) {
1858         $serial = URI::Escape::uri_unescape($serial);
1859         $device .= ",serial=$serial";
1860     }
1861
1862
1863     return $device;
1864 }
1865
1866 sub get_initiator_name {
1867     my $initiator;
1868
1869     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return undef;
1870     while (defined(my $line = <$fh>)) {
1871         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1872         $initiator = $1;
1873         last;
1874     }
1875     $fh->close();
1876
1877     return $initiator;
1878 }
1879
1880 sub print_drive_full {
1881     my ($storecfg, $vmid, $drive) = @_;
1882
1883     my $path;
1884     my $volid = $drive->{file};
1885     my $format;
1886
1887     if (drive_is_cdrom($drive)) {
1888         $path = get_iso_path($storecfg, $vmid, $volid);
1889     } else {
1890         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1891         if ($storeid) {
1892             $path = PVE::Storage::path($storecfg, $volid);
1893             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
1894             $format = qemu_img_format($scfg, $volname);
1895         } else {
1896             $path = $volid;
1897             $format = "raw";
1898         }
1899    }
1900
1901     my $opts = '';
1902     my @qemu_drive_options = qw(heads secs cyls trans media format cache rerror werror aio discard);
1903     foreach my $o (@qemu_drive_options) {
1904         $opts .= ",$o=$drive->{$o}" if defined($drive->{$o});
1905     }
1906
1907     # snapshot only accepts on|off
1908     if (defined($drive->{snapshot})) {
1909         my $v = $drive->{snapshot} ? 'on' : 'off';
1910         $opts .= ",snapshot=$v";
1911     }
1912
1913     foreach my $type (['', '-total'], [_rd => '-read'], [_wr => '-write']) {
1914         my ($dir, $qmpname) = @$type;
1915         if (my $v = $drive->{"mbps$dir"}) {
1916             $opts .= ",throttling.bps$qmpname=".int($v*1024*1024);
1917         }
1918         if (my $v = $drive->{"mbps${dir}_max"}) {
1919             $opts .= ",throttling.bps$qmpname-max=".int($v*1024*1024);
1920         }
1921         if (my $v = $drive->{"bps${dir}_max_length"}) {
1922             $opts .= ",throttling.bps$qmpname-max-length=$v";
1923         }
1924         if (my $v = $drive->{"iops${dir}"}) {
1925             $opts .= ",throttling.iops$qmpname=$v";
1926         }
1927         if (my $v = $drive->{"iops${dir}_max"}) {
1928             $opts .= ",throttling.iops$qmpname-max=$v";
1929         }
1930         if (my $v = $drive->{"iops${dir}_max_length"}) {
1931             $opts .= ",throttling.iops$qmpname-max-length=$v";
1932         }
1933     }
1934
1935     $opts .= ",format=$format" if $format && !$drive->{format};
1936
1937     my $cache_direct = 0;
1938
1939     if (my $cache = $drive->{cache}) {
1940         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1941     } elsif (!drive_is_cdrom($drive)) {
1942         $opts .= ",cache=none";
1943         $cache_direct = 1;
1944     }
1945
1946     # aio native works only with O_DIRECT
1947     if (!$drive->{aio}) {
1948         if($cache_direct) {
1949             $opts .= ",aio=native";
1950         } else {
1951             $opts .= ",aio=threads";
1952         }
1953     }
1954
1955     if (!drive_is_cdrom($drive)) {
1956         my $detectzeroes;
1957         if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
1958             $detectzeroes = 'off';
1959         } elsif ($drive->{discard}) {
1960             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
1961         } else {
1962             # This used to be our default with discard not being specified:
1963             $detectzeroes = 'on';
1964         }
1965         $opts .= ",detect-zeroes=$detectzeroes" if $detectzeroes;
1966     }
1967
1968     my $pathinfo = $path ? "file=$path," : '';
1969
1970     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1971 }
1972
1973 sub print_netdevice_full {
1974     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files, $arch, $machine_type) = @_;
1975
1976     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
1977
1978     my $device = $net->{model};
1979     if ($net->{model} eq 'virtio') {
1980          $device = 'virtio-net-pci';
1981      };
1982
1983     my $pciaddr = print_pci_addr("$netid", $bridges, $arch, $machine_type);
1984     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1985     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
1986         #Consider we have N queues, the number of vectors needed is 2*N + 2 (plus one config interrupt and control vq)
1987         my $vectors = $net->{queues} * 2 + 2;
1988         $tmpstr .= ",vectors=$vectors,mq=on";
1989     }
1990     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
1991
1992     if ($use_old_bios_files) {
1993         my $romfile;
1994         if ($device eq 'virtio-net-pci') {
1995             $romfile = 'pxe-virtio.rom';
1996         } elsif ($device eq 'e1000') {
1997             $romfile = 'pxe-e1000.rom';
1998         } elsif ($device eq 'ne2k') {
1999             $romfile = 'pxe-ne2k_pci.rom';
2000         } elsif ($device eq 'pcnet') {
2001             $romfile = 'pxe-pcnet.rom';
2002         } elsif ($device eq 'rtl8139') {
2003             $romfile = 'pxe-rtl8139.rom';
2004         }
2005         $tmpstr .= ",romfile=$romfile" if $romfile;
2006     }
2007
2008     return $tmpstr;
2009 }
2010
2011 sub print_netdev_full {
2012     my ($vmid, $conf, $arch, $net, $netid, $hotplug) = @_;
2013
2014     my $i = '';
2015     if ($netid =~ m/^net(\d+)$/) {
2016         $i = int($1);
2017     }
2018
2019     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
2020
2021     my $ifname = "tap${vmid}i$i";
2022
2023     # kvm uses TUNSETIFF ioctl, and that limits ifname length
2024     die "interface name '$ifname' is too long (max 15 character)\n"
2025         if length($ifname) >= 16;
2026
2027     my $vhostparam = '';
2028     if (is_native($arch)) {
2029         $vhostparam = ',vhost=on' if kernel_has_vhost_net() && $net->{model} eq 'virtio';
2030     }
2031
2032     my $vmname = $conf->{name} || "vm$vmid";
2033
2034     my $netdev = "";
2035     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
2036
2037     if ($net->{bridge}) {
2038         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script,downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
2039     } else {
2040         $netdev = "type=user,id=$netid,hostname=$vmname";
2041     }
2042
2043     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
2044
2045     return $netdev;
2046 }
2047
2048
2049 sub print_cpu_device {
2050     my ($conf, $id) = @_;
2051
2052     my $kvm = $conf->{kvm} // 1;
2053     my $cpu = $kvm ? "kvm64" : "qemu64";
2054     if (my $cputype = $conf->{cpu}) {
2055         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
2056             or die "Cannot parse cpu description: $cputype\n";
2057         $cpu = $cpuconf->{cputype};
2058     }
2059
2060     my $cores = $conf->{cores} || 1;
2061
2062     my $current_core = ($id - 1) % $cores;
2063     my $current_socket = int(($id - 1 - $current_core)/$cores);
2064
2065     return "$cpu-x86_64-cpu,id=cpu$id,socket-id=$current_socket,core-id=$current_core,thread-id=0";
2066 }
2067
2068 my $vga_map = {
2069     'cirrus' => 'cirrus-vga',
2070     'std' => 'VGA',
2071     'vmware' => 'vmware-svga',
2072     'virtio' => 'virtio-vga',
2073 };
2074
2075 sub print_vga_device {
2076     my ($conf, $vga, $arch, $machine, $id, $qxlnum, $bridges) = @_;
2077
2078     my $type = $vga_map->{$vga->{type}};
2079     if ($arch eq 'aarch64' && defined($type) && $type eq 'virtio-vga') {
2080         $type = 'virtio-gpu';
2081     }
2082     my $vgamem_mb = $vga->{memory};
2083     if ($qxlnum) {
2084         $type = $id ? 'qxl' : 'qxl-vga';
2085     }
2086     die "no devicetype for $vga->{type}\n" if !$type;
2087
2088     my $memory = "";
2089     if ($vgamem_mb) {
2090         if ($vga->{type} eq 'virtio') {
2091             my $bytes = PVE::Tools::convert_size($vgamem_mb, "mb" => "b");
2092             $memory = ",max_hostmem=$bytes";
2093         } elsif ($qxlnum) {
2094             # from https://www.spice-space.org/multiple-monitors.html
2095             $memory = ",vgamem_mb=$vga->{memory}";
2096             my $ram = $vgamem_mb * 4;
2097             my $vram = $vgamem_mb * 2;
2098             $memory .= ",ram_size_mb=$ram,vram_size_mb=$vram";
2099         } else {
2100             $memory = ",vgamem_mb=$vga->{memory}";
2101         }
2102     } elsif ($qxlnum && $id) {
2103         $memory = ",ram_size=67108864,vram_size=33554432";
2104     }
2105
2106     my $q35 = machine_type_is_q35($conf);
2107     my $vgaid = "vga" . ($id // '');
2108     my $pciaddr;
2109
2110     if ($q35 && $vgaid eq 'vga') {
2111         # the first display uses pcie.0 bus on q35 machines
2112         $pciaddr = print_pcie_addr($vgaid, $bridges, $arch, $machine);
2113     } else {
2114         $pciaddr = print_pci_addr($vgaid, $bridges, $arch, $machine);
2115     }
2116
2117     return "$type,id=${vgaid}${memory}${pciaddr}";
2118 }
2119
2120 sub drive_is_cloudinit {
2121     my ($drive) = @_;
2122     return $drive->{file} =~ m@[:/]vm-\d+-cloudinit(?:\.$QEMU_FORMAT_RE)?$@;
2123 }
2124
2125 sub drive_is_cdrom {
2126     my ($drive, $exclude_cloudinit) = @_;
2127
2128     return 0 if $exclude_cloudinit && drive_is_cloudinit($drive);
2129
2130     return $drive && $drive->{media} && ($drive->{media} eq 'cdrom');
2131
2132 }
2133
2134 sub parse_number_sets {
2135     my ($set) = @_;
2136     my $res = [];
2137     foreach my $part (split(/;/, $set)) {
2138         if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
2139             die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
2140             push @$res, [ $1, $2 ];
2141         } else {
2142             die "invalid range: $part\n";
2143         }
2144     }
2145     return $res;
2146 }
2147
2148 sub parse_numa {
2149     my ($data) = @_;
2150
2151     my $res = PVE::JSONSchema::parse_property_string($numa_fmt, $data);
2152     $res->{cpus} = parse_number_sets($res->{cpus}) if defined($res->{cpus});
2153     $res->{hostnodes} = parse_number_sets($res->{hostnodes}) if defined($res->{hostnodes});
2154     return $res;
2155 }
2156
2157 sub parse_hostpci {
2158     my ($value) = @_;
2159
2160     return undef if !$value;
2161
2162     my $res = PVE::JSONSchema::parse_property_string($hostpci_fmt, $value);
2163
2164     my @idlist = split(/;/, $res->{host});
2165     delete $res->{host};
2166     foreach my $id (@idlist) {
2167         if ($id =~ m/\./) { # full id 00:00.1
2168             push @{$res->{pciid}}, {
2169                 id => $id,
2170             };
2171         } else { # partial id 00:00
2172             $res->{pciid} = PVE::SysFSTools::lspci($id);
2173         }
2174     }
2175     return $res;
2176 }
2177
2178 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
2179 sub parse_net {
2180     my ($data) = @_;
2181
2182     my $res = eval { PVE::JSONSchema::parse_property_string($net_fmt, $data) };
2183     if ($@) {
2184         warn $@;
2185         return undef;
2186     }
2187     if (!defined($res->{macaddr})) {
2188         my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
2189         $res->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix});
2190     }
2191     return $res;
2192 }
2193
2194 # ipconfigX ip=cidr,gw=ip,ip6=cidr,gw6=ip
2195 sub parse_ipconfig {
2196     my ($data) = @_;
2197
2198     my $res = eval { PVE::JSONSchema::parse_property_string($ipconfig_fmt, $data) };
2199     if ($@) {
2200         warn $@;
2201         return undef;
2202     }
2203
2204     if ($res->{gw} && !$res->{ip}) {
2205         warn 'gateway specified without specifying an IP address';
2206         return undef;
2207     }
2208     if ($res->{gw6} && !$res->{ip6}) {
2209         warn 'IPv6 gateway specified without specifying an IPv6 address';
2210         return undef;
2211     }
2212     if ($res->{gw} && $res->{ip} eq 'dhcp') {
2213         warn 'gateway specified together with DHCP';
2214         return undef;
2215     }
2216     if ($res->{gw6} && $res->{ip6} !~ /^$IPV6RE/) {
2217         # gw6 + auto/dhcp
2218         warn "IPv6 gateway specified together with $res->{ip6} address";
2219         return undef;
2220     }
2221
2222     if (!$res->{ip} && !$res->{ip6}) {
2223         return { ip => 'dhcp', ip6 => 'dhcp' };
2224     }
2225
2226     return $res;
2227 }
2228
2229 sub print_net {
2230     my $net = shift;
2231
2232     return PVE::JSONSchema::print_property_string($net, $net_fmt);
2233 }
2234
2235 sub add_random_macs {
2236     my ($settings) = @_;
2237
2238     foreach my $opt (keys %$settings) {
2239         next if $opt !~ m/^net(\d+)$/;
2240         my $net = parse_net($settings->{$opt});
2241         next if !$net;
2242         $settings->{$opt} = print_net($net);
2243     }
2244 }
2245
2246 sub vm_is_volid_owner {
2247     my ($storecfg, $vmid, $volid) = @_;
2248
2249     if ($volid !~  m|^/|) {
2250         my ($path, $owner);
2251         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
2252         if ($owner && ($owner == $vmid)) {
2253             return 1;
2254         }
2255     }
2256
2257     return undef;
2258 }
2259
2260 sub split_flagged_list {
2261     my $text = shift || '';
2262     $text =~ s/[,;]/ /g;
2263     $text =~ s/^\s+//;
2264     return { map { /^(!?)(.*)$/ && ($2, $1) } ($text =~ /\S+/g) };
2265 }
2266
2267 sub join_flagged_list {
2268     my ($how, $lst) = @_;
2269     join $how, map { $lst->{$_} . $_ } keys %$lst;
2270 }
2271
2272 sub vmconfig_delete_pending_option {
2273     my ($conf, $key, $force) = @_;
2274
2275     delete $conf->{pending}->{$key};
2276     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
2277     $pending_delete_hash->{$key} = $force ? '!' : '';
2278     $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
2279 }
2280
2281 sub vmconfig_undelete_pending_option {
2282     my ($conf, $key) = @_;
2283
2284     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
2285     delete $pending_delete_hash->{$key};
2286
2287     if (%$pending_delete_hash) {
2288         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
2289     } else {
2290         delete $conf->{pending}->{delete};
2291     }
2292 }
2293
2294 sub vmconfig_register_unused_drive {
2295     my ($storecfg, $vmid, $conf, $drive) = @_;
2296
2297     if (drive_is_cloudinit($drive)) {
2298         eval { PVE::Storage::vdisk_free($storecfg, $drive->{file}) };
2299         warn $@ if $@;
2300     } elsif (!drive_is_cdrom($drive)) {
2301         my $volid = $drive->{file};
2302         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
2303             PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
2304         }
2305     }
2306 }
2307
2308 sub vmconfig_cleanup_pending {
2309     my ($conf) = @_;
2310
2311     # remove pending changes when nothing changed
2312     my $changes;
2313     foreach my $opt (keys %{$conf->{pending}}) {
2314         if (defined($conf->{$opt}) && ($conf->{pending}->{$opt} eq  $conf->{$opt})) {
2315             $changes = 1;
2316             delete $conf->{pending}->{$opt};
2317         }
2318     }
2319
2320     my $current_delete_hash = split_flagged_list($conf->{pending}->{delete});
2321     my $pending_delete_hash = {};
2322     while (my ($opt, $force) = each %$current_delete_hash) {
2323         if (defined($conf->{$opt})) {
2324             $pending_delete_hash->{$opt} = $force;
2325         } else {
2326             $changes = 1;
2327         }
2328     }
2329
2330     if (%$pending_delete_hash) {
2331         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
2332     } else {
2333         delete $conf->{pending}->{delete};
2334     }
2335
2336     return $changes;
2337 }
2338
2339 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str]
2340 my $smbios1_fmt = {
2341     uuid => {
2342         type => 'string',
2343         pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
2344         format_description => 'UUID',
2345         description => "Set SMBIOS1 UUID.",
2346         optional => 1,
2347     },
2348     version => {
2349         type => 'string',
2350         pattern => '\S+',
2351         format_description => 'string',
2352         description => "Set SMBIOS1 version.",
2353         optional => 1,
2354     },
2355     serial => {
2356         type => 'string',
2357         pattern => '\S+',
2358         format_description => 'string',
2359         description => "Set SMBIOS1 serial number.",
2360         optional => 1,
2361     },
2362     manufacturer => {
2363         type => 'string',
2364         pattern => '\S+',
2365         format_description => 'string',
2366         description => "Set SMBIOS1 manufacturer.",
2367         optional => 1,
2368     },
2369     product => {
2370         type => 'string',
2371         pattern => '\S+',
2372         format_description => 'string',
2373         description => "Set SMBIOS1 product ID.",
2374         optional => 1,
2375     },
2376     sku => {
2377         type => 'string',
2378         pattern => '\S+',
2379         format_description => 'string',
2380         description => "Set SMBIOS1 SKU string.",
2381         optional => 1,
2382     },
2383     family => {
2384         type => 'string',
2385         pattern => '\S+',
2386         format_description => 'string',
2387         description => "Set SMBIOS1 family string.",
2388         optional => 1,
2389     },
2390 };
2391
2392 sub parse_smbios1 {
2393     my ($data) = @_;
2394
2395     my $res = eval { PVE::JSONSchema::parse_property_string($smbios1_fmt, $data) };
2396     warn $@ if $@;
2397     return $res;
2398 }
2399
2400 sub print_smbios1 {
2401     my ($smbios1) = @_;
2402     return PVE::JSONSchema::print_property_string($smbios1, $smbios1_fmt);
2403 }
2404
2405 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_fmt);
2406
2407 PVE::JSONSchema::register_format('pve-qm-bootdisk', \&verify_bootdisk);
2408 sub verify_bootdisk {
2409     my ($value, $noerr) = @_;
2410
2411     return $value if is_valid_drivename($value);
2412
2413     return undef if $noerr;
2414
2415     die "invalid boot disk '$value'\n";
2416 }
2417
2418 sub parse_watchdog {
2419     my ($value) = @_;
2420
2421     return undef if !$value;
2422
2423     my $res = eval { PVE::JSONSchema::parse_property_string($watchdog_fmt, $value) };
2424     warn $@ if $@;
2425     return $res;
2426 }
2427
2428 sub parse_guest_agent {
2429     my ($value) = @_;
2430
2431     return {} if !defined($value->{agent});
2432
2433     my $res = eval { PVE::JSONSchema::parse_property_string($agent_fmt, $value->{agent}) };
2434     warn $@ if $@;
2435
2436     # if the agent is disabled ignore the other potentially set properties
2437     return {} if !$res->{enabled};
2438     return $res;
2439 }
2440
2441 sub parse_vga {
2442     my ($value) = @_;
2443
2444     return {} if !$value;
2445     my $res = eval { PVE::JSONSchema::parse_property_string($vga_fmt, $value) };
2446     warn $@ if $@;
2447     return $res;
2448 }
2449
2450 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
2451 sub verify_usb_device {
2452     my ($value, $noerr) = @_;
2453
2454     return $value if parse_usb_device($value);
2455
2456     return undef if $noerr;
2457
2458     die "unable to parse usb device\n";
2459 }
2460
2461 # add JSON properties for create and set function
2462 sub json_config_properties {
2463     my $prop = shift;
2464
2465     foreach my $opt (keys %$confdesc) {
2466         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate' || $opt eq 'runningmachine';
2467         $prop->{$opt} = $confdesc->{$opt};
2468     }
2469
2470     return $prop;
2471 }
2472
2473 # return copy of $confdesc_cloudinit to generate documentation
2474 sub cloudinit_config_properties {
2475
2476     return dclone($confdesc_cloudinit);
2477 }
2478
2479 sub check_type {
2480     my ($key, $value) = @_;
2481
2482     die "unknown setting '$key'\n" if !$confdesc->{$key};
2483
2484     my $type = $confdesc->{$key}->{type};
2485
2486     if (!defined($value)) {
2487         die "got undefined value\n";
2488     }
2489
2490     if ($value =~ m/[\n\r]/) {
2491         die "property contains a line feed\n";
2492     }
2493
2494     if ($type eq 'boolean') {
2495         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
2496         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
2497         die "type check ('boolean') failed - got '$value'\n";
2498     } elsif ($type eq 'integer') {
2499         return int($1) if $value =~ m/^(\d+)$/;
2500         die "type check ('integer') failed - got '$value'\n";
2501     } elsif ($type eq 'number') {
2502         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
2503         die "type check ('number') failed - got '$value'\n";
2504     } elsif ($type eq 'string') {
2505         if (my $fmt = $confdesc->{$key}->{format}) {
2506             PVE::JSONSchema::check_format($fmt, $value);
2507             return $value;
2508         }
2509         $value =~ s/^\"(.*)\"$/$1/;
2510         return $value;
2511     } else {
2512         die "internal error"
2513     }
2514 }
2515
2516 sub touch_config {
2517     my ($vmid) = @_;
2518
2519     my $conf = PVE::QemuConfig->config_file($vmid);
2520     utime undef, undef, $conf;
2521 }
2522
2523 sub destroy_vm {
2524     my ($storecfg, $vmid, $keep_empty_config, $skiplock) = @_;
2525
2526     my $conffile = PVE::QemuConfig->config_file($vmid);
2527
2528     my $conf = PVE::QemuConfig->load_config($vmid);
2529
2530     PVE::QemuConfig->check_lock($conf) if !$skiplock;
2531
2532     if ($conf->{template}) {
2533         # check if any base image is still used by a linked clone
2534         foreach_drive($conf, sub {
2535                 my ($ds, $drive) = @_;
2536
2537                 return if drive_is_cdrom($drive);
2538
2539                 my $volid = $drive->{file};
2540
2541                 return if !$volid || $volid =~ m|^/|;
2542
2543                 die "base volume '$volid' is still in use by linked cloned\n"
2544                     if PVE::Storage::volume_is_base_and_used($storecfg, $volid);
2545
2546         });
2547     }
2548
2549     # only remove disks owned by this VM
2550     foreach_drive($conf, sub {
2551         my ($ds, $drive) = @_;
2552
2553         return if drive_is_cdrom($drive, 1);
2554
2555         my $volid = $drive->{file};
2556
2557         return if !$volid || $volid =~ m|^/|;
2558
2559         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
2560         return if !$path || !$owner || ($owner != $vmid);
2561
2562         eval {
2563             PVE::Storage::vdisk_free($storecfg, $volid);
2564         };
2565         warn "Could not remove disk '$volid', check manually: $@" if $@;
2566
2567     });
2568
2569     if ($keep_empty_config) {
2570         PVE::Tools::file_set_contents($conffile, "memory: 128\n");
2571     } else {
2572         unlink $conffile;
2573     }
2574
2575     # also remove unused disk
2576     eval {
2577         my $dl = PVE::Storage::vdisk_list($storecfg, undef, $vmid);
2578
2579         eval {
2580             PVE::Storage::foreach_volid($dl, sub {
2581                 my ($volid, $sid, $volname, $d) = @_;
2582                 PVE::Storage::vdisk_free($storecfg, $volid);
2583             });
2584         };
2585         warn $@ if $@;
2586
2587     };
2588     warn $@ if $@;
2589 }
2590
2591 sub parse_vm_config {
2592     my ($filename, $raw) = @_;
2593
2594     return undef if !defined($raw);
2595
2596     my $res = {
2597         digest => Digest::SHA::sha1_hex($raw),
2598         snapshots => {},
2599         pending => {},
2600     };
2601
2602     $filename =~ m|/qemu-server/(\d+)\.conf$|
2603         || die "got strange filename '$filename'";
2604
2605     my $vmid = $1;
2606
2607     my $conf = $res;
2608     my $descr;
2609     my $section = '';
2610
2611     my @lines = split(/\n/, $raw);
2612     foreach my $line (@lines) {
2613         next if $line =~ m/^\s*$/;
2614
2615         if ($line =~ m/^\[PENDING\]\s*$/i) {
2616             $section = 'pending';
2617             if (defined($descr)) {
2618                 $descr =~ s/\s+$//;
2619                 $conf->{description} = $descr;
2620             }
2621             $descr = undef;
2622             $conf = $res->{$section} = {};
2623             next;
2624
2625         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2626             $section = $1;
2627             if (defined($descr)) {
2628                 $descr =~ s/\s+$//;
2629                 $conf->{description} = $descr;
2630             }
2631             $descr = undef;
2632             $conf = $res->{snapshots}->{$section} = {};
2633             next;
2634         }
2635
2636         if ($line =~ m/^\#(.*)\s*$/) {
2637             $descr = '' if !defined($descr);
2638             $descr .= PVE::Tools::decode_text($1) . "\n";
2639             next;
2640         }
2641
2642         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2643             $descr = '' if !defined($descr);
2644             $descr .= PVE::Tools::decode_text($2);
2645         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2646             $conf->{snapstate} = $1;
2647         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2648             my $key = $1;
2649             my $value = $2;
2650             $conf->{$key} = $value;
2651         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2652             my $value = $1;
2653             if ($section eq 'pending') {
2654                 $conf->{delete} = $value; # we parse this later
2655             } else {
2656                 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
2657             }
2658         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(.+?)\s*$/) {
2659             my $key = $1;
2660             my $value = $2;
2661             eval { $value = check_type($key, $value); };
2662             if ($@) {
2663                 warn "vm $vmid - unable to parse value of '$key' - $@";
2664             } else {
2665                 $key = 'ide2' if $key eq 'cdrom';
2666                 my $fmt = $confdesc->{$key}->{format};
2667                 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2668                     my $v = parse_drive($key, $value);
2669                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2670                         $v->{file} = $volid;
2671                         $value = print_drive($vmid, $v);
2672                     } else {
2673                         warn "vm $vmid - unable to parse value of '$key'\n";
2674                         next;
2675                     }
2676                 }
2677
2678                 $conf->{$key} = $value;
2679             }
2680         }
2681     }
2682
2683     if (defined($descr)) {
2684         $descr =~ s/\s+$//;
2685         $conf->{description} = $descr;
2686     }
2687     delete $res->{snapstate}; # just to be sure
2688
2689     return $res;
2690 }
2691
2692 sub write_vm_config {
2693     my ($filename, $conf) = @_;
2694
2695     delete $conf->{snapstate}; # just to be sure
2696
2697     if ($conf->{cdrom}) {
2698         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2699         $conf->{ide2} = $conf->{cdrom};
2700         delete $conf->{cdrom};
2701     }
2702
2703     # we do not use 'smp' any longer
2704     if ($conf->{sockets}) {
2705         delete $conf->{smp};
2706     } elsif ($conf->{smp}) {
2707         $conf->{sockets} = $conf->{smp};
2708         delete $conf->{cores};
2709         delete $conf->{smp};
2710     }
2711
2712     my $used_volids = {};
2713
2714     my $cleanup_config = sub {
2715         my ($cref, $pending, $snapname) = @_;
2716
2717         foreach my $key (keys %$cref) {
2718             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2719                 $key eq 'snapstate' || $key eq 'pending';
2720             my $value = $cref->{$key};
2721             if ($key eq 'delete') {
2722                 die "propertry 'delete' is only allowed in [PENDING]\n"
2723                     if !$pending;
2724                 # fixme: check syntax?
2725                 next;
2726             }
2727             eval { $value = check_type($key, $value); };
2728             die "unable to parse value of '$key' - $@" if $@;
2729
2730             $cref->{$key} = $value;
2731
2732             if (!$snapname && is_valid_drivename($key)) {
2733                 my $drive = parse_drive($key, $value);
2734                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2735             }
2736         }
2737     };
2738
2739     &$cleanup_config($conf);
2740
2741     &$cleanup_config($conf->{pending}, 1);
2742
2743     foreach my $snapname (keys %{$conf->{snapshots}}) {
2744         die "internal error" if $snapname eq 'pending';
2745         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2746     }
2747
2748     # remove 'unusedX' settings if we re-add a volume
2749     foreach my $key (keys %$conf) {
2750         my $value = $conf->{$key};
2751         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2752             delete $conf->{$key};
2753         }
2754     }
2755
2756     my $generate_raw_config = sub {
2757         my ($conf, $pending) = @_;
2758
2759         my $raw = '';
2760
2761         # add description as comment to top of file
2762         if (defined(my $descr = $conf->{description})) {
2763             if ($descr) {
2764                 foreach my $cl (split(/\n/, $descr)) {
2765                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2766                 }
2767             } else {
2768                 $raw .= "#\n" if $pending;
2769             }
2770         }
2771
2772         foreach my $key (sort keys %$conf) {
2773             next if $key eq 'digest' || $key eq 'description' || $key eq 'pending' || $key eq 'snapshots';
2774             $raw .= "$key: $conf->{$key}\n";
2775         }
2776         return $raw;
2777     };
2778
2779     my $raw = &$generate_raw_config($conf);
2780
2781     if (scalar(keys %{$conf->{pending}})){
2782         $raw .= "\n[PENDING]\n";
2783         $raw .= &$generate_raw_config($conf->{pending}, 1);
2784     }
2785
2786     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2787         $raw .= "\n[$snapname]\n";
2788         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2789     }
2790
2791     return $raw;
2792 }
2793
2794 sub load_defaults {
2795
2796     my $res = {};
2797
2798     # we use static defaults from our JSON schema configuration
2799     foreach my $key (keys %$confdesc) {
2800         if (defined(my $default = $confdesc->{$key}->{default})) {
2801             $res->{$key} = $default;
2802         }
2803     }
2804
2805     return $res;
2806 }
2807
2808 sub config_list {
2809     my $vmlist = PVE::Cluster::get_vmlist();
2810     my $res = {};
2811     return $res if !$vmlist || !$vmlist->{ids};
2812     my $ids = $vmlist->{ids};
2813
2814     foreach my $vmid (keys %$ids) {
2815         my $d = $ids->{$vmid};
2816         next if !$d->{node} || $d->{node} ne $nodename;
2817         next if !$d->{type} || $d->{type} ne 'qemu';
2818         $res->{$vmid}->{exists} = 1;
2819     }
2820     return $res;
2821 }
2822
2823 # test if VM uses local resources (to prevent migration)
2824 sub check_local_resources {
2825     my ($conf, $noerr) = @_;
2826
2827     my $loc_res = 0;
2828
2829     $loc_res = 1 if $conf->{hostusb}; # old syntax
2830     $loc_res = 1 if $conf->{hostpci}; # old syntax
2831
2832     $loc_res = 1 if $conf->{ivshmem};
2833
2834     foreach my $k (keys %$conf) {
2835         next if $k =~ m/^usb/ && ($conf->{$k} eq 'spice');
2836         # sockets are safe: they will recreated be on the target side post-migrate
2837         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2838         $loc_res = 1 if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2839     }
2840
2841     die "VM uses local resources\n" if $loc_res && !$noerr;
2842
2843     return $loc_res;
2844 }
2845
2846 # check if used storages are available on all nodes (use by migrate)
2847 sub check_storage_availability {
2848     my ($storecfg, $conf, $node) = @_;
2849
2850     foreach_drive($conf, sub {
2851         my ($ds, $drive) = @_;
2852
2853         my $volid = $drive->{file};
2854         return if !$volid;
2855
2856         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2857         return if !$sid;
2858
2859         # check if storage is available on both nodes
2860         my $scfg = PVE::Storage::storage_check_node($storecfg, $sid);
2861         PVE::Storage::storage_check_node($storecfg, $sid, $node);
2862    });
2863 }
2864
2865 # list nodes where all VM images are available (used by has_feature API)
2866 sub shared_nodes {
2867     my ($conf, $storecfg) = @_;
2868
2869     my $nodelist = PVE::Cluster::get_nodelist();
2870     my $nodehash = { map { $_ => 1 } @$nodelist };
2871     my $nodename = PVE::INotify::nodename();
2872
2873     foreach_drive($conf, sub {
2874         my ($ds, $drive) = @_;
2875
2876         my $volid = $drive->{file};
2877         return if !$volid;
2878
2879         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2880         if ($storeid) {
2881             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2882             if ($scfg->{disable}) {
2883                 $nodehash = {};
2884             } elsif (my $avail = $scfg->{nodes}) {
2885                 foreach my $node (keys %$nodehash) {
2886                     delete $nodehash->{$node} if !$avail->{$node};
2887                 }
2888             } elsif (!$scfg->{shared}) {
2889                 foreach my $node (keys %$nodehash) {
2890                     delete $nodehash->{$node} if $node ne $nodename
2891                 }
2892             }
2893         }
2894     });
2895
2896     return $nodehash
2897 }
2898
2899 sub check_cmdline {
2900     my ($pidfile, $pid) = @_;
2901
2902     my $fh = IO::File->new("/proc/$pid/cmdline", "r");
2903     if (defined($fh)) {
2904         my $line = <$fh>;
2905         $fh->close;
2906         return undef if !$line;
2907         my @param = split(/\0/, $line);
2908
2909         my $cmd = $param[0];
2910         return if !$cmd || ($cmd !~ m|kvm$| && $cmd !~ m@(?:^|/)qemu-system-[^/]+$@);
2911
2912         for (my $i = 0; $i < scalar (@param); $i++) {
2913             my $p = $param[$i];
2914             next if !$p;
2915             if (($p eq '-pidfile') || ($p eq '--pidfile')) {
2916                 my $p = $param[$i+1];
2917                 return 1 if $p && ($p eq $pidfile);
2918                 return undef;
2919             }
2920         }
2921     }
2922     return undef;
2923 }
2924
2925 sub check_running {
2926     my ($vmid, $nocheck, $node) = @_;
2927
2928     my $filename = PVE::QemuConfig->config_file($vmid, $node);
2929
2930     die "unable to find configuration file for VM $vmid - no such machine\n"
2931         if !$nocheck && ! -f $filename;
2932
2933     my $pidfile = pidfile_name($vmid);
2934
2935     if (my $fd = IO::File->new("<$pidfile")) {
2936         my $st = stat($fd);
2937         my $line = <$fd>;
2938         close($fd);
2939
2940         my $mtime = $st->mtime;
2941         if ($mtime > time()) {
2942             warn "file '$filename' modified in future\n";
2943         }
2944
2945         if ($line =~ m/^(\d+)$/) {
2946             my $pid = $1;
2947             if (check_cmdline($pidfile, $pid)) {
2948                 if (my $pinfo = PVE::ProcFSTools::check_process_running($pid)) {
2949                     return $pid;
2950                 }
2951             }
2952         }
2953     }
2954
2955     return undef;
2956 }
2957
2958 sub vzlist {
2959
2960     my $vzlist = config_list();
2961
2962     my $fd = IO::Dir->new($var_run_tmpdir) || return $vzlist;
2963
2964     while (defined(my $de = $fd->read)) {
2965         next if $de !~ m/^(\d+)\.pid$/;
2966         my $vmid = $1;
2967         next if !defined($vzlist->{$vmid});
2968         if (my $pid = check_running($vmid)) {
2969             $vzlist->{$vmid}->{pid} = $pid;
2970         }
2971     }
2972
2973     return $vzlist;
2974 }
2975
2976 sub disksize {
2977     my ($storecfg, $conf) = @_;
2978
2979     my $bootdisk = $conf->{bootdisk};
2980     return undef if !$bootdisk;
2981     return undef if !is_valid_drivename($bootdisk);
2982
2983     return undef if !$conf->{$bootdisk};
2984
2985     my $drive = parse_drive($bootdisk, $conf->{$bootdisk});
2986     return undef if !defined($drive);
2987
2988     return undef if drive_is_cdrom($drive);
2989
2990     my $volid = $drive->{file};
2991     return undef if !$volid;
2992
2993     return $drive->{size};
2994 }
2995
2996 our $vmstatus_return_properties = {
2997     vmid => get_standard_option('pve-vmid'),
2998     status => {
2999         description => "Qemu process status.",
3000         type => 'string',
3001         enum => ['stopped', 'running'],
3002     },
3003     maxmem => {
3004         description => "Maximum memory in bytes.",
3005         type => 'integer',
3006         optional => 1,
3007         renderer => 'bytes',
3008     },
3009     maxdisk => {
3010         description => "Root disk size in bytes.",
3011         type => 'integer',
3012         optional => 1,
3013         renderer => 'bytes',
3014     },
3015     name => {
3016         description => "VM name.",
3017         type => 'string',
3018         optional => 1,
3019     },
3020     qmpstatus => {
3021         description => "Qemu QMP agent status.",
3022         type => 'string',
3023         optional => 1,
3024     },
3025     pid => {
3026         description => "PID of running qemu process.",
3027         type => 'integer',
3028         optional => 1,
3029     },
3030     uptime => {
3031         description => "Uptime.",
3032         type => 'integer',
3033         optional => 1,
3034         renderer => 'duration',
3035     },
3036     cpus => {
3037         description => "Maximum usable CPUs.",
3038         type => 'number',
3039         optional => 1,
3040     },
3041 };
3042
3043 my $last_proc_pid_stat;
3044
3045 # get VM status information
3046 # This must be fast and should not block ($full == false)
3047 # We only query KVM using QMP if $full == true (this can be slow)
3048 sub vmstatus {
3049     my ($opt_vmid, $full) = @_;
3050
3051     my $res = {};
3052
3053     my $storecfg = PVE::Storage::config();
3054
3055     my $list = vzlist();
3056     my $defaults = load_defaults();
3057
3058     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
3059
3060     my $cpucount = $cpuinfo->{cpus} || 1;
3061
3062     foreach my $vmid (keys %$list) {
3063         next if $opt_vmid && ($vmid ne $opt_vmid);
3064
3065         my $cfspath = PVE::QemuConfig->cfs_config_path($vmid);
3066         my $conf = PVE::Cluster::cfs_read_file($cfspath) || {};
3067
3068         my $d = { vmid => $vmid };
3069         $d->{pid} = $list->{$vmid}->{pid};
3070
3071         # fixme: better status?
3072         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
3073
3074         my $size = disksize($storecfg, $conf);
3075         if (defined($size)) {
3076             $d->{disk} = 0; # no info available
3077             $d->{maxdisk} = $size;
3078         } else {
3079             $d->{disk} = 0;
3080             $d->{maxdisk} = 0;
3081         }
3082
3083         $d->{cpus} = ($conf->{sockets} || $defaults->{sockets})
3084             * ($conf->{cores} || $defaults->{cores});
3085         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
3086         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
3087
3088         $d->{name} = $conf->{name} || "VM $vmid";
3089         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024)
3090             : $defaults->{memory}*(1024*1024);
3091
3092         if ($conf->{balloon}) {
3093             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
3094             $d->{shares} = defined($conf->{shares}) ? $conf->{shares}
3095                 : $defaults->{shares};
3096         }
3097
3098         $d->{uptime} = 0;
3099         $d->{cpu} = 0;
3100         $d->{mem} = 0;
3101
3102         $d->{netout} = 0;
3103         $d->{netin} = 0;
3104
3105         $d->{diskread} = 0;
3106         $d->{diskwrite} = 0;
3107
3108         $d->{template} = PVE::QemuConfig->is_template($conf);
3109
3110         $d->{serial} = 1 if conf_has_serial($conf);
3111
3112         $res->{$vmid} = $d;
3113     }
3114
3115     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
3116     foreach my $dev (keys %$netdev) {
3117         next if $dev !~ m/^tap([1-9]\d*)i/;
3118         my $vmid = $1;
3119         my $d = $res->{$vmid};
3120         next if !$d;
3121
3122         $d->{netout} += $netdev->{$dev}->{receive};
3123         $d->{netin} += $netdev->{$dev}->{transmit};
3124
3125         if ($full) {
3126             $d->{nics}->{$dev}->{netout} = $netdev->{$dev}->{receive};
3127             $d->{nics}->{$dev}->{netin} = $netdev->{$dev}->{transmit};
3128         }
3129
3130     }
3131
3132     my $ctime = gettimeofday;
3133
3134     foreach my $vmid (keys %$list) {
3135
3136         my $d = $res->{$vmid};
3137         my $pid = $d->{pid};
3138         next if !$pid;
3139
3140         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
3141         next if !$pstat; # not running
3142
3143         my $used = $pstat->{utime} + $pstat->{stime};
3144
3145         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
3146
3147         if ($pstat->{vsize}) {
3148             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
3149         }
3150
3151         my $old = $last_proc_pid_stat->{$pid};
3152         if (!$old) {
3153             $last_proc_pid_stat->{$pid} = {
3154                 time => $ctime,
3155                 used => $used,
3156                 cpu => 0,
3157             };
3158             next;
3159         }
3160
3161         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
3162
3163         if ($dtime > 1000) {
3164             my $dutime = $used -  $old->{used};
3165
3166             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
3167             $last_proc_pid_stat->{$pid} = {
3168                 time => $ctime,
3169                 used => $used,
3170                 cpu => $d->{cpu},
3171             };
3172         } else {
3173             $d->{cpu} = $old->{cpu};
3174         }
3175     }
3176
3177     return $res if !$full;
3178
3179     my $qmpclient = PVE::QMPClient->new();
3180
3181     my $ballooncb = sub {
3182         my ($vmid, $resp) = @_;
3183
3184         my $info = $resp->{'return'};
3185         return if !$info->{max_mem};
3186
3187         my $d = $res->{$vmid};
3188
3189         # use memory assigned to VM
3190         $d->{maxmem} = $info->{max_mem};
3191         $d->{balloon} = $info->{actual};
3192
3193         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
3194             $d->{mem} = $info->{total_mem} - $info->{free_mem};
3195             $d->{freemem} = $info->{free_mem};
3196         }
3197
3198         $d->{ballooninfo} = $info;
3199     };
3200
3201     my $blockstatscb = sub {
3202         my ($vmid, $resp) = @_;
3203         my $data = $resp->{'return'} || [];
3204         my $totalrdbytes = 0;
3205         my $totalwrbytes = 0;
3206
3207         for my $blockstat (@$data) {
3208             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
3209             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
3210
3211             $blockstat->{device} =~ s/drive-//;
3212             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
3213         }
3214         $res->{$vmid}->{diskread} = $totalrdbytes;
3215         $res->{$vmid}->{diskwrite} = $totalwrbytes;
3216     };
3217
3218     my $statuscb = sub {
3219         my ($vmid, $resp) = @_;
3220
3221         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
3222         # this fails if ballon driver is not loaded, so this must be
3223         # the last commnand (following command are aborted if this fails).
3224         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
3225
3226         my $status = 'unknown';
3227         if (!defined($status = $resp->{'return'}->{status})) {
3228             warn "unable to get VM status\n";
3229             return;
3230         }
3231
3232         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
3233     };
3234
3235     foreach my $vmid (keys %$list) {
3236         next if $opt_vmid && ($vmid ne $opt_vmid);
3237         next if !$res->{$vmid}->{pid}; # not running
3238         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
3239     }
3240
3241     $qmpclient->queue_execute(undef, 2);
3242
3243     foreach my $vmid (keys %$list) {
3244         next if $opt_vmid && ($vmid ne $opt_vmid);
3245         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
3246     }
3247
3248     return $res;
3249 }
3250
3251 sub foreach_drive {
3252     my ($conf, $func, @param) = @_;
3253
3254     foreach my $ds (valid_drive_names()) {
3255         next if !defined($conf->{$ds});
3256
3257         my $drive = parse_drive($ds, $conf->{$ds});
3258         next if !$drive;
3259
3260         &$func($ds, $drive, @param);
3261     }
3262 }
3263
3264 sub foreach_volid {
3265     my ($conf, $func, @param) = @_;
3266
3267     my $volhash = {};
3268
3269     my $test_volid = sub {
3270         my ($volid, $is_cdrom, $replicate, $shared, $snapname) = @_;
3271
3272         return if !$volid;
3273
3274         $volhash->{$volid}->{cdrom} //= 1;
3275         $volhash->{$volid}->{cdrom} = 0 if !$is_cdrom;
3276
3277         $volhash->{$volid}->{replicate} //= 0;
3278         $volhash->{$volid}->{replicate} = 1 if $replicate;
3279
3280         $volhash->{$volid}->{shared} //= 0;
3281         $volhash->{$volid}->{shared} = 1 if $shared;
3282
3283         $volhash->{$volid}->{referenced_in_config} //= 0;
3284         $volhash->{$volid}->{referenced_in_config} = 1 if !defined($snapname);
3285
3286         $volhash->{$volid}->{referenced_in_snapshot}->{$snapname} = 1
3287             if defined($snapname);
3288     };
3289
3290     foreach_drive($conf, sub {
3291         my ($ds, $drive) = @_;
3292         $test_volid->($drive->{file}, drive_is_cdrom($drive), $drive->{replicate} // 1, $drive->{shared}, undef);
3293     });
3294
3295     foreach my $snapname (keys %{$conf->{snapshots}}) {
3296         my $snap = $conf->{snapshots}->{$snapname};
3297         $test_volid->($snap->{vmstate}, 0, 1, $snapname);
3298         foreach_drive($snap, sub {
3299             my ($ds, $drive) = @_;
3300             $test_volid->($drive->{file}, drive_is_cdrom($drive), $drive->{replicate} // 1, $drive->{shared}, $snapname);
3301         });
3302     }
3303
3304     foreach my $volid (keys %$volhash) {
3305         &$func($volid, $volhash->{$volid}, @param);
3306     }
3307 }
3308
3309 sub conf_has_serial {
3310     my ($conf) = @_;
3311
3312     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3313         if ($conf->{"serial$i"}) {
3314             return 1;
3315         }
3316     }
3317
3318     return 0;
3319 }
3320
3321 sub vga_conf_has_spice {
3322     my ($vga) = @_;
3323
3324     my $vgaconf = parse_vga($vga);
3325     my $vgatype = $vgaconf->{type};
3326     return 0 if !$vgatype || $vgatype !~ m/^qxl([234])?$/;
3327
3328     return $1 || 1;
3329 }
3330
3331 my $host_arch; # FIXME: fix PVE::Tools::get_host_arch
3332 sub get_host_arch() {
3333     $host_arch = (POSIX::uname())[4] if !$host_arch;
3334     return $host_arch;
3335 }
3336
3337 sub is_native($) {
3338     my ($arch) = @_;
3339     return get_host_arch() eq $arch;
3340 }
3341
3342 my $default_machines = {
3343     x86_64 => 'pc',
3344     aarch64 => 'virt',
3345 };
3346
3347 sub get_basic_machine_info {
3348     my ($conf, $forcemachine) = @_;
3349
3350     my $arch = $conf->{arch} // get_host_arch();
3351     my $machine = $forcemachine || $conf->{machine} || $default_machines->{$arch};
3352     return ($arch, $machine);
3353 }
3354
3355 sub get_ovmf_files($) {
3356     my ($arch) = @_;
3357
3358     my $ovmf = $OVMF->{$arch}
3359         or die "no OVMF images known for architecture '$arch'\n";
3360
3361     return @$ovmf;
3362 }
3363
3364 my $Arch2Qemu = {
3365     aarch64 => '/usr/bin/qemu-system-aarch64',
3366     x86_64 => '/usr/bin/qemu-system-x86_64',
3367 };
3368 sub get_command_for_arch($) {
3369     my ($arch) = @_;
3370     return '/usr/bin/kvm' if is_native($arch);
3371
3372     my $cmd = $Arch2Qemu->{$arch}
3373         or die "don't know how to emulate architecture '$arch'\n";
3374     return $cmd;
3375 }
3376
3377 sub get_cpu_options {
3378     my ($conf, $arch, $kvm, $machine_type, $kvm_off, $kvmver, $winversion, $gpu_passthrough) = @_;
3379
3380     my $cpuFlags = [];
3381     my $ostype = $conf->{ostype};
3382
3383     my $cpu = $kvm ? "kvm64" : "qemu64";
3384     if ($arch eq 'aarch64') {
3385         $cpu = 'cortex-a57';
3386     }
3387     my $hv_vendor_id;
3388     if (my $cputype = $conf->{cpu}) {
3389         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
3390             or die "Cannot parse cpu description: $cputype\n";
3391         $cpu = $cpuconf->{cputype};
3392         $kvm_off = 1 if $cpuconf->{hidden};
3393         $hv_vendor_id = $cpuconf->{'hv-vendor-id'};
3394
3395         if (defined(my $flags = $cpuconf->{flags})) {
3396             push @$cpuFlags, split(";", $flags);
3397         }
3398     }
3399
3400     push @$cpuFlags , '+lahf_lm' if $cpu eq 'kvm64' && $arch eq 'x86_64';
3401
3402     push @$cpuFlags , '-x2apic'
3403         if $conf->{ostype} && $conf->{ostype} eq 'solaris';
3404
3405     push @$cpuFlags, '+sep' if $cpu eq 'kvm64' || $cpu eq 'kvm32';
3406
3407     push @$cpuFlags, '-rdtscp' if $cpu =~ m/^Opteron/;
3408
3409     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3) && $arch eq 'x86_64') {
3410
3411         push @$cpuFlags , '+kvm_pv_unhalt' if $kvm;
3412         push @$cpuFlags , '+kvm_pv_eoi' if $kvm;
3413     }
3414
3415     add_hyperv_enlightenments($cpuFlags, $winversion, $machine_type, $kvmver, $conf->{bios}, $gpu_passthrough, $hv_vendor_id) if $kvm;
3416
3417     push @$cpuFlags, 'enforce' if $cpu ne 'host' && $kvm && $arch eq 'x86_64';
3418
3419     push @$cpuFlags, 'kvm=off' if $kvm_off;
3420
3421     if (my $cpu_vendor = $cpu_vendor_list->{$cpu}) {
3422         push @$cpuFlags, "vendor=${cpu_vendor}"
3423             if $cpu_vendor ne 'default';
3424     } elsif ($arch ne 'aarch64') {
3425         die "internal error"; # should not happen
3426     }
3427
3428     $cpu .= "," . join(',', @$cpuFlags) if scalar(@$cpuFlags);
3429
3430     return ('-cpu', $cpu);
3431 }
3432
3433 sub config_to_command {
3434     my ($storecfg, $vmid, $conf, $defaults, $forcemachine) = @_;
3435
3436     my $cmd = [];
3437     my $globalFlags = [];
3438     my $machineFlags = [];
3439     my $rtcFlags = [];
3440     my $devices = [];
3441     my $pciaddr = '';
3442     my $bridges = {};
3443     my $kvmver = kvm_user_version();
3444     my $vernum = 0; # unknown
3445     my $ostype = $conf->{ostype};
3446     my $winversion = windows_version($ostype);
3447     my $kvm = $conf->{kvm};
3448
3449     my ($arch, $machine_type) = get_basic_machine_info($conf, $forcemachine);
3450     $kvm //= 1 if is_native($arch);
3451
3452     if ($kvm) {
3453         die "KVM virtualisation configured, but not available. Either disable in VM configuration or enable in BIOS.\n"
3454             if !defined kvm_version();
3455     }
3456
3457     if ($kvmver =~ m/^(\d+)\.(\d+)$/) {
3458         $vernum = $1*1000000+$2*1000;
3459     } elsif ($kvmver =~ m/^(\d+)\.(\d+)\.(\d+)$/) {
3460         $vernum = $1*1000000+$2*1000+$3;
3461     }
3462
3463     die "detected old qemu-kvm binary ($kvmver)\n" if $vernum < 15000;
3464
3465     my $have_ovz = -f '/proc/vz/vestat';
3466
3467     my $q35 = machine_type_is_q35($conf);
3468     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
3469     my $use_old_bios_files = undef;
3470     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3471
3472     my $cpuunits = defined($conf->{cpuunits}) ?
3473             $conf->{cpuunits} : $defaults->{cpuunits};
3474
3475     push @$cmd, get_command_for_arch($arch);
3476
3477     push @$cmd, '-id', $vmid;
3478
3479     my $vmname = $conf->{name} || "vm$vmid";
3480
3481     push @$cmd, '-name', $vmname;
3482
3483     my $use_virtio = 0;
3484
3485     my $qmpsocket = qmp_socket($vmid);
3486     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server,nowait";
3487     push @$cmd, '-mon', "chardev=qmp,mode=control";
3488
3489     if (qemu_machine_feature_enabled($machine_type, $kvmver, 2, 12)) {
3490         push @$cmd, '-chardev', "socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5";
3491         push @$cmd, '-mon', "chardev=qmp-event,mode=control";
3492     }
3493
3494     push @$cmd, '-pidfile' , pidfile_name($vmid);
3495
3496     push @$cmd, '-daemonize';
3497
3498     if ($conf->{smbios1}) {
3499         push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
3500     }
3501
3502     if ($conf->{vmgenid}) {
3503         push @$devices, '-device', 'vmgenid,guid='.$conf->{vmgenid};
3504     }
3505
3506     my ($ovmf_code, $ovmf_vars) = get_ovmf_files($arch);
3507     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3508         die "uefi base image not found\n" if ! -f $ovmf_code;
3509
3510         my $path;
3511         my $format;
3512         if (my $efidisk = $conf->{efidisk0}) {
3513             my $d = PVE::JSONSchema::parse_property_string($efidisk_fmt, $efidisk);
3514             my ($storeid, $volname) = PVE::Storage::parse_volume_id($d->{file}, 1);
3515             $format = $d->{format};
3516             if ($storeid) {
3517                 $path = PVE::Storage::path($storecfg, $d->{file});
3518                 if (!defined($format)) {
3519                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
3520                     $format = qemu_img_format($scfg, $volname);
3521                 }
3522             } else {
3523                 $path = $d->{file};
3524                 die "efidisk format must be specified\n"
3525                     if !defined($format);
3526             }
3527         } else {
3528             warn "no efidisk configured! Using temporary efivars disk.\n";
3529             $path = "/tmp/$vmid-ovmf.fd";
3530             PVE::Tools::file_copy($ovmf_vars, $path, -s $ovmf_vars);
3531             $format = 'raw';
3532         }
3533
3534         push @$cmd, '-drive', "if=pflash,unit=0,format=raw,readonly,file=$ovmf_code";
3535         push @$cmd, '-drive', "if=pflash,unit=1,format=$format,id=drive-efidisk0,file=$path";
3536     }
3537
3538
3539     # add usb controllers
3540     my @usbcontrollers = PVE::QemuServer::USB::get_usb_controllers($conf, $bridges, $arch, $machine_type, $usbdesc->{format}, $MAX_USB_DEVICES);
3541     push @$devices, @usbcontrollers if @usbcontrollers;
3542     my $vga = parse_vga($conf->{vga});
3543
3544     my $qxlnum = vga_conf_has_spice($conf->{vga});
3545     $vga->{type} = 'qxl' if $qxlnum;
3546
3547     if (!$vga->{type}) {
3548         if ($arch eq 'aarch64') {
3549             $vga->{type} = 'virtio';
3550         } elsif (qemu_machine_feature_enabled($machine_type, $kvmver, 2, 9)) {
3551             $vga->{type} = (!$winversion || $winversion >= 6) ? 'std' : 'cirrus';
3552         } else {
3553             $vga->{type} = ($winversion >= 6) ? 'std' : 'cirrus';
3554         }
3555     }
3556
3557     # enable absolute mouse coordinates (needed by vnc)
3558     my $tablet;
3559     if (defined($conf->{tablet})) {
3560         $tablet = $conf->{tablet};
3561     } else {
3562         $tablet = $defaults->{tablet};
3563         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
3564         $tablet = 0 if $vga->{type} =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
3565     }
3566
3567     if ($tablet) {
3568         push @$devices, '-device', print_tabletdevice_full($conf, $arch) if $tablet;
3569         my $kbd = print_keyboarddevice_full($conf, $arch);
3570         push @$devices, '-device', $kbd if defined($kbd);
3571     }
3572
3573     my $kvm_off = 0;
3574     my $gpu_passthrough;
3575
3576     # host pci devices
3577     for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
3578         my $d = parse_hostpci($conf->{"hostpci$i"});
3579         next if !$d;
3580
3581         my $pcie = $d->{pcie};
3582         if($pcie){
3583             die "q35 machine model is not enabled" if !$q35;
3584             # win7 wants to have the pcie devices directly on the pcie bus
3585             # instead of in the root port
3586             if ($winversion == 7) {
3587                 $pciaddr = print_pcie_addr("hostpci${i}bus0");
3588             } else {
3589                 $pciaddr = print_pcie_addr("hostpci$i");
3590             }
3591         }else{
3592             $pciaddr = print_pci_addr("hostpci$i", $bridges, $arch, $machine_type);
3593         }
3594
3595         my $rombar = defined($d->{rombar}) && !$d->{rombar} ? ',rombar=0' : '';
3596         my $romfile = $d->{romfile};
3597
3598         my $xvga = '';
3599         if ($d->{'x-vga'}) {
3600             $xvga = ',x-vga=on';
3601             $kvm_off = 1;
3602             $vga->{type} = 'none' if !defined($conf->{vga});
3603             $gpu_passthrough = 1;
3604
3605             if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3606                 $xvga = "";
3607             }
3608         }
3609         my $pcidevices = $d->{pciid};
3610         my $multifunction = 1 if @$pcidevices > 1;
3611         my $sysfspath;
3612         if ($d->{mdev} && scalar(@$pcidevices) == 1) {
3613             my $id = $pcidevices->[0]->{id};
3614             my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $i);
3615             $sysfspath = "/sys/bus/pci/devices/0000:$id/$uuid";
3616         } elsif ($d->{mdev}) {
3617             warn "ignoring mediated device with multifunction device\n";
3618         }
3619
3620         my $j=0;
3621         foreach my $pcidevice (@$pcidevices) {
3622
3623             my $id = "hostpci$i";
3624             $id .= ".$j" if $multifunction;
3625             my $addr = $pciaddr;
3626             $addr .= ".$j" if $multifunction;
3627             my $devicestr = "vfio-pci";
3628             if ($sysfspath) {
3629                 $devicestr .= ",sysfsdev=$sysfspath";
3630             } else {
3631                 $devicestr .= ",host=$pcidevice->{id}";
3632             }
3633             $devicestr .= ",id=$id$addr";
3634
3635             if($j == 0){
3636                 $devicestr .= "$rombar$xvga";
3637                 $devicestr .= ",multifunction=on" if $multifunction;
3638                 $devicestr .= ",romfile=/usr/share/kvm/$romfile" if $romfile;
3639             }
3640
3641             push @$devices, '-device', $devicestr;
3642             $j++;
3643         }
3644     }
3645
3646     # usb devices
3647     my @usbdevices = PVE::QemuServer::USB::get_usb_devices($conf, $usbdesc->{format}, $MAX_USB_DEVICES);
3648     push @$devices, @usbdevices if @usbdevices;
3649     # serial devices
3650     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3651         if (my $path = $conf->{"serial$i"}) {
3652             if ($path eq 'socket') {
3653                 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
3654                 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server,nowait";
3655                 # On aarch64, serial0 is the UART device. Qemu only allows
3656                 # connecting UART devices via the '-serial' command line, as
3657                 # the device has a fixed slot on the hardware...
3658                 if ($arch eq 'aarch64' && $i == 0) {
3659                     push @$devices, '-serial', "chardev:serial$i";
3660                 } else {
3661                     push @$devices, '-device', "isa-serial,chardev=serial$i";
3662                 }
3663             } else {
3664                 die "no such serial device\n" if ! -c $path;
3665                 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
3666                 push @$devices, '-device', "isa-serial,chardev=serial$i";
3667             }
3668         }
3669     }
3670
3671     # parallel devices
3672     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
3673         if (my $path = $conf->{"parallel$i"}) {
3674             die "no such parallel device\n" if ! -c $path;
3675             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
3676             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
3677             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
3678         }
3679     }
3680
3681
3682     my $sockets = 1;
3683     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3684     $sockets = $conf->{sockets} if  $conf->{sockets};
3685
3686     my $cores = $conf->{cores} || 1;
3687
3688     my $maxcpus = $sockets * $cores;
3689
3690     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
3691
3692     my $allowed_vcpus = $cpuinfo->{cpus};
3693
3694     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n"
3695         if ($allowed_vcpus < $maxcpus);
3696
3697     if($hotplug_features->{cpu} && qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 7)) {
3698
3699         push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3700         for (my $i = 2; $i <= $vcpus; $i++)  {
3701             my $cpustr = print_cpu_device($conf,$i);
3702             push @$cmd, '-device', $cpustr;
3703         }
3704
3705     } else {
3706
3707         push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3708     }
3709     push @$cmd, '-nodefaults';
3710
3711     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
3712
3713     my $bootindex_hash = {};
3714     my $i = 1;
3715     foreach my $o (split(//, $bootorder)) {
3716         $bootindex_hash->{$o} = $i*100;
3717         $i++;
3718     }
3719
3720     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
3721
3722     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
3723
3724     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
3725
3726     if ($vga->{type} && $vga->{type} !~ m/^serial\d+$/ && $vga->{type} ne 'none'){
3727         push @$devices, '-device', print_vga_device($conf, $vga, $arch, $machine_type, undef, $qxlnum, $bridges);
3728         my $socket = vnc_socket($vmid);
3729         push @$cmd,  '-vnc', "unix:$socket,x509,password";
3730     } else {
3731         push @$cmd, '-vga', 'none' if $vga->{type} eq 'none';
3732         push @$cmd, '-nographic';
3733     }
3734
3735     # time drift fix
3736     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
3737
3738     my $useLocaltime = $conf->{localtime};
3739
3740     if ($winversion >= 5) { # windows
3741         $useLocaltime = 1 if !defined($conf->{localtime});
3742
3743         # use time drift fix when acpi is enabled
3744         if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
3745             $tdf = 1 if !defined($conf->{tdf});
3746         }
3747     }
3748
3749     if ($winversion >= 6) {
3750         push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3751         push @$cmd, '-no-hpet';
3752     }
3753
3754     push @$rtcFlags, 'driftfix=slew' if $tdf;
3755
3756     if (!$kvm) {
3757         push @$machineFlags, 'accel=tcg';
3758     }
3759
3760     if ($machine_type) {
3761         push @$machineFlags, "type=${machine_type}";
3762     }
3763
3764     if ($conf->{startdate}) {
3765         push @$rtcFlags, "base=$conf->{startdate}";
3766     } elsif ($useLocaltime) {
3767         push @$rtcFlags, 'base=localtime';
3768     }
3769
3770     push @$cmd, get_cpu_options($conf, $arch, $kvm, $machine_type, $kvm_off, $kvmver, $winversion, $gpu_passthrough);
3771
3772     PVE::QemuServer::Memory::config($conf, $vmid, $sockets, $cores, $defaults, $hotplug_features, $cmd);
3773
3774     push @$cmd, '-S' if $conf->{freeze};
3775
3776     push @$cmd, '-k', $conf->{keyboard} if defined($conf->{keyboard});
3777
3778     # enable sound
3779     #my $soundhw = $conf->{soundhw} || $defaults->{soundhw};
3780     #push @$cmd, '-soundhw', 'es1370';
3781     #push @$cmd, '-soundhw', $soundhw if $soundhw;
3782
3783     if (parse_guest_agent($conf)->{enabled}) {
3784         my $qgasocket = qmp_socket($vmid, 1);
3785         my $pciaddr = print_pci_addr("qga0", $bridges, $arch, $machine_type);
3786         push @$devices, '-chardev', "socket,path=$qgasocket,server,nowait,id=qga0";
3787         push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3788         push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3789     }
3790
3791     my $spice_port;
3792
3793     if ($qxlnum) {
3794         if ($qxlnum > 1) {
3795             if ($winversion){
3796                 for(my $i = 1; $i < $qxlnum; $i++){
3797                     push @$devices, '-device', print_vga_device($conf, $vga, $arch, $machine_type, $i, $qxlnum, $bridges);
3798                 }
3799             } else {
3800                 # assume other OS works like Linux
3801                 my ($ram, $vram) = ("134217728", "67108864");
3802                 if ($vga->{memory}) {
3803                     $ram = PVE::Tools::convert_size($qxlnum*4*$vga->{memory}, 'mb' => 'b');
3804                     $vram = PVE::Tools::convert_size($qxlnum*2*$vga->{memory}, 'mb' => 'b');
3805                 }
3806                 push @$cmd, '-global', "qxl-vga.ram_size=$ram";
3807                 push @$cmd, '-global', "qxl-vga.vram_size=$vram";
3808             }
3809         }
3810
3811         my $pciaddr = print_pci_addr("spice", $bridges, $arch, $machine_type);
3812
3813         my $nodename = PVE::INotify::nodename();
3814         my $pfamily = PVE::Tools::get_host_address_family($nodename);
3815         my @nodeaddrs = PVE::Tools::getaddrinfo_all('localhost', family => $pfamily);
3816         die "failed to get an ip address of type $pfamily for 'localhost'\n" if !@nodeaddrs;
3817         my $localhost = PVE::Network::addr_to_ip($nodeaddrs[0]->{addr});
3818         $spice_port = PVE::Tools::next_spice_port($pfamily, $localhost);
3819
3820         push @$devices, '-spice', "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";
3821
3822         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3823         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3824         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3825     }
3826
3827     # enable balloon by default, unless explicitly disabled
3828     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3829         $pciaddr = print_pci_addr("balloon0", $bridges, $arch, $machine_type);
3830         push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
3831     }
3832
3833     if ($conf->{watchdog}) {
3834         my $wdopts = parse_watchdog($conf->{watchdog});
3835         $pciaddr = print_pci_addr("watchdog", $bridges, $arch, $machine_type);
3836         my $watchdog = $wdopts->{model} || 'i6300esb';
3837         push @$devices, '-device', "$watchdog$pciaddr";
3838         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3839     }
3840
3841     my $vollist = [];
3842     my $scsicontroller = {};
3843     my $ahcicontroller = {};
3844     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3845
3846     # Add iscsi initiator name if available
3847     if (my $initiator = get_initiator_name()) {
3848         push @$devices, '-iscsi', "initiator-name=$initiator";
3849     }
3850
3851     foreach_drive($conf, sub {
3852         my ($ds, $drive) = @_;
3853
3854         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3855             push @$vollist, $drive->{file};
3856         }
3857
3858         # ignore efidisk here, already added in bios/fw handling code above
3859         return if $drive->{interface} eq 'efidisk';
3860
3861         $use_virtio = 1 if $ds =~ m/^virtio/;
3862
3863         if (drive_is_cdrom ($drive)) {
3864             if ($bootindex_hash->{d}) {
3865                 $drive->{bootindex} = $bootindex_hash->{d};
3866                 $bootindex_hash->{d} += 1;
3867             }
3868         } else {
3869             if ($bootindex_hash->{c}) {
3870                 $drive->{bootindex} = $bootindex_hash->{c} if $conf->{bootdisk} && ($conf->{bootdisk} eq $ds);
3871                 $bootindex_hash->{c} += 1;
3872             }
3873         }
3874
3875         if($drive->{interface} eq 'virtio'){
3876            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3877         }
3878
3879         if ($drive->{interface} eq 'scsi') {
3880
3881             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
3882
3883             $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges, $arch, $machine_type);
3884             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
3885
3886             my $iothread = '';
3887             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
3888                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3889                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3890             } elsif ($drive->{iothread}) {
3891                 warn "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n";
3892             }
3893
3894             my $queues = '';
3895             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
3896                 $queues = ",num_queues=$drive->{queues}";
3897             }
3898
3899             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues" if !$scsicontroller->{$controller};
3900             $scsicontroller->{$controller}=1;
3901         }
3902
3903         if ($drive->{interface} eq 'sata') {
3904            my $controller = int($drive->{index} / $MAX_SATA_DISKS);
3905            $pciaddr = print_pci_addr("ahci$controller", $bridges, $arch, $machine_type);
3906            push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr" if !$ahcicontroller->{$controller};
3907            $ahcicontroller->{$controller}=1;
3908         }
3909
3910         my $drive_cmd = print_drive_full($storecfg, $vmid, $drive);
3911         push @$devices, '-drive',$drive_cmd;
3912         push @$devices, '-device', print_drivedevice_full($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type);
3913     });
3914
3915     for (my $i = 0; $i < $MAX_NETS; $i++) {
3916          next if !$conf->{"net$i"};
3917          my $d = parse_net($conf->{"net$i"});
3918          next if !$d;
3919
3920          $use_virtio = 1 if $d->{model} eq 'virtio';
3921
3922          if ($bootindex_hash->{n}) {
3923             $d->{bootindex} = $bootindex_hash->{n};
3924             $bootindex_hash->{n} += 1;
3925          }
3926
3927          my $netdevfull = print_netdev_full($vmid, $conf, $arch, $d, "net$i");
3928          push @$devices, '-netdev', $netdevfull;
3929
3930          my $netdevicefull = print_netdevice_full($vmid, $conf, $d, "net$i", $bridges, $use_old_bios_files, $arch, $machine_type);
3931          push @$devices, '-device', $netdevicefull;
3932     }
3933
3934     if ($conf->{ivshmem}) {
3935         my $ivshmem = PVE::JSONSchema::parse_property_string($ivshmem_fmt, $conf->{ivshmem});
3936
3937         my $bus;
3938         if ($q35) {
3939             $bus = print_pcie_addr("ivshmem");
3940         } else {
3941             $bus = print_pci_addr("ivshmem", $bridges, $arch, $machine_type);
3942         }
3943
3944         my $ivshmem_name = $ivshmem->{name} // $vmid;
3945         my $path = '/dev/shm/pve-shm-' . $ivshmem_name;
3946
3947         push @$devices, '-device', "ivshmem-plain,memdev=ivshmem$bus,";
3948         push @$devices, '-object', "memory-backend-file,id=ivshmem,share=on,mem-path=$path,size=$ivshmem->{size}M";
3949     }
3950
3951     if (!$q35) {
3952         # add pci bridges
3953         if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3954            $bridges->{1} = 1;
3955            $bridges->{2} = 1;
3956         }
3957
3958         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
3959
3960         while (my ($k, $v) = each %$bridges) {
3961             $pciaddr = print_pci_addr("pci.$k", undef, $arch, $machine_type);
3962             unshift @$devices, '-device', "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr" if $k > 0;
3963         }
3964     }
3965
3966     push @$cmd, @$devices;
3967     push @$cmd, '-rtc', join(',', @$rtcFlags)
3968         if scalar(@$rtcFlags);
3969     push @$cmd, '-machine', join(',', @$machineFlags)
3970         if scalar(@$machineFlags);
3971     push @$cmd, '-global', join(',', @$globalFlags)
3972         if scalar(@$globalFlags);
3973
3974     # add custom args
3975     if ($conf->{args}) {
3976         my $aa = PVE::Tools::split_args($conf->{args});
3977         push @$cmd, @$aa;
3978     }
3979
3980     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
3981 }
3982
3983 sub vnc_socket {
3984     my ($vmid) = @_;
3985     return "${var_run_tmpdir}/$vmid.vnc";
3986 }
3987
3988 sub spice_port {
3989     my ($vmid) = @_;
3990
3991     my $res = vm_mon_cmd($vmid, 'query-spice');
3992
3993     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
3994 }
3995
3996 sub qmp_socket {
3997     my ($vmid, $qga, $name) = @_;
3998     my $sockettype = $qga ? 'qga' : 'qmp';
3999     my $ext = $name ? '-'.$name : '';
4000     return "${var_run_tmpdir}/$vmid$ext.$sockettype";
4001 }
4002
4003 sub pidfile_name {
4004     my ($vmid) = @_;
4005     return "${var_run_tmpdir}/$vmid.pid";
4006 }
4007
4008 sub vm_devices_list {
4009     my ($vmid) = @_;
4010
4011     my $res = vm_mon_cmd($vmid, 'query-pci');
4012     my $devices_to_check = [];
4013     my $devices = {};
4014     foreach my $pcibus (@$res) {
4015         push @$devices_to_check, @{$pcibus->{devices}},
4016     }
4017
4018     while (@$devices_to_check) {
4019         my $to_check = [];
4020         for my $d (@$devices_to_check) {
4021             $devices->{$d->{'qdev_id'}} = 1 if $d->{'qdev_id'};
4022             next if !$d->{'pci_bridge'};
4023
4024             $devices->{$d->{'qdev_id'}} += scalar(@{$d->{'pci_bridge'}->{devices}});
4025             push @$to_check, @{$d->{'pci_bridge'}->{devices}};
4026         }
4027         $devices_to_check = $to_check;
4028     }
4029
4030     my $resblock = vm_mon_cmd($vmid, 'query-block');
4031     foreach my $block (@$resblock) {
4032         if($block->{device} =~ m/^drive-(\S+)/){
4033                 $devices->{$1} = 1;
4034         }
4035     }
4036
4037     my $resmice = vm_mon_cmd($vmid, 'query-mice');
4038     foreach my $mice (@$resmice) {
4039         if ($mice->{name} eq 'QEMU HID Tablet') {
4040             $devices->{tablet} = 1;
4041             last;
4042         }
4043     }
4044
4045     # for usb devices there is no query-usb
4046     # but we can iterate over the entries in
4047     # qom-list path=/machine/peripheral
4048     my $resperipheral = vm_mon_cmd($vmid, 'qom-list', path => '/machine/peripheral');
4049     foreach my $per (@$resperipheral) {
4050         if ($per->{name} =~ m/^usb\d+$/) {
4051             $devices->{$per->{name}} = 1;
4052         }
4053     }
4054
4055     return $devices;
4056 }
4057
4058 sub vm_deviceplug {
4059     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4060
4061     my $q35 = machine_type_is_q35($conf);
4062
4063     my $devices_list = vm_devices_list($vmid);
4064     return 1 if defined($devices_list->{$deviceid});
4065
4066     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid, $arch, $machine_type); # add PCI bridge if we need it for the device
4067
4068     if ($deviceid eq 'tablet') {
4069
4070         qemu_deviceadd($vmid, print_tabletdevice_full($conf, $arch));
4071
4072     } elsif ($deviceid eq 'keyboard') {
4073
4074         qemu_deviceadd($vmid, print_keyboarddevice_full($conf, $arch));
4075
4076     } elsif ($deviceid =~ m/^usb(\d+)$/) {
4077
4078         die "usb hotplug currently not reliable\n";
4079         # since we can't reliably hot unplug all added usb devices
4080         # and usb passthrough disables live migration
4081         # we disable usb hotplugging for now
4082         qemu_deviceadd($vmid, PVE::QemuServer::USB::print_usbdevice_full($conf, $deviceid, $device));
4083
4084     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4085
4086         qemu_iothread_add($vmid, $deviceid, $device);
4087
4088         qemu_driveadd($storecfg, $vmid, $device);
4089         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, $arch, $machine_type);
4090
4091         qemu_deviceadd($vmid, $devicefull);
4092         eval { qemu_deviceaddverify($vmid, $deviceid); };
4093         if (my $err = $@) {
4094             eval { qemu_drivedel($vmid, $deviceid); };
4095             warn $@ if $@;
4096             die $err;
4097         }
4098
4099     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4100
4101
4102         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
4103         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
4104         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
4105
4106         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
4107
4108         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
4109             qemu_iothread_add($vmid, $deviceid, $device);
4110             $devicefull .= ",iothread=iothread-$deviceid";
4111         }
4112
4113         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
4114             $devicefull .= ",num_queues=$device->{queues}";
4115         }
4116
4117         qemu_deviceadd($vmid, $devicefull);
4118         qemu_deviceaddverify($vmid, $deviceid);
4119
4120     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4121
4122         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device, $arch, $machine_type);
4123         qemu_driveadd($storecfg, $vmid, $device);
4124
4125         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, $arch, $machine_type);
4126         eval { qemu_deviceadd($vmid, $devicefull); };
4127         if (my $err = $@) {
4128             eval { qemu_drivedel($vmid, $deviceid); };
4129             warn $@ if $@;
4130             die $err;
4131         }
4132
4133     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4134
4135         return undef if !qemu_netdevadd($vmid, $conf, $arch, $device, $deviceid);
4136
4137         my $machine_type = PVE::QemuServer::qemu_machine_pxe($vmid, $conf);
4138         my $use_old_bios_files = undef;
4139         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
4140
4141         my $netdevicefull = print_netdevice_full($vmid, $conf, $device, $deviceid, undef, $use_old_bios_files, $arch, $machine_type);
4142         qemu_deviceadd($vmid, $netdevicefull);
4143         eval { qemu_deviceaddverify($vmid, $deviceid); };
4144         if (my $err = $@) {
4145             eval { qemu_netdevdel($vmid, $deviceid); };
4146             warn $@ if $@;
4147             die $err;
4148         }
4149
4150     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
4151
4152         my $bridgeid = $2;
4153         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
4154         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
4155
4156         qemu_deviceadd($vmid, $devicefull);
4157         qemu_deviceaddverify($vmid, $deviceid);
4158
4159     } else {
4160         die "can't hotplug device '$deviceid'\n";
4161     }
4162
4163     return 1;
4164 }
4165
4166 # fixme: this should raise exceptions on error!
4167 sub vm_deviceunplug {
4168     my ($vmid, $conf, $deviceid) = @_;
4169
4170     my $devices_list = vm_devices_list($vmid);
4171     return 1 if !defined($devices_list->{$deviceid});
4172
4173     die "can't unplug bootdisk" if $conf->{bootdisk} && $conf->{bootdisk} eq $deviceid;
4174
4175     if ($deviceid eq 'tablet' || $deviceid eq 'keyboard') {
4176
4177         qemu_devicedel($vmid, $deviceid);
4178
4179     } elsif ($deviceid =~ m/^usb\d+$/) {
4180
4181         die "usb hotplug currently not reliable\n";
4182         # when unplugging usb devices this way,
4183         # there may be remaining usb controllers/hubs
4184         # so we disable it for now
4185         qemu_devicedel($vmid, $deviceid);
4186         qemu_devicedelverify($vmid, $deviceid);
4187
4188     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4189
4190         qemu_devicedel($vmid, $deviceid);
4191         qemu_devicedelverify($vmid, $deviceid);
4192         qemu_drivedel($vmid, $deviceid);
4193         qemu_iothread_del($conf, $vmid, $deviceid);
4194
4195     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4196
4197         qemu_devicedel($vmid, $deviceid);
4198         qemu_devicedelverify($vmid, $deviceid);
4199         qemu_iothread_del($conf, $vmid, $deviceid);
4200
4201     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4202
4203         qemu_devicedel($vmid, $deviceid);
4204         qemu_drivedel($vmid, $deviceid);
4205         qemu_deletescsihw($conf, $vmid, $deviceid);
4206
4207     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4208
4209         qemu_devicedel($vmid, $deviceid);
4210         qemu_devicedelverify($vmid, $deviceid);
4211         qemu_netdevdel($vmid, $deviceid);
4212
4213     } else {
4214         die "can't unplug device '$deviceid'\n";
4215     }
4216
4217     return 1;
4218 }
4219
4220 sub qemu_deviceadd {
4221     my ($vmid, $devicefull) = @_;
4222
4223     $devicefull = "driver=".$devicefull;
4224     my %options =  split(/[=,]/, $devicefull);
4225
4226     vm_mon_cmd($vmid, "device_add" , %options);
4227 }
4228
4229 sub qemu_devicedel {
4230     my ($vmid, $deviceid) = @_;
4231
4232     my $ret = vm_mon_cmd($vmid, "device_del", id => $deviceid);
4233 }
4234
4235 sub qemu_iothread_add {
4236     my($vmid, $deviceid, $device) = @_;
4237
4238     if ($device->{iothread}) {
4239         my $iothreads = vm_iothreads_list($vmid);
4240         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
4241     }
4242 }
4243
4244 sub qemu_iothread_del {
4245     my($conf, $vmid, $deviceid) = @_;
4246
4247     my $device = parse_drive($deviceid, $conf->{$deviceid});
4248     if ($device->{iothread}) {
4249         my $iothreads = vm_iothreads_list($vmid);
4250         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
4251     }
4252 }
4253
4254 sub qemu_objectadd {
4255     my($vmid, $objectid, $qomtype) = @_;
4256
4257     vm_mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
4258
4259     return 1;
4260 }
4261
4262 sub qemu_objectdel {
4263     my($vmid, $objectid) = @_;
4264
4265     vm_mon_cmd($vmid, "object-del", id => $objectid);
4266
4267     return 1;
4268 }
4269
4270 sub qemu_driveadd {
4271     my ($storecfg, $vmid, $device) = @_;
4272
4273     my $drive = print_drive_full($storecfg, $vmid, $device);
4274     $drive =~ s/\\/\\\\/g;
4275     my $ret = vm_human_monitor_command($vmid, "drive_add auto \"$drive\"");
4276
4277     # If the command succeeds qemu prints: "OK"
4278     return 1 if $ret =~ m/OK/s;
4279
4280     die "adding drive failed: $ret\n";
4281 }
4282
4283 sub qemu_drivedel {
4284     my($vmid, $deviceid) = @_;
4285
4286     my $ret = vm_human_monitor_command($vmid, "drive_del drive-$deviceid");
4287     $ret =~ s/^\s+//;
4288
4289     return 1 if $ret eq "";
4290
4291     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
4292     return 1 if $ret =~ m/Device \'.*?\' not found/s;
4293
4294     die "deleting drive $deviceid failed : $ret\n";
4295 }
4296
4297 sub qemu_deviceaddverify {
4298     my ($vmid, $deviceid) = @_;
4299
4300     for (my $i = 0; $i <= 5; $i++) {
4301          my $devices_list = vm_devices_list($vmid);
4302          return 1 if defined($devices_list->{$deviceid});
4303          sleep 1;
4304     }
4305
4306     die "error on hotplug device '$deviceid'\n";
4307 }
4308
4309
4310 sub qemu_devicedelverify {
4311     my ($vmid, $deviceid) = @_;
4312
4313     # need to verify that the device is correctly removed as device_del
4314     # is async and empty return is not reliable
4315
4316     for (my $i = 0; $i <= 5; $i++) {
4317          my $devices_list = vm_devices_list($vmid);
4318          return 1 if !defined($devices_list->{$deviceid});
4319          sleep 1;
4320     }
4321
4322     die "error on hot-unplugging device '$deviceid'\n";
4323 }
4324
4325 sub qemu_findorcreatescsihw {
4326     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4327
4328     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4329
4330     my $scsihwid="$controller_prefix$controller";
4331     my $devices_list = vm_devices_list($vmid);
4332
4333     if(!defined($devices_list->{$scsihwid})) {
4334         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device, $arch, $machine_type);
4335     }
4336
4337     return 1;
4338 }
4339
4340 sub qemu_deletescsihw {
4341     my ($conf, $vmid, $opt) = @_;
4342
4343     my $device = parse_drive($opt, $conf->{$opt});
4344
4345     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
4346         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
4347         return 1;
4348     }
4349
4350     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4351
4352     my $devices_list = vm_devices_list($vmid);
4353     foreach my $opt (keys %{$devices_list}) {
4354         if (PVE::QemuServer::is_valid_drivename($opt)) {
4355             my $drive = PVE::QemuServer::parse_drive($opt, $conf->{$opt});
4356             if($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
4357                 return 1;
4358             }
4359         }
4360     }
4361
4362     my $scsihwid="scsihw$controller";
4363
4364     vm_deviceunplug($vmid, $conf, $scsihwid);
4365
4366     return 1;
4367 }
4368
4369 sub qemu_add_pci_bridge {
4370     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4371
4372     my $bridges = {};
4373
4374     my $bridgeid;
4375
4376     print_pci_addr($device, $bridges, $arch, $machine_type);
4377
4378     while (my ($k, $v) = each %$bridges) {
4379         $bridgeid = $k;
4380     }
4381     return 1 if !defined($bridgeid) || $bridgeid < 1;
4382
4383     my $bridge = "pci.$bridgeid";
4384     my $devices_list = vm_devices_list($vmid);
4385
4386     if (!defined($devices_list->{$bridge})) {
4387         vm_deviceplug($storecfg, $conf, $vmid, $bridge, $arch, $machine_type);
4388     }
4389
4390     return 1;
4391 }
4392
4393 sub qemu_set_link_status {
4394     my ($vmid, $device, $up) = @_;
4395
4396     vm_mon_cmd($vmid, "set_link", name => $device,
4397                up => $up ? JSON::true : JSON::false);
4398 }
4399
4400 sub qemu_netdevadd {
4401     my ($vmid, $conf, $arch, $device, $deviceid) = @_;
4402
4403     my $netdev = print_netdev_full($vmid, $conf, $arch, $device, $deviceid, 1);
4404     my %options =  split(/[=,]/, $netdev);
4405
4406     vm_mon_cmd($vmid, "netdev_add",  %options);
4407     return 1;
4408 }
4409
4410 sub qemu_netdevdel {
4411     my ($vmid, $deviceid) = @_;
4412
4413     vm_mon_cmd($vmid, "netdev_del", id => $deviceid);
4414 }
4415
4416 sub qemu_usb_hotplug {
4417     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4418
4419     return if !$device;
4420
4421     # remove the old one first
4422     vm_deviceunplug($vmid, $conf, $deviceid);
4423
4424     # check if xhci controller is necessary and available
4425     if ($device->{usb3}) {
4426
4427         my $devicelist = vm_devices_list($vmid);
4428
4429         if (!$devicelist->{xhci}) {
4430             my $pciaddr = print_pci_addr("xhci", undef, $arch, $machine_type);
4431             qemu_deviceadd($vmid, "nec-usb-xhci,id=xhci$pciaddr");
4432         }
4433     }
4434     my $d = parse_usb_device($device->{host});
4435     $d->{usb3} = $device->{usb3};
4436
4437     # add the new one
4438     vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $d, $arch, $machine_type);
4439 }
4440
4441 sub qemu_cpu_hotplug {
4442     my ($vmid, $conf, $vcpus) = @_;
4443
4444     my $machine_type = PVE::QemuServer::get_current_qemu_machine($vmid);
4445
4446     my $sockets = 1;
4447     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
4448     $sockets = $conf->{sockets} if  $conf->{sockets};
4449     my $cores = $conf->{cores} || 1;
4450     my $maxcpus = $sockets * $cores;
4451
4452     $vcpus = $maxcpus if !$vcpus;
4453
4454     die "you can't add more vcpus than maxcpus\n"
4455         if $vcpus > $maxcpus;
4456
4457     my $currentvcpus = $conf->{vcpus} || $maxcpus;
4458
4459     if ($vcpus < $currentvcpus) {
4460
4461         if (qemu_machine_feature_enabled ($machine_type, undef, 2, 7)) {
4462
4463             for (my $i = $currentvcpus; $i > $vcpus; $i--) {
4464                 qemu_devicedel($vmid, "cpu$i");
4465                 my $retry = 0;
4466                 my $currentrunningvcpus = undef;
4467                 while (1) {
4468                     $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
4469                     last if scalar(@{$currentrunningvcpus}) == $i-1;
4470                     raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
4471                     $retry++;
4472                     sleep 1;
4473                 }
4474                 #update conf after each succesfull cpu unplug
4475                 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4476                 PVE::QemuConfig->write_config($vmid, $conf);
4477             }
4478         } else {
4479             die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
4480         }
4481
4482         return;
4483     }
4484
4485     my $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
4486     die "vcpus in running vm does not match its configuration\n"
4487         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
4488
4489     if (qemu_machine_feature_enabled ($machine_type, undef, 2, 7)) {
4490
4491         for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
4492             my $cpustr = print_cpu_device($conf, $i);
4493             qemu_deviceadd($vmid, $cpustr);
4494
4495             my $retry = 0;
4496             my $currentrunningvcpus = undef;
4497             while (1) {
4498                 $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
4499                 last if scalar(@{$currentrunningvcpus}) == $i;
4500                 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
4501                 sleep 1;
4502                 $retry++;
4503             }
4504             #update conf after each succesfull cpu hotplug
4505             $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4506             PVE::QemuConfig->write_config($vmid, $conf);
4507         }
4508     } else {
4509
4510         for (my $i = $currentvcpus; $i < $vcpus; $i++) {
4511             vm_mon_cmd($vmid, "cpu-add", id => int($i));
4512         }
4513     }
4514 }
4515
4516 sub qemu_block_set_io_throttle {
4517     my ($vmid, $deviceid,
4518         $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
4519         $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max,
4520         $bps_max_length, $bps_rd_max_length, $bps_wr_max_length,
4521         $iops_max_length, $iops_rd_max_length, $iops_wr_max_length) = @_;
4522
4523     return if !check_running($vmid) ;
4524
4525     vm_mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
4526         bps => int($bps),
4527         bps_rd => int($bps_rd),
4528         bps_wr => int($bps_wr),
4529         iops => int($iops),
4530         iops_rd => int($iops_rd),
4531         iops_wr => int($iops_wr),
4532         bps_max => int($bps_max),
4533         bps_rd_max => int($bps_rd_max),
4534         bps_wr_max => int($bps_wr_max),
4535         iops_max => int($iops_max),
4536         iops_rd_max => int($iops_rd_max),
4537         iops_wr_max => int($iops_wr_max),
4538         bps_max_length => int($bps_max_length),
4539         bps_rd_max_length => int($bps_rd_max_length),
4540         bps_wr_max_length => int($bps_wr_max_length),
4541         iops_max_length => int($iops_max_length),
4542         iops_rd_max_length => int($iops_rd_max_length),
4543         iops_wr_max_length => int($iops_wr_max_length),
4544     );
4545
4546 }
4547
4548 # old code, only used to shutdown old VM after update
4549 sub __read_avail {
4550     my ($fh, $timeout) = @_;
4551
4552     my $sel = new IO::Select;
4553     $sel->add($fh);
4554
4555     my $res = '';
4556     my $buf;
4557
4558     my @ready;
4559     while (scalar (@ready = $sel->can_read($timeout))) {
4560         my $count;
4561         if ($count = $fh->sysread($buf, 8192)) {
4562             if ($buf =~ /^(.*)\(qemu\) $/s) {
4563                 $res .= $1;
4564                 last;
4565             } else {
4566                 $res .= $buf;
4567             }
4568         } else {
4569             if (!defined($count)) {
4570                 die "$!\n";
4571             }
4572             last;
4573         }
4574     }
4575
4576     die "monitor read timeout\n" if !scalar(@ready);
4577
4578     return $res;
4579 }
4580
4581 sub qemu_block_resize {
4582     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4583
4584     my $running = check_running($vmid);
4585
4586     $size = 0 if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4587
4588     return if !$running;
4589
4590     vm_mon_cmd($vmid, "block_resize", device => $deviceid, size => int($size));
4591
4592 }
4593
4594 sub qemu_volume_snapshot {
4595     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4596
4597     my $running = check_running($vmid);
4598
4599     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4600         vm_mon_cmd($vmid, 'blockdev-snapshot-internal-sync', device => $deviceid, name => $snap);
4601     } else {
4602         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4603     }
4604 }
4605
4606 sub qemu_volume_snapshot_delete {
4607     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4608
4609     my $running = check_running($vmid);
4610
4611     if($running) {
4612
4613         $running = undef;
4614         my $conf = PVE::QemuConfig->load_config($vmid);
4615         foreach_drive($conf, sub {
4616             my ($ds, $drive) = @_;
4617             $running = 1 if $drive->{file} eq $volid;
4618         });
4619     }
4620
4621     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4622         vm_mon_cmd($vmid, 'blockdev-snapshot-delete-internal-sync', device => $deviceid, name => $snap);
4623     } else {
4624         PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4625     }
4626 }
4627
4628 sub set_migration_caps {
4629     my ($vmid) = @_;
4630
4631     my $cap_ref = [];
4632
4633     my $enabled_cap = {
4634         "auto-converge" => 1,
4635         "xbzrle" => 1,
4636         "x-rdma-pin-all" => 0,
4637         "zero-blocks" => 0,
4638         "compress" => 0
4639     };
4640
4641     my $supported_capabilities = vm_mon_cmd_nocheck($vmid, "query-migrate-capabilities");
4642
4643     for my $supported_capability (@$supported_capabilities) {
4644         push @$cap_ref, {
4645             capability => $supported_capability->{capability},
4646             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4647         };
4648     }
4649
4650     vm_mon_cmd_nocheck($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4651 }
4652
4653 my $fast_plug_option = {
4654     'lock' => 1,
4655     'name' => 1,
4656     'onboot' => 1,
4657     'shares' => 1,
4658     'startup' => 1,
4659     'description' => 1,
4660     'protection' => 1,
4661     'vmstatestorage' => 1,
4662     'hookscript' => 1,
4663 };
4664
4665 # hotplug changes in [PENDING]
4666 # $selection hash can be used to only apply specified options, for
4667 # example: { cores => 1 } (only apply changed 'cores')
4668 # $errors ref is used to return error messages
4669 sub vmconfig_hotplug_pending {
4670     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4671
4672     my $defaults = load_defaults();
4673     my ($arch, $machine_type) = get_basic_machine_info($conf, undef);
4674
4675     # commit values which do not have any impact on running VM first
4676     # Note: those option cannot raise errors, we we do not care about
4677     # $selection and always apply them.
4678
4679     my $add_error = sub {
4680         my ($opt, $msg) = @_;
4681         $errors->{$opt} = "hotplug problem - $msg";
4682     };
4683
4684     my $changes = 0;
4685     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4686         if ($fast_plug_option->{$opt}) {
4687             $conf->{$opt} = $conf->{pending}->{$opt};
4688             delete $conf->{pending}->{$opt};
4689             $changes = 1;
4690         }
4691     }
4692
4693     if ($changes) {
4694         PVE::QemuConfig->write_config($vmid, $conf);
4695         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4696     }
4697
4698     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4699
4700     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4701     while (my ($opt, $force) = each %$pending_delete_hash) {
4702         next if $selection && !$selection->{$opt};
4703         eval {
4704             if ($opt eq 'hotplug') {
4705                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
4706             } elsif ($opt eq 'tablet') {
4707                 die "skip\n" if !$hotplug_features->{usb};
4708                 if ($defaults->{tablet}) {
4709                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4710                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4711                         if $arch eq 'aarch64';
4712                 } else {
4713                     vm_deviceunplug($vmid, $conf, 'tablet');
4714                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4715                 }
4716             } elsif ($opt =~ m/^usb\d+/) {
4717                 die "skip\n";
4718                 # since we cannot reliably hot unplug usb devices
4719                 # we are disabling it
4720                 die "skip\n" if !$hotplug_features->{usb} || $conf->{$opt} =~ m/spice/i;
4721                 vm_deviceunplug($vmid, $conf, $opt);
4722             } elsif ($opt eq 'vcpus') {
4723                 die "skip\n" if !$hotplug_features->{cpu};
4724                 qemu_cpu_hotplug($vmid, $conf, undef);
4725             } elsif ($opt eq 'balloon') {
4726                 # enable balloon device is not hotpluggable
4727                 die "skip\n" if defined($conf->{balloon}) && $conf->{balloon} == 0;
4728                 # here we reset the ballooning value to memory
4729                 my $balloon = $conf->{memory} || $defaults->{memory};
4730                 vm_mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4731             } elsif ($fast_plug_option->{$opt}) {
4732                 # do nothing
4733             } elsif ($opt =~ m/^net(\d+)$/) {
4734                 die "skip\n" if !$hotplug_features->{network};
4735                 vm_deviceunplug($vmid, $conf, $opt);
4736             } elsif (is_valid_drivename($opt)) {
4737                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4738                 vm_deviceunplug($vmid, $conf, $opt);
4739                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4740             } elsif ($opt =~ m/^memory$/) {
4741                 die "skip\n" if !$hotplug_features->{memory};
4742                 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
4743             } elsif ($opt eq 'cpuunits') {
4744                 cgroups_write("cpu", $vmid, "cpu.shares", $defaults->{cpuunits});
4745             } elsif ($opt eq 'cpulimit') {
4746                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", -1);
4747             } else {
4748                 die "skip\n";
4749             }
4750         };
4751         if (my $err = $@) {
4752             &$add_error($opt, $err) if $err ne "skip\n";
4753         } else {
4754             # save new config if hotplug was successful
4755             delete $conf->{$opt};
4756             vmconfig_undelete_pending_option($conf, $opt);
4757             PVE::QemuConfig->write_config($vmid, $conf);
4758             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4759         }
4760     }
4761
4762     my $apply_pending_cloudinit;
4763     $apply_pending_cloudinit = sub {
4764         my ($key, $value) = @_;
4765         $apply_pending_cloudinit = sub {}; # once is enough
4766
4767         my @cloudinit_opts = keys %$confdesc_cloudinit;
4768         foreach my $opt (keys %{$conf->{pending}}) {
4769             next if !grep { $_ eq $opt } @cloudinit_opts;
4770             $conf->{$opt} = delete $conf->{pending}->{$opt};
4771         }
4772
4773         my $new_conf = { %$conf };
4774         $new_conf->{$key} = $value;
4775         PVE::QemuServer::Cloudinit::generate_cloudinitconfig($new_conf, $vmid);
4776     };
4777
4778     foreach my $opt (keys %{$conf->{pending}}) {
4779         next if $selection && !$selection->{$opt};
4780         my $value = $conf->{pending}->{$opt};
4781         eval {
4782             if ($opt eq 'hotplug') {
4783                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
4784             } elsif ($opt eq 'tablet') {
4785                 die "skip\n" if !$hotplug_features->{usb};
4786                 if ($value == 1) {
4787                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4788                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4789                         if $arch eq 'aarch64';
4790                 } elsif ($value == 0) {
4791                     vm_deviceunplug($vmid, $conf, 'tablet');
4792                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4793                 }
4794             } elsif ($opt =~ m/^usb\d+$/) {
4795                 die "skip\n";
4796                 # since we cannot reliably hot unplug usb devices
4797                 # we are disabling it
4798                 die "skip\n" if !$hotplug_features->{usb} || $value =~ m/spice/i;
4799                 my $d = eval { PVE::JSONSchema::parse_property_string($usbdesc->{format}, $value) };
4800                 die "skip\n" if !$d;
4801                 qemu_usb_hotplug($storecfg, $conf, $vmid, $opt, $d, $arch, $machine_type);
4802             } elsif ($opt eq 'vcpus') {
4803                 die "skip\n" if !$hotplug_features->{cpu};
4804                 qemu_cpu_hotplug($vmid, $conf, $value);
4805             } elsif ($opt eq 'balloon') {
4806                 # enable/disable balloning device is not hotpluggable
4807                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
4808                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
4809                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
4810
4811                 # allow manual ballooning if shares is set to zero
4812                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
4813                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
4814                     vm_mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4815                 }
4816             } elsif ($opt =~ m/^net(\d+)$/) {
4817                 # some changes can be done without hotplug
4818                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
4819                                     $vmid, $opt, $value, $arch, $machine_type);
4820             } elsif (is_valid_drivename($opt)) {
4821                 # some changes can be done without hotplug
4822                 my $drive = parse_drive($opt, $value);
4823                 if (drive_is_cloudinit($drive)) {
4824                     &$apply_pending_cloudinit($opt, $value);
4825                 }
4826                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
4827                                      $vmid, $opt, $value, 1, $arch, $machine_type);
4828             } elsif ($opt =~ m/^memory$/) { #dimms
4829                 die "skip\n" if !$hotplug_features->{memory};
4830                 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
4831             } elsif ($opt eq 'cpuunits') {
4832                 cgroups_write("cpu", $vmid, "cpu.shares", $conf->{pending}->{$opt});
4833             } elsif ($opt eq 'cpulimit') {
4834                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
4835                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", $cpulimit);
4836             } else {
4837                 die "skip\n";  # skip non-hot-pluggable options
4838             }
4839         };
4840         if (my $err = $@) {
4841             &$add_error($opt, $err) if $err ne "skip\n";
4842         } else {
4843             # save new config if hotplug was successful
4844             $conf->{$opt} = $value;
4845             delete $conf->{pending}->{$opt};
4846             PVE::QemuConfig->write_config($vmid, $conf);
4847             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4848         }
4849     }
4850 }
4851
4852 sub try_deallocate_drive {
4853     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
4854
4855     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
4856         my $volid = $drive->{file};
4857         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
4858             my $sid = PVE::Storage::parse_volume_id($volid);
4859             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
4860
4861             # check if the disk is really unused
4862             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
4863                 if is_volume_in_use($storecfg, $conf, $key, $volid);
4864             PVE::Storage::vdisk_free($storecfg, $volid);
4865             return 1;
4866         } else {
4867             # If vm is not owner of this disk remove from config
4868             return 1;
4869         }
4870     }
4871
4872     return undef;
4873 }
4874
4875 sub vmconfig_delete_or_detach_drive {
4876     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
4877
4878     my $drive = parse_drive($opt, $conf->{$opt});
4879
4880     my $rpcenv = PVE::RPCEnvironment::get();
4881     my $authuser = $rpcenv->get_user();
4882
4883     if ($force) {
4884         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
4885         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
4886     } else {
4887         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
4888     }
4889 }
4890
4891 sub vmconfig_apply_pending {
4892     my ($vmid, $conf, $storecfg) = @_;
4893
4894     # cold plug
4895
4896     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4897     while (my ($opt, $force) = each %$pending_delete_hash) {
4898         die "internal error" if $opt =~ m/^unused/;
4899         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4900         if (!defined($conf->{$opt})) {
4901             vmconfig_undelete_pending_option($conf, $opt);
4902             PVE::QemuConfig->write_config($vmid, $conf);
4903         } elsif (is_valid_drivename($opt)) {
4904             vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4905             vmconfig_undelete_pending_option($conf, $opt);
4906             delete $conf->{$opt};
4907             PVE::QemuConfig->write_config($vmid, $conf);
4908         } else {
4909             vmconfig_undelete_pending_option($conf, $opt);
4910             delete $conf->{$opt};
4911             PVE::QemuConfig->write_config($vmid, $conf);
4912         }
4913     }
4914
4915     $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4916
4917     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4918         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4919
4920         if (defined($conf->{$opt}) && ($conf->{$opt} eq $conf->{pending}->{$opt})) {
4921             # skip if nothing changed
4922         } elsif (is_valid_drivename($opt)) {
4923             vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
4924                 if defined($conf->{$opt});
4925             $conf->{$opt} = $conf->{pending}->{$opt};
4926         } else {
4927             $conf->{$opt} = $conf->{pending}->{$opt};
4928         }
4929
4930         delete $conf->{pending}->{$opt};
4931         PVE::QemuConfig->write_config($vmid, $conf);
4932     }
4933 }
4934
4935 my $safe_num_ne = sub {
4936     my ($a, $b) = @_;
4937
4938     return 0 if !defined($a) && !defined($b);
4939     return 1 if !defined($a);
4940     return 1 if !defined($b);
4941
4942     return $a != $b;
4943 };
4944
4945 my $safe_string_ne = sub {
4946     my ($a, $b) = @_;
4947
4948     return 0 if !defined($a) && !defined($b);
4949     return 1 if !defined($a);
4950     return 1 if !defined($b);
4951
4952     return $a ne $b;
4953 };
4954
4955 sub vmconfig_update_net {
4956     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
4957
4958     my $newnet = parse_net($value);
4959
4960     if ($conf->{$opt}) {
4961         my $oldnet = parse_net($conf->{$opt});
4962
4963         if (&$safe_string_ne($oldnet->{model}, $newnet->{model}) ||
4964             &$safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
4965             &$safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
4966             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
4967
4968             # for non online change, we try to hot-unplug
4969             die "skip\n" if !$hotplug;
4970             vm_deviceunplug($vmid, $conf, $opt);
4971         } else {
4972
4973             die "internal error" if $opt !~ m/net(\d+)/;
4974             my $iface = "tap${vmid}i$1";
4975
4976             if (&$safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
4977                 &$safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
4978                 &$safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
4979                 &$safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
4980                 PVE::Network::tap_unplug($iface);
4981                 PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
4982             } elsif (&$safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
4983                 # Rate can be applied on its own but any change above needs to
4984                 # include the rate in tap_plug since OVS resets everything.
4985                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
4986             }
4987
4988             if (&$safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
4989                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
4990             }
4991
4992             return 1;
4993         }
4994     }
4995
4996     if ($hotplug) {
4997         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet, $arch, $machine_type);
4998     } else {
4999         die "skip\n";
5000     }
5001 }
5002
5003 sub vmconfig_update_disk {
5004     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $force, $arch, $machine_type) = @_;
5005
5006     # fixme: do we need force?
5007
5008     my $drive = parse_drive($opt, $value);
5009
5010     if ($conf->{$opt}) {
5011
5012         if (my $old_drive = parse_drive($opt, $conf->{$opt}))  {
5013
5014             my $media = $drive->{media} || 'disk';
5015             my $oldmedia = $old_drive->{media} || 'disk';
5016             die "unable to change media type\n" if $media ne $oldmedia;
5017
5018             if (!drive_is_cdrom($old_drive)) {
5019
5020                 if ($drive->{file} ne $old_drive->{file}) {
5021
5022                     die "skip\n" if !$hotplug;
5023
5024                     # unplug and register as unused
5025                     vm_deviceunplug($vmid, $conf, $opt);
5026                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
5027
5028                 } else {
5029                     # update existing disk
5030
5031                     # skip non hotpluggable value
5032                     if (&$safe_string_ne($drive->{discard}, $old_drive->{discard}) ||
5033                         &$safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
5034                         &$safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
5035                         &$safe_string_ne($drive->{cache}, $old_drive->{cache})) {
5036                         die "skip\n";
5037                     }
5038
5039                     # apply throttle
5040                     if (&$safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
5041                         &$safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
5042                         &$safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
5043                         &$safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
5044                         &$safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
5045                         &$safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
5046                         &$safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
5047                         &$safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
5048                         &$safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
5049                         &$safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
5050                         &$safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
5051                         &$safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max}) ||
5052                         &$safe_num_ne($drive->{bps_max_length}, $old_drive->{bps_max_length}) ||
5053                         &$safe_num_ne($drive->{bps_rd_max_length}, $old_drive->{bps_rd_max_length}) ||
5054                         &$safe_num_ne($drive->{bps_wr_max_length}, $old_drive->{bps_wr_max_length}) ||
5055                         &$safe_num_ne($drive->{iops_max_length}, $old_drive->{iops_max_length}) ||
5056                         &$safe_num_ne($drive->{iops_rd_max_length}, $old_drive->{iops_rd_max_length}) ||
5057                         &$safe_num_ne($drive->{iops_wr_max_length}, $old_drive->{iops_wr_max_length})) {
5058
5059                         qemu_block_set_io_throttle($vmid,"drive-$opt",
5060                                                    ($drive->{mbps} || 0)*1024*1024,
5061                                                    ($drive->{mbps_rd} || 0)*1024*1024,
5062                                                    ($drive->{mbps_wr} || 0)*1024*1024,
5063                                                    $drive->{iops} || 0,
5064                                                    $drive->{iops_rd} || 0,
5065                                                    $drive->{iops_wr} || 0,
5066                                                    ($drive->{mbps_max} || 0)*1024*1024,
5067                                                    ($drive->{mbps_rd_max} || 0)*1024*1024,
5068                                                    ($drive->{mbps_wr_max} || 0)*1024*1024,
5069                                                    $drive->{iops_max} || 0,
5070                                                    $drive->{iops_rd_max} || 0,
5071                                                    $drive->{iops_wr_max} || 0,
5072                                                    $drive->{bps_max_length} || 1,
5073                                                    $drive->{bps_rd_max_length} || 1,
5074                                                    $drive->{bps_wr_max_length} || 1,
5075                                                    $drive->{iops_max_length} || 1,
5076                                                    $drive->{iops_rd_max_length} || 1,
5077                                                    $drive->{iops_wr_max_length} || 1);
5078
5079                     }
5080
5081                     return 1;
5082                 }
5083
5084             } else { # cdrom
5085
5086                 if ($drive->{file} eq 'none') {
5087                     vm_mon_cmd($vmid, "eject",force => JSON::true,device => "drive-$opt");
5088                     if (drive_is_cloudinit($old_drive)) {
5089                         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive);
5090                     }
5091                 } else {
5092                     my $path = get_iso_path($storecfg, $vmid, $drive->{file});
5093                     vm_mon_cmd($vmid, "eject", force => JSON::true,device => "drive-$opt"); # force eject if locked
5094                     vm_mon_cmd($vmid, "change", device => "drive-$opt",target => "$path") if $path;
5095                 }
5096
5097                 return 1;
5098             }
5099         }
5100     }
5101
5102     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
5103     # hotplug new disks
5104     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
5105     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive, $arch, $machine_type);
5106 }
5107
5108 sub vm_start {
5109     my ($storecfg, $vmid, $statefile, $skiplock, $migratedfrom, $paused,
5110         $forcemachine, $spice_ticket, $migration_network, $migration_type, $targetstorage) = @_;
5111
5112     PVE::QemuConfig->lock_config($vmid, sub {
5113         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
5114
5115         die "you can't start a vm if it's a template\n" if PVE::QemuConfig->is_template($conf);
5116
5117         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5118
5119         die "VM $vmid already running\n" if check_running($vmid, undef, $migratedfrom);
5120
5121         if (!$statefile && scalar(keys %{$conf->{pending}})) {
5122             vmconfig_apply_pending($vmid, $conf, $storecfg);
5123             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5124         }
5125
5126         PVE::QemuServer::Cloudinit::generate_cloudinitconfig($conf, $vmid);
5127
5128         my $defaults = load_defaults();
5129
5130         # set environment variable useful inside network script
5131         $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
5132
5133         my $local_volumes = {};
5134
5135         if ($targetstorage) {
5136             foreach_drive($conf, sub {
5137                 my ($ds, $drive) = @_;
5138
5139                 return if drive_is_cdrom($drive);
5140
5141                 my $volid = $drive->{file};
5142
5143                 return if !$volid;
5144
5145                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
5146
5147                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5148                 return if $scfg->{shared};
5149                 $local_volumes->{$ds} = [$volid, $storeid, $volname];
5150             });
5151
5152             my $format = undef;
5153
5154             foreach my $opt (sort keys %$local_volumes) {
5155
5156                 my ($volid, $storeid, $volname) = @{$local_volumes->{$opt}};
5157                 my $drive = parse_drive($opt, $conf->{$opt});
5158
5159                 #if remote storage is specified, use default format
5160                 if ($targetstorage && $targetstorage ne "1") {
5161                     $storeid = $targetstorage;
5162                     my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
5163                     $format = $defFormat;
5164                 } else {
5165                     #else we use same format than original
5166                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5167                     $format = qemu_img_format($scfg, $volid);
5168                 }
5169
5170                 my $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $format, undef, ($drive->{size}/1024));
5171                 my $newdrive = $drive;
5172                 $newdrive->{format} = $format;
5173                 $newdrive->{file} = $newvolid;
5174                 my $drivestr = PVE::QemuServer::print_drive($vmid, $newdrive);
5175                 $local_volumes->{$opt} = $drivestr;
5176                 #pass drive to conf for command line
5177                 $conf->{$opt} = $drivestr;
5178             }
5179         }
5180
5181         PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-start', 1);
5182
5183         my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine);
5184
5185         my $migrate_port = 0;
5186         my $migrate_uri;
5187         if ($statefile) {
5188             if ($statefile eq 'tcp') {
5189                 my $localip = "localhost";
5190                 my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
5191                 my $nodename = PVE::INotify::nodename();
5192
5193                 if (!defined($migration_type)) {
5194                     if (defined($datacenterconf->{migration}->{type})) {
5195                         $migration_type = $datacenterconf->{migration}->{type};
5196                     } else {
5197                         $migration_type = 'secure';
5198                     }
5199                 }
5200
5201                 if ($migration_type eq 'insecure') {
5202                     my $migrate_network_addr = PVE::Cluster::get_local_migration_ip($migration_network);
5203                     if ($migrate_network_addr) {
5204                         $localip = $migrate_network_addr;
5205                     } else {
5206                         $localip = PVE::Cluster::remote_node_ip($nodename, 1);
5207                     }
5208
5209                     $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5210                 }
5211
5212                 my $pfamily = PVE::Tools::get_host_address_family($nodename);
5213                 $migrate_port = PVE::Tools::next_migrate_port($pfamily);
5214                 $migrate_uri = "tcp:${localip}:${migrate_port}";
5215                 push @$cmd, '-incoming', $migrate_uri;
5216                 push @$cmd, '-S';
5217
5218             } elsif ($statefile eq 'unix') {
5219                 # should be default for secure migrations as a ssh TCP forward
5220                 # tunnel is not deterministic reliable ready and fails regurarly
5221                 # to set up in time, so use UNIX socket forwards
5222                 my $socket_addr = "/run/qemu-server/$vmid.migrate";
5223                 unlink $socket_addr;
5224
5225                 $migrate_uri = "unix:$socket_addr";
5226
5227                 push @$cmd, '-incoming', $migrate_uri;
5228                 push @$cmd, '-S';
5229
5230             } else {
5231                 push @$cmd, '-loadstate', $statefile;
5232             }
5233         } elsif ($paused) {
5234             push @$cmd, '-S';
5235         }
5236
5237         # host pci devices
5238         for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
5239           my $d = parse_hostpci($conf->{"hostpci$i"});
5240           next if !$d;
5241           my $pcidevices = $d->{pciid};
5242           foreach my $pcidevice (@$pcidevices) {
5243                 my $pciid = $pcidevice->{id};
5244
5245                 my $info = PVE::SysFSTools::pci_device_info("0000:$pciid");
5246                 die "IOMMU not present\n" if !PVE::SysFSTools::check_iommu_support();
5247                 die "no pci device info for device '$pciid'\n" if !$info;
5248
5249                 if ($d->{mdev}) {
5250                     my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $i);
5251                     PVE::SysFSTools::pci_create_mdev_device($pciid, $uuid, $d->{mdev});
5252                 } else {
5253                     die "can't unbind/bind pci group to vfio '$pciid'\n"
5254                         if !PVE::SysFSTools::pci_dev_group_bind_to_vfio($pciid);
5255                     die "can't reset pci device '$pciid'\n"
5256                         if $info->{has_fl_reset} and !PVE::SysFSTools::pci_dev_reset($info);
5257                 }
5258           }
5259         }
5260
5261         PVE::Storage::activate_volumes($storecfg, $vollist);
5262
5263         if (-d "/sys/fs/cgroup/systemd/qemu.slice/$vmid.scope") {
5264             eval {
5265                 run_command(['/bin/systemctl', 'stop', "$vmid.scope"],
5266                     outfunc => sub {}, errfunc => sub {});
5267             };
5268         }
5269
5270         my $cpuunits = defined($conf->{cpuunits}) ? $conf->{cpuunits}
5271                                                   : $defaults->{cpuunits};
5272
5273         my $start_timeout = $conf->{hugepages} ? 300 : 30;
5274         my %run_params = (timeout => $statefile ? undef : $start_timeout, umask => 0077);
5275
5276         my %properties = (
5277             Slice => 'qemu.slice',
5278             KillMode => 'none',
5279             CPUShares => $cpuunits
5280         );
5281
5282         if (my $cpulimit = $conf->{cpulimit}) {
5283             $properties{CPUQuota} = int($cpulimit * 100);
5284         }
5285         $properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
5286
5287         my $run_qemu = sub {
5288             PVE::Tools::run_fork sub {
5289                 PVE::Systemd::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
5290                 run_command($cmd, %run_params);
5291             };
5292         };
5293
5294         if ($conf->{hugepages}) {
5295
5296             my $code = sub {
5297                 my $hugepages_topology = PVE::QemuServer::Memory::hugepages_topology($conf);
5298                 my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
5299
5300                 PVE::QemuServer::Memory::hugepages_mount();
5301                 PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
5302
5303                 eval { $run_qemu->() };
5304                 if (my $err = $@) {
5305                     PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology);
5306                     die $err;
5307                 }
5308
5309                 PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology);
5310             };
5311             eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
5312
5313         } else {
5314             eval { $run_qemu->() };
5315         }
5316
5317         if (my $err = $@) {
5318             # deactivate volumes if start fails
5319             eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
5320             die "start failed: $err";
5321         }
5322
5323         print "migration listens on $migrate_uri\n" if $migrate_uri;
5324
5325         if ($statefile && $statefile ne 'tcp' && $statefile ne 'unix')  {
5326             eval { vm_mon_cmd_nocheck($vmid, "cont"); };
5327             warn $@ if $@;
5328         }
5329
5330         #start nbd server for storage migration
5331         if ($targetstorage) {
5332             my $nodename = PVE::INotify::nodename();
5333             my $migrate_network_addr = PVE::Cluster::get_local_migration_ip($migration_network);
5334             my $localip = $migrate_network_addr ? $migrate_network_addr : PVE::Cluster::remote_node_ip($nodename, 1);
5335             my $pfamily = PVE::Tools::get_host_address_family($nodename);
5336             $migrate_port = PVE::Tools::next_migrate_port($pfamily);
5337
5338             vm_mon_cmd_nocheck($vmid, "nbd-server-start", addr => { type => 'inet', data => { host => "${localip}", port => "${migrate_port}" } } );
5339
5340             $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5341
5342             foreach my $opt (sort keys %$local_volumes) {
5343                 my $volid = $local_volumes->{$opt};
5344                 vm_mon_cmd_nocheck($vmid, "nbd-server-add", device => "drive-$opt", writable => JSON::true );
5345                 my $migrate_storage_uri = "nbd:${localip}:${migrate_port}:exportname=drive-$opt";
5346                 print "storage migration listens on $migrate_storage_uri volume:$volid\n";
5347             }
5348         }
5349
5350         if ($migratedfrom) {
5351             eval {
5352                 set_migration_caps($vmid);
5353             };
5354             warn $@ if $@;
5355
5356             if ($spice_port) {
5357                 print "spice listens on port $spice_port\n";
5358                 if ($spice_ticket) {
5359                     vm_mon_cmd_nocheck($vmid, "set_password", protocol => 'spice', password => $spice_ticket);
5360                     vm_mon_cmd_nocheck($vmid, "expire_password", protocol => 'spice', time => "+30");
5361                 }
5362             }
5363
5364         } else {
5365             vm_mon_cmd_nocheck($vmid, "balloon", value => $conf->{balloon}*1024*1024)
5366                 if !$statefile && $conf->{balloon};
5367
5368             foreach my $opt (keys %$conf) {
5369                 next if $opt !~  m/^net\d+$/;
5370                 my $nicconf = parse_net($conf->{$opt});
5371                 qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
5372             }
5373         }
5374
5375         vm_mon_cmd_nocheck($vmid, 'qom-set',
5376                     path => "machine/peripheral/balloon0",
5377                     property => "guest-stats-polling-interval",
5378                     value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
5379
5380         PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'post-start');
5381     });
5382 }
5383
5384 sub vm_mon_cmd {
5385     my ($vmid, $execute, %params) = @_;
5386
5387     my $cmd = { execute => $execute, arguments => \%params };
5388     vm_qmp_command($vmid, $cmd);
5389 }
5390
5391 sub vm_mon_cmd_nocheck {
5392     my ($vmid, $execute, %params) = @_;
5393
5394     my $cmd = { execute => $execute, arguments => \%params };
5395     vm_qmp_command($vmid, $cmd, 1);
5396 }
5397
5398 sub vm_qmp_command {
5399     my ($vmid, $cmd, $nocheck) = @_;
5400
5401     my $res;
5402
5403     my $timeout;
5404     if ($cmd->{arguments} && $cmd->{arguments}->{timeout}) {
5405         $timeout = $cmd->{arguments}->{timeout};
5406         delete $cmd->{arguments}->{timeout};
5407     }
5408
5409     eval {
5410         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
5411         my $sname = qmp_socket($vmid);
5412         if (-e $sname) { # test if VM is reasonambe new and supports qmp/qga
5413             my $qmpclient = PVE::QMPClient->new();
5414
5415             $res = $qmpclient->cmd($vmid, $cmd, $timeout);
5416         } else {
5417             die "unable to open monitor socket\n";
5418         }
5419     };
5420     if (my $err = $@) {
5421         syslog("err", "VM $vmid qmp command failed - $err");
5422         die $err;
5423     }
5424
5425     return $res;
5426 }
5427
5428 sub vm_human_monitor_command {
5429     my ($vmid, $cmdline) = @_;
5430
5431     my $res;
5432
5433     my $cmd = {
5434         execute => 'human-monitor-command',
5435         arguments => { 'command-line' => $cmdline},
5436     };
5437
5438     return vm_qmp_command($vmid, $cmd);
5439 }
5440
5441 sub vm_commandline {
5442     my ($storecfg, $vmid, $snapname) = @_;
5443
5444     my $conf = PVE::QemuConfig->load_config($vmid);
5445
5446     if ($snapname) {
5447         my $snapshot = $conf->{snapshots}->{$snapname};
5448         die "snapshot '$snapname' does not exist\n" if !defined($snapshot);
5449
5450         $snapshot->{digest} = $conf->{digest}; # keep file digest for API
5451
5452         $conf = $snapshot;
5453     }
5454
5455     my $defaults = load_defaults();
5456
5457     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults);
5458
5459     return PVE::Tools::cmd2string($cmd);
5460 }
5461
5462 sub vm_reset {
5463     my ($vmid, $skiplock) = @_;
5464
5465     PVE::QemuConfig->lock_config($vmid, sub {
5466
5467         my $conf = PVE::QemuConfig->load_config($vmid);
5468
5469         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5470
5471         vm_mon_cmd($vmid, "system_reset");
5472     });
5473 }
5474
5475 sub get_vm_volumes {
5476     my ($conf) = @_;
5477
5478     my $vollist = [];
5479     foreach_volid($conf, sub {
5480         my ($volid, $attr) = @_;
5481
5482         return if $volid =~ m|^/|;
5483
5484         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5485         return if !$sid;
5486
5487         push @$vollist, $volid;
5488     });
5489
5490     return $vollist;
5491 }
5492
5493 sub vm_stop_cleanup {
5494     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
5495
5496     eval {
5497
5498         if (!$keepActive) {
5499             my $vollist = get_vm_volumes($conf);
5500             PVE::Storage::deactivate_volumes($storecfg, $vollist);
5501         }
5502
5503         foreach my $ext (qw(mon qmp pid vnc qga)) {
5504             unlink "/var/run/qemu-server/${vmid}.$ext";
5505         }
5506
5507         if ($conf->{ivshmem}) {
5508             my $ivshmem = PVE::JSONSchema::parse_property_string($ivshmem_fmt, $conf->{ivshmem});
5509             unlink '/dev/shm/pve-shm-' . ($ivshmem->{name} // $vmid);
5510         }
5511
5512         foreach my $key (keys %$conf) {
5513             next if $key !~ m/^hostpci(\d+)$/;
5514             my $hostpciindex = $1;
5515             my $d = parse_hostpci($conf->{$key});
5516             my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $hostpciindex);
5517
5518             foreach my $pci (@{$d->{pciid}}) {
5519                 my $pciid = $pci->{id};
5520                 PVE::SysFSTools::pci_cleanup_mdev_device($pciid, $uuid);
5521             }
5522         }
5523
5524         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
5525     };
5526     warn $@ if $@; # avoid errors - just warn
5527 }
5528
5529 # Note: use $nockeck to skip tests if VM configuration file exists.
5530 # We need that when migration VMs to other nodes (files already moved)
5531 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
5532 sub vm_stop {
5533     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
5534
5535     $force = 1 if !defined($force) && !$shutdown;
5536
5537     if ($migratedfrom){
5538         my $pid = check_running($vmid, $nocheck, $migratedfrom);
5539         kill 15, $pid if $pid;
5540         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
5541         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
5542         return;
5543     }
5544
5545     PVE::QemuConfig->lock_config($vmid, sub {
5546
5547         my $pid = check_running($vmid, $nocheck);
5548         return if !$pid;
5549
5550         my $conf;
5551         if (!$nocheck) {
5552             $conf = PVE::QemuConfig->load_config($vmid);
5553             PVE::QemuConfig->check_lock($conf) if !$skiplock;
5554             if (!defined($timeout) && $shutdown && $conf->{startup}) {
5555                 my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
5556                 $timeout = $opts->{down} if $opts->{down};
5557             }
5558             PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-stop');
5559         }
5560
5561         $timeout = 60 if !defined($timeout);
5562
5563         eval {
5564             if ($shutdown) {
5565                 if (defined($conf) && parse_guest_agent($conf)->{enabled}) {
5566                     vm_qmp_command($vmid, { execute => "guest-shutdown" }, $nocheck);
5567                 } else {
5568                     vm_qmp_command($vmid, { execute => "system_powerdown" }, $nocheck);
5569                 }
5570             } else {
5571                 vm_qmp_command($vmid, { execute => "quit" }, $nocheck);
5572             }
5573         };
5574         my $err = $@;
5575
5576         if (!$err) {
5577             my $count = 0;
5578             while (($count < $timeout) && check_running($vmid, $nocheck)) {
5579                 $count++;
5580                 sleep 1;
5581             }
5582
5583             if ($count >= $timeout) {
5584                 if ($force) {
5585                     warn "VM still running - terminating now with SIGTERM\n";
5586                     kill 15, $pid;
5587                 } else {
5588                     die "VM quit/powerdown failed - got timeout\n";
5589                 }
5590             } else {
5591                 vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5592                 return;
5593             }
5594         } else {
5595             if ($force) {
5596                 warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
5597                 kill 15, $pid;
5598             } else {
5599                 die "VM quit/powerdown failed\n";
5600             }
5601         }
5602
5603         # wait again
5604         $timeout = 10;
5605
5606         my $count = 0;
5607         while (($count < $timeout) && check_running($vmid, $nocheck)) {
5608             $count++;
5609             sleep 1;
5610         }
5611
5612         if ($count >= $timeout) {
5613             warn "VM still running - terminating now with SIGKILL\n";
5614             kill 9, $pid;
5615             sleep 1;
5616         }
5617
5618         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5619    });
5620 }
5621
5622 sub vm_suspend {
5623     my ($vmid, $skiplock) = @_;
5624
5625     PVE::QemuConfig->lock_config($vmid, sub {
5626
5627         my $conf = PVE::QemuConfig->load_config($vmid);
5628
5629         PVE::QemuConfig->check_lock($conf)
5630             if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
5631
5632         vm_mon_cmd($vmid, "stop");
5633     });
5634 }
5635
5636 sub vm_resume {
5637     my ($vmid, $skiplock, $nocheck) = @_;
5638
5639     PVE::QemuConfig->lock_config($vmid, sub {
5640
5641         my $res = vm_mon_cmd($vmid, 'query-status');
5642         my $resume_cmd = 'cont';
5643
5644         if ($res->{status} && $res->{status} eq 'suspended') {
5645             $resume_cmd = 'system_wakeup';
5646         }
5647
5648         if (!$nocheck) {
5649
5650             my $conf = PVE::QemuConfig->load_config($vmid);
5651
5652             PVE::QemuConfig->check_lock($conf)
5653                 if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
5654
5655             vm_mon_cmd($vmid, $resume_cmd);
5656
5657         } else {
5658             vm_mon_cmd_nocheck($vmid, $resume_cmd);
5659         }
5660     });
5661 }
5662
5663 sub vm_sendkey {
5664     my ($vmid, $skiplock, $key) = @_;
5665
5666     PVE::QemuConfig->lock_config($vmid, sub {
5667
5668         my $conf = PVE::QemuConfig->load_config($vmid);
5669
5670         # there is no qmp command, so we use the human monitor command
5671         vm_human_monitor_command($vmid, "sendkey $key");
5672     });
5673 }
5674
5675 sub vm_destroy {
5676     my ($storecfg, $vmid, $skiplock) = @_;
5677
5678     PVE::QemuConfig->lock_config($vmid, sub {
5679
5680         my $conf = PVE::QemuConfig->load_config($vmid);
5681
5682         if (!check_running($vmid)) {
5683             destroy_vm($storecfg, $vmid, undef, $skiplock);
5684         } else {
5685             die "VM $vmid is running - destroy failed\n";
5686         }
5687     });
5688 }
5689
5690 # vzdump restore implementaion
5691
5692 sub tar_archive_read_firstfile {
5693     my $archive = shift;
5694
5695     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
5696
5697     # try to detect archive type first
5698     my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
5699         die "unable to open file '$archive'\n";
5700     my $firstfile = <$fh>;
5701     kill 15, $pid;
5702     close $fh;
5703
5704     die "ERROR: archive contaions no data\n" if !$firstfile;
5705     chomp $firstfile;
5706
5707     return $firstfile;
5708 }
5709
5710 sub tar_restore_cleanup {
5711     my ($storecfg, $statfile) = @_;
5712
5713     print STDERR "starting cleanup\n";
5714
5715     if (my $fd = IO::File->new($statfile, "r")) {
5716         while (defined(my $line = <$fd>)) {
5717             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5718                 my $volid = $2;
5719                 eval {
5720                     if ($volid =~ m|^/|) {
5721                         unlink $volid || die 'unlink failed\n';
5722                     } else {
5723                         PVE::Storage::vdisk_free($storecfg, $volid);
5724                     }
5725                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
5726                 };
5727                 print STDERR "unable to cleanup '$volid' - $@" if $@;
5728             } else {
5729                 print STDERR "unable to parse line in statfile - $line";
5730             }
5731         }
5732         $fd->close();
5733     }
5734 }
5735
5736 sub restore_archive {
5737     my ($archive, $vmid, $user, $opts) = @_;
5738
5739     my $format = $opts->{format};
5740     my $comp;
5741
5742     if ($archive =~ m/\.tgz$/ || $archive =~ m/\.tar\.gz$/) {
5743         $format = 'tar' if !$format;
5744         $comp = 'gzip';
5745     } elsif ($archive =~ m/\.tar$/) {
5746         $format = 'tar' if !$format;
5747     } elsif ($archive =~ m/.tar.lzo$/) {
5748         $format = 'tar' if !$format;
5749         $comp = 'lzop';
5750     } elsif ($archive =~ m/\.vma$/) {
5751         $format = 'vma' if !$format;
5752     } elsif ($archive =~ m/\.vma\.gz$/) {
5753         $format = 'vma' if !$format;
5754         $comp = 'gzip';
5755     } elsif ($archive =~ m/\.vma\.lzo$/) {
5756         $format = 'vma' if !$format;
5757         $comp = 'lzop';
5758     } else {
5759         $format = 'vma' if !$format; # default
5760     }
5761
5762     # try to detect archive format
5763     if ($format eq 'tar') {
5764         return restore_tar_archive($archive, $vmid, $user, $opts);
5765     } else {
5766         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
5767     }
5768 }
5769
5770 sub restore_update_config_line {
5771     my ($outfd, $cookie, $vmid, $map, $line, $unique) = @_;
5772
5773     return if $line =~ m/^\#qmdump\#/;
5774     return if $line =~ m/^\#vzdump\#/;
5775     return if $line =~ m/^lock:/;
5776     return if $line =~ m/^unused\d+:/;
5777     return if $line =~ m/^parent:/;
5778     return if $line =~ m/^template:/; # restored VM is never a template
5779
5780     my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
5781     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
5782         # try to convert old 1.X settings
5783         my ($id, $ind, $ethcfg) = ($1, $2, $3);
5784         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
5785             my ($model, $macaddr) = split(/\=/, $devconfig);
5786             $macaddr = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if !$macaddr || $unique;
5787             my $net = {
5788                 model => $model,
5789                 bridge => "vmbr$ind",
5790                 macaddr => $macaddr,
5791             };
5792             my $netstr = print_net($net);
5793
5794             print $outfd "net$cookie->{netcount}: $netstr\n";
5795             $cookie->{netcount}++;
5796         }
5797     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
5798         my ($id, $netstr) = ($1, $2);
5799         my $net = parse_net($netstr);
5800         $net->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if $net->{macaddr};
5801         $netstr = print_net($net);
5802         print $outfd "$id: $netstr\n";
5803     } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk)\d+):\s*(\S+)\s*$/) {
5804         my $virtdev = $1;
5805         my $value = $3;
5806         my $di = parse_drive($virtdev, $value);
5807         if (defined($di->{backup}) && !$di->{backup}) {
5808             print $outfd "#$line";
5809         } elsif ($map->{$virtdev}) {
5810             delete $di->{format}; # format can change on restore
5811             $di->{file} = $map->{$virtdev};
5812             $value = print_drive($vmid, $di);
5813             print $outfd "$virtdev: $value\n";
5814         } else {
5815             print $outfd $line;
5816         }
5817     } elsif (($line =~ m/^vmgenid: (.*)/)) {
5818         my $vmgenid = $1;
5819         if ($vmgenid ne '0') {
5820             # always generate a new vmgenid if there was a valid one setup
5821             $vmgenid = generate_uuid();
5822         }
5823         print $outfd "vmgenid: $vmgenid\n";
5824     } elsif (($line =~ m/^(smbios1: )(.*)/) && $unique) {
5825         my ($uuid, $uuid_str);
5826         UUID::generate($uuid);
5827         UUID::unparse($uuid, $uuid_str);
5828         my $smbios1 = parse_smbios1($2);
5829         $smbios1->{uuid} = $uuid_str;
5830         print $outfd $1.print_smbios1($smbios1)."\n";
5831     } else {
5832         print $outfd $line;
5833     }
5834 }
5835
5836 sub scan_volids {
5837     my ($cfg, $vmid) = @_;
5838
5839     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid);
5840
5841     my $volid_hash = {};
5842     foreach my $storeid (keys %$info) {
5843         foreach my $item (@{$info->{$storeid}}) {
5844             next if !($item->{volid} && $item->{size});
5845             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
5846             $volid_hash->{$item->{volid}} = $item;
5847         }
5848     }
5849
5850     return $volid_hash;
5851 }
5852
5853 sub is_volume_in_use {
5854     my ($storecfg, $conf, $skip_drive, $volid) = @_;
5855
5856     my $path = PVE::Storage::path($storecfg, $volid);
5857
5858     my $scan_config = sub {
5859         my ($cref, $snapname) = @_;
5860
5861         foreach my $key (keys %$cref) {
5862             my $value = $cref->{$key};
5863             if (is_valid_drivename($key)) {
5864                 next if $skip_drive && $key eq $skip_drive;
5865                 my $drive = parse_drive($key, $value);
5866                 next if !$drive || !$drive->{file} || drive_is_cdrom($drive);
5867                 return 1 if $volid eq $drive->{file};
5868                 if ($drive->{file} =~ m!^/!) {
5869                     return 1 if $drive->{file} eq $path;
5870                 } else {
5871                     my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file}, 1);
5872                     next if !$storeid;
5873                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid, 1);
5874                     next if !$scfg;
5875                     return 1 if $path eq PVE::Storage::path($storecfg, $drive->{file}, $snapname);
5876                 }
5877             }
5878         }
5879
5880         return 0;
5881     };
5882
5883     return 1 if &$scan_config($conf);
5884
5885     undef $skip_drive;
5886
5887     foreach my $snapname (keys %{$conf->{snapshots}}) {
5888         return 1 if &$scan_config($conf->{snapshots}->{$snapname}, $snapname);
5889     }
5890
5891     return 0;
5892 }
5893
5894 sub update_disksize {
5895     my ($vmid, $conf, $volid_hash) = @_;
5896
5897     my $changes;
5898     my $prefix = "VM $vmid:";
5899
5900     # used and unused disks
5901     my $referenced = {};
5902
5903     # Note: it is allowed to define multiple storages with same path (alias), so
5904     # we need to check both 'volid' and real 'path' (two different volid can point
5905     # to the same path).
5906
5907     my $referencedpath = {};
5908
5909     # update size info
5910     foreach my $opt (keys %$conf) {
5911         if (is_valid_drivename($opt)) {
5912             my $drive = parse_drive($opt, $conf->{$opt});
5913             my $volid = $drive->{file};
5914             next if !$volid;
5915
5916             $referenced->{$volid} = 1;
5917             if ($volid_hash->{$volid} &&
5918                 (my $path = $volid_hash->{$volid}->{path})) {
5919                 $referencedpath->{$path} = 1;
5920             }
5921
5922             next if drive_is_cdrom($drive);
5923             next if !$volid_hash->{$volid};
5924
5925             $drive->{size} = $volid_hash->{$volid}->{size};
5926             my $new = print_drive($vmid, $drive);
5927             if ($new ne $conf->{$opt}) {
5928                 $changes = 1;
5929                 $conf->{$opt} = $new;
5930                 print "$prefix update disk '$opt' information.\n";
5931             }
5932         }
5933     }
5934
5935     # remove 'unusedX' entry if volume is used
5936     foreach my $opt (keys %$conf) {
5937         next if $opt !~ m/^unused\d+$/;
5938         my $volid = $conf->{$opt};
5939         my $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
5940         if ($referenced->{$volid} || ($path && $referencedpath->{$path})) {
5941             print "$prefix remove entry '$opt', its volume '$volid' is in use.\n";
5942             $changes = 1;
5943             delete $conf->{$opt};
5944         }
5945
5946         $referenced->{$volid} = 1;
5947         $referencedpath->{$path} = 1 if $path;
5948     }
5949
5950     foreach my $volid (sort keys %$volid_hash) {
5951         next if $volid =~ m/vm-$vmid-state-/;
5952         next if $referenced->{$volid};
5953         my $path = $volid_hash->{$volid}->{path};
5954         next if !$path; # just to be sure
5955         next if $referencedpath->{$path};
5956         $changes = 1;
5957         my $key = PVE::QemuConfig->add_unused_volume($conf, $volid);
5958         print "$prefix add unreferenced volume '$volid' as '$key' to config.\n";
5959         $referencedpath->{$path} = 1; # avoid to add more than once (aliases)
5960     }
5961
5962     return $changes;
5963 }
5964
5965 sub rescan {
5966     my ($vmid, $nolock, $dryrun) = @_;
5967
5968     my $cfg = PVE::Storage::config();
5969
5970     # FIXME: Remove once our RBD plugin can handle CT and VM on a single storage
5971     # see: https://pve.proxmox.com/pipermail/pve-devel/2018-July/032900.html
5972     foreach my $stor (keys %{$cfg->{ids}}) {
5973         delete($cfg->{ids}->{$stor}) if ! $cfg->{ids}->{$stor}->{content}->{images};
5974     }
5975
5976     print "rescan volumes...\n";
5977     my $volid_hash = scan_volids($cfg, $vmid);
5978
5979     my $updatefn =  sub {
5980         my ($vmid) = @_;
5981
5982         my $conf = PVE::QemuConfig->load_config($vmid);
5983
5984         PVE::QemuConfig->check_lock($conf);
5985
5986         my $vm_volids = {};
5987         foreach my $volid (keys %$volid_hash) {
5988             my $info = $volid_hash->{$volid};
5989             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
5990         }
5991
5992         my $changes = update_disksize($vmid, $conf, $vm_volids);
5993
5994         PVE::QemuConfig->write_config($vmid, $conf) if $changes && !$dryrun;
5995     };
5996
5997     if (defined($vmid)) {
5998         if ($nolock) {
5999             &$updatefn($vmid);
6000         } else {
6001             PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6002         }
6003     } else {
6004         my $vmlist = config_list();
6005         foreach my $vmid (keys %$vmlist) {
6006             if ($nolock) {
6007                 &$updatefn($vmid);
6008             } else {
6009                 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6010             }
6011         }
6012     }
6013 }
6014
6015 sub restore_vma_archive {
6016     my ($archive, $vmid, $user, $opts, $comp) = @_;
6017
6018     my $readfrom = $archive;
6019
6020     my $cfg = PVE::Storage::config();
6021     my $commands = [];
6022     my $bwlimit = $opts->{bwlimit};
6023
6024     my $dbg_cmdstring = '';
6025     my $add_pipe = sub {
6026         my ($cmd) = @_;
6027         push @$commands, $cmd;
6028         $dbg_cmdstring .= ' | ' if length($dbg_cmdstring);
6029         $dbg_cmdstring .= PVE::Tools::cmd2string($cmd);
6030         $readfrom = '-';
6031     };
6032
6033     my $input = undef;
6034     if ($archive eq '-') {
6035         $input = '<&STDIN';
6036     } else {
6037         # If we use a backup from a PVE defined storage we also consider that
6038         # storage's rate limit:
6039         my (undef, $volid) = PVE::Storage::path_to_volume_id($cfg, $archive);
6040         if (defined($volid)) {
6041             my ($sid, undef) = PVE::Storage::parse_volume_id($volid);
6042             my $readlimit = PVE::Storage::get_bandwidth_limit('restore', [$sid], $bwlimit);
6043             if ($readlimit) {
6044                 print STDERR "applying read rate limit: $readlimit\n";
6045                 my $cstream = ['cstream', '-t', $readlimit*1024, '--', $readfrom];
6046                 $add_pipe->($cstream);
6047             }
6048         }
6049     }
6050
6051     if ($comp) {
6052         my $cmd;
6053         if ($comp eq 'gzip') {
6054             $cmd = ['zcat', $readfrom];
6055         } elsif ($comp eq 'lzop') {
6056             $cmd = ['lzop', '-d', '-c', $readfrom];
6057         } else {
6058             die "unknown compression method '$comp'\n";
6059         }
6060         $add_pipe->($cmd);
6061     }
6062
6063     my $tmpdir = "/var/tmp/vzdumptmp$$";
6064     rmtree $tmpdir;
6065
6066     # disable interrupts (always do cleanups)
6067     local $SIG{INT} =
6068         local $SIG{TERM} =
6069         local $SIG{QUIT} =
6070         local $SIG{HUP} = sub { warn "got interrupt - ignored\n"; };
6071
6072     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
6073     POSIX::mkfifo($mapfifo, 0600);
6074     my $fifofh;
6075
6076     my $openfifo = sub {
6077         open($fifofh, '>', $mapfifo) || die $!;
6078     };
6079
6080     $add_pipe->(['vma', 'extract', '-v', '-r', $mapfifo, $readfrom, $tmpdir]);
6081
6082     my $oldtimeout;
6083     my $timeout = 5;
6084
6085     my $devinfo = {};
6086
6087     my $rpcenv = PVE::RPCEnvironment::get();
6088
6089     my $conffile = PVE::QemuConfig->config_file($vmid);
6090     my $tmpfn = "$conffile.$$.tmp";
6091
6092     # Note: $oldconf is undef if VM does not exists
6093     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
6094     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
6095
6096     my %storage_limits;
6097
6098     my $print_devmap = sub {
6099         my $virtdev_hash = {};
6100
6101         my $cfgfn = "$tmpdir/qemu-server.conf";
6102
6103         # we can read the config - that is already extracted
6104         my $fh = IO::File->new($cfgfn, "r") ||
6105             "unable to read qemu-server.conf - $!\n";
6106
6107         my $fwcfgfn = "$tmpdir/qemu-server.fw";
6108         if (-f $fwcfgfn) {
6109             my $pve_firewall_dir = '/etc/pve/firewall';
6110             mkdir $pve_firewall_dir; # make sure the dir exists
6111             PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
6112         }
6113
6114         while (defined(my $line = <$fh>)) {
6115             if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
6116                 my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
6117                 die "archive does not contain data for drive '$virtdev'\n"
6118                     if !$devinfo->{$devname};
6119                 if (defined($opts->{storage})) {
6120                     $storeid = $opts->{storage} || 'local';
6121                 } elsif (!$storeid) {
6122                     $storeid = 'local';
6123                 }
6124                 $format = 'raw' if !$format;
6125                 $devinfo->{$devname}->{devname} = $devname;
6126                 $devinfo->{$devname}->{virtdev} = $virtdev;
6127                 $devinfo->{$devname}->{format} = $format;
6128                 $devinfo->{$devname}->{storeid} = $storeid;
6129
6130                 # check permission on storage
6131                 my $pool = $opts->{pool}; # todo: do we need that?
6132                 if ($user ne 'root@pam') {
6133                     $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
6134                 }
6135
6136                 $storage_limits{$storeid} = $bwlimit;
6137
6138                 $virtdev_hash->{$virtdev} = $devinfo->{$devname};
6139             }
6140         }
6141
6142         foreach my $key (keys %storage_limits) {
6143             my $limit = PVE::Storage::get_bandwidth_limit('restore', [$key], $bwlimit);
6144             next if !$limit;
6145             print STDERR "rate limit for storage $key: $limit KiB/s\n";
6146             $storage_limits{$key} = $limit * 1024;
6147         }
6148
6149         foreach my $devname (keys %$devinfo) {
6150             die "found no device mapping information for device '$devname'\n"
6151                 if !$devinfo->{$devname}->{virtdev};
6152         }
6153
6154         # create empty/temp config
6155         if ($oldconf) {
6156             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
6157             foreach_drive($oldconf, sub {
6158                 my ($ds, $drive) = @_;
6159
6160                 return if drive_is_cdrom($drive);
6161
6162                 my $volid = $drive->{file};
6163
6164                 return if !$volid || $volid =~ m|^/|;
6165
6166                 my ($path, $owner) = PVE::Storage::path($cfg, $volid);
6167                 return if !$path || !$owner || ($owner != $vmid);
6168
6169                 # Note: only delete disk we want to restore
6170                 # other volumes will become unused
6171                 if ($virtdev_hash->{$ds}) {
6172                     eval { PVE::Storage::vdisk_free($cfg, $volid); };
6173                     if (my $err = $@) {
6174                         warn $err;
6175                     }
6176                 }
6177             });
6178
6179             # delete vmstate files
6180             # since after the restore we have no snapshots anymore
6181             foreach my $snapname (keys %{$oldconf->{snapshots}}) {
6182                 my $snap = $oldconf->{snapshots}->{$snapname};
6183                 if ($snap->{vmstate}) {
6184                     eval { PVE::Storage::vdisk_free($cfg, $snap->{vmstate}); };
6185                     if (my $err = $@) {
6186                         warn $err;
6187                     }
6188                 }
6189             }
6190         }
6191
6192         my $map = {};
6193         foreach my $virtdev (sort keys %$virtdev_hash) {
6194             my $d = $virtdev_hash->{$virtdev};
6195             my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
6196             my $storeid = $d->{storeid};
6197             my $scfg = PVE::Storage::storage_config($cfg, $storeid);
6198
6199             my $map_opts = '';
6200             if (my $limit = $storage_limits{$storeid}) {
6201                 $map_opts .= "throttling.bps=$limit:throttling.group=$storeid:";
6202             }
6203
6204             # test if requested format is supported
6205             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($cfg, $storeid);
6206             my $supported = grep { $_ eq $d->{format} } @$validFormats;
6207             $d->{format} = $defFormat if !$supported;
6208
6209             my $volid = PVE::Storage::vdisk_alloc($cfg, $storeid, $vmid,
6210                                                   $d->{format}, undef, $alloc_size);
6211             print STDERR "new volume ID is '$volid'\n";
6212             $d->{volid} = $volid;
6213             my $path = PVE::Storage::path($cfg, $volid);
6214
6215             PVE::Storage::activate_volumes($cfg,[$volid]);
6216
6217             my $write_zeros = 1;
6218             if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
6219                 $write_zeros = 0;
6220             }
6221
6222             print $fifofh "${map_opts}format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
6223
6224             print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
6225             $map->{$virtdev} = $volid;
6226         }
6227
6228         $fh->seek(0, 0) || die "seek failed - $!\n";
6229
6230         my $outfd = new IO::File ($tmpfn, "w") ||
6231             die "unable to write config for VM $vmid\n";
6232
6233         my $cookie = { netcount => 0 };
6234         while (defined(my $line = <$fh>)) {
6235             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
6236         }
6237
6238         $fh->close();
6239         $outfd->close();
6240     };
6241
6242     eval {
6243         # enable interrupts
6244         local $SIG{INT} =
6245             local $SIG{TERM} =
6246             local $SIG{QUIT} =
6247             local $SIG{HUP} =
6248             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6249         local $SIG{ALRM} = sub { die "got timeout\n"; };
6250
6251         $oldtimeout = alarm($timeout);
6252
6253         my $parser = sub {
6254             my $line = shift;
6255
6256             print "$line\n";
6257
6258             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
6259                 my ($dev_id, $size, $devname) = ($1, $2, $3);
6260                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
6261             } elsif ($line =~ m/^CTIME: /) {
6262                 # we correctly received the vma config, so we can disable
6263                 # the timeout now for disk allocation (set to 10 minutes, so
6264                 # that we always timeout if something goes wrong)
6265                 alarm(600);
6266                 &$print_devmap();
6267                 print $fifofh "done\n";
6268                 my $tmp = $oldtimeout || 0;
6269                 $oldtimeout = undef;
6270                 alarm($tmp);
6271                 close($fifofh);
6272             }
6273         };
6274
6275         print "restore vma archive: $dbg_cmdstring\n";
6276         run_command($commands, input => $input, outfunc => $parser, afterfork => $openfifo);
6277     };
6278     my $err = $@;
6279
6280     alarm($oldtimeout) if $oldtimeout;
6281
6282     my $vollist = [];
6283     foreach my $devname (keys %$devinfo) {
6284         my $volid = $devinfo->{$devname}->{volid};
6285         push @$vollist, $volid if $volid;
6286     }
6287
6288     PVE::Storage::deactivate_volumes($cfg, $vollist);
6289
6290     unlink $mapfifo;
6291
6292     if ($err) {
6293         rmtree $tmpdir;
6294         unlink $tmpfn;
6295
6296         foreach my $devname (keys %$devinfo) {
6297             my $volid = $devinfo->{$devname}->{volid};
6298             next if !$volid;
6299             eval {
6300                 if ($volid =~ m|^/|) {
6301                     unlink $volid || die 'unlink failed\n';
6302                 } else {
6303                     PVE::Storage::vdisk_free($cfg, $volid);
6304                 }
6305                 print STDERR "temporary volume '$volid' sucessfuly removed\n";
6306             };
6307             print STDERR "unable to cleanup '$volid' - $@" if $@;
6308         }
6309         die $err;
6310     }
6311
6312     rmtree $tmpdir;
6313
6314     rename($tmpfn, $conffile) ||
6315         die "unable to commit configuration file '$conffile'\n";
6316
6317     PVE::Cluster::cfs_update(); # make sure we read new file
6318
6319     eval { rescan($vmid, 1); };
6320     warn $@ if $@;
6321 }
6322
6323 sub restore_tar_archive {
6324     my ($archive, $vmid, $user, $opts) = @_;
6325
6326     if ($archive ne '-') {
6327         my $firstfile = tar_archive_read_firstfile($archive);
6328         die "ERROR: file '$archive' dos not lock like a QemuServer vzdump backup\n"
6329             if $firstfile ne 'qemu-server.conf';
6330     }
6331
6332     my $storecfg = PVE::Storage::config();
6333
6334     # destroy existing data - keep empty config
6335     my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
6336     destroy_vm($storecfg, $vmid, 1) if -f $vmcfgfn;
6337
6338     my $tocmd = "/usr/lib/qemu-server/qmextract";
6339
6340     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
6341     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
6342     $tocmd .= ' --prealloc' if $opts->{prealloc};
6343     $tocmd .= ' --info' if $opts->{info};
6344
6345     # tar option "xf" does not autodetect compression when read from STDIN,
6346     # so we pipe to zcat
6347     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
6348         PVE::Tools::shellquote("--to-command=$tocmd");
6349
6350     my $tmpdir = "/var/tmp/vzdumptmp$$";
6351     mkpath $tmpdir;
6352
6353     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
6354     local $ENV{VZDUMP_VMID} = $vmid;
6355     local $ENV{VZDUMP_USER} = $user;
6356
6357     my $conffile = PVE::QemuConfig->config_file($vmid);
6358     my $tmpfn = "$conffile.$$.tmp";
6359
6360     # disable interrupts (always do cleanups)
6361     local $SIG{INT} =
6362         local $SIG{TERM} =
6363         local $SIG{QUIT} =
6364         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
6365
6366     eval {
6367         # enable interrupts
6368         local $SIG{INT} =
6369             local $SIG{TERM} =
6370             local $SIG{QUIT} =
6371             local $SIG{HUP} =
6372             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6373
6374         if ($archive eq '-') {
6375             print "extracting archive from STDIN\n";
6376             run_command($cmd, input => "<&STDIN");
6377         } else {
6378             print "extracting archive '$archive'\n";
6379             run_command($cmd);
6380         }
6381
6382         return if $opts->{info};
6383
6384         # read new mapping
6385         my $map = {};
6386         my $statfile = "$tmpdir/qmrestore.stat";
6387         if (my $fd = IO::File->new($statfile, "r")) {
6388             while (defined (my $line = <$fd>)) {
6389                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
6390                     $map->{$1} = $2 if $1;
6391                 } else {
6392                     print STDERR "unable to parse line in statfile - $line\n";
6393                 }
6394             }
6395             $fd->close();
6396         }
6397
6398         my $confsrc = "$tmpdir/qemu-server.conf";
6399
6400         my $srcfd = new IO::File($confsrc, "r") ||
6401             die "unable to open file '$confsrc'\n";
6402
6403         my $outfd = new IO::File ($tmpfn, "w") ||
6404             die "unable to write config for VM $vmid\n";
6405
6406         my $cookie = { netcount => 0 };
6407         while (defined (my $line = <$srcfd>)) {
6408             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
6409         }
6410
6411         $srcfd->close();
6412         $outfd->close();
6413     };
6414     my $err = $@;
6415
6416     if ($err) {
6417
6418         unlink $tmpfn;
6419
6420         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
6421
6422         die $err;
6423     }
6424
6425     rmtree $tmpdir;
6426
6427     rename $tmpfn, $conffile ||
6428         die "unable to commit configuration file '$conffile'\n";
6429
6430     PVE::Cluster::cfs_update(); # make sure we read new file
6431
6432     eval { rescan($vmid, 1); };
6433     warn $@ if $@;
6434 };
6435
6436 sub foreach_storage_used_by_vm {
6437     my ($conf, $func) = @_;
6438
6439     my $sidhash = {};
6440
6441     foreach_drive($conf, sub {
6442         my ($ds, $drive) = @_;
6443         return if drive_is_cdrom($drive);
6444
6445         my $volid = $drive->{file};
6446
6447         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
6448         $sidhash->{$sid} = $sid if $sid;
6449     });
6450
6451     foreach my $sid (sort keys %$sidhash) {
6452         &$func($sid);
6453     }
6454 }
6455
6456 sub do_snapshots_with_qemu {
6457     my ($storecfg, $volid) = @_;
6458
6459     my $storage_name = PVE::Storage::parse_volume_id($volid);
6460
6461     if ($qemu_snap_storage->{$storecfg->{ids}->{$storage_name}->{type}}
6462         && !$storecfg->{ids}->{$storage_name}->{krbd}){
6463         return 1;
6464     }
6465
6466     if ($volid =~ m/\.(qcow2|qed)$/){
6467         return 1;
6468     }
6469
6470     return undef;
6471 }
6472
6473 sub qga_check_running {
6474     my ($vmid, $nowarn) = @_;
6475
6476     eval { vm_mon_cmd($vmid, "guest-ping", timeout => 3); };
6477     if ($@) {
6478         warn "Qemu Guest Agent is not running - $@" if !$nowarn;
6479         return 0;
6480     }
6481     return 1;
6482 }
6483
6484 sub template_create {
6485     my ($vmid, $conf, $disk) = @_;
6486
6487     my $storecfg = PVE::Storage::config();
6488
6489     foreach_drive($conf, sub {
6490         my ($ds, $drive) = @_;
6491
6492         return if drive_is_cdrom($drive);
6493         return if $disk && $ds ne $disk;
6494
6495         my $volid = $drive->{file};
6496         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
6497
6498         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
6499         $drive->{file} = $voliddst;
6500         $conf->{$ds} = print_drive($vmid, $drive);
6501         PVE::QemuConfig->write_config($vmid, $conf);
6502     });
6503 }
6504
6505 sub qemu_img_convert {
6506     my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized) = @_;
6507
6508     my $storecfg = PVE::Storage::config();
6509     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
6510     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
6511
6512     if ($src_storeid && $dst_storeid) {
6513
6514         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
6515
6516         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
6517         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6518
6519         my $src_format = qemu_img_format($src_scfg, $src_volname);
6520         my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
6521
6522         my $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
6523         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6524
6525         my $cmd = [];
6526         push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
6527         push @$cmd, '-l', "snapshot.name=$snapname" if($snapname && $src_format eq "qcow2");
6528         push @$cmd, '-t', 'none' if $dst_scfg->{type} eq 'zfspool';
6529         push @$cmd, '-T', 'none' if $src_scfg->{type} eq 'zfspool';
6530         push @$cmd, '-f', $src_format, '-O', $dst_format, $src_path;
6531         if ($is_zero_initialized) {
6532             push @$cmd, "zeroinit:$dst_path";
6533         } else {
6534             push @$cmd, $dst_path;
6535         }
6536
6537         my $parser = sub {
6538             my $line = shift;
6539             if($line =~ m/\((\S+)\/100\%\)/){
6540                 my $percent = $1;
6541                 my $transferred = int($size * $percent / 100);
6542                 my $remaining = $size - $transferred;
6543
6544                 print "transferred: $transferred bytes remaining: $remaining bytes total: $size bytes progression: $percent %\n";
6545             }
6546
6547         };
6548
6549         eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
6550         my $err = $@;
6551         die "copy failed: $err" if $err;
6552     }
6553 }
6554
6555 sub qemu_img_format {
6556     my ($scfg, $volname) = @_;
6557
6558     if ($scfg->{path} && $volname =~ m/\.($QEMU_FORMAT_RE)$/) {
6559         return $1;
6560     } else {
6561         return "raw";
6562     }
6563 }
6564
6565 sub qemu_drive_mirror {
6566     my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $skipcomplete, $qga) = @_;
6567
6568     $jobs = {} if !$jobs;
6569
6570     my $qemu_target;
6571     my $format;
6572     $jobs->{"drive-$drive"} = {};
6573
6574     if ($dst_volid =~ /^nbd:/) {
6575         $qemu_target = $dst_volid;
6576         $format = "nbd";
6577     } else {
6578         my $storecfg = PVE::Storage::config();
6579         my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
6580
6581         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6582
6583         $format = qemu_img_format($dst_scfg, $dst_volname);
6584
6585         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6586
6587         $qemu_target = $is_zero_initialized ? "zeroinit:$dst_path" : $dst_path;
6588     }
6589
6590     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $qemu_target };
6591     $opts->{format} = $format if $format;
6592
6593     print "drive mirror is starting for drive-$drive\n";
6594
6595     eval { vm_mon_cmd($vmid, "drive-mirror", %$opts); }; #if a job already run for this device,it's throw an error
6596
6597     if (my $err = $@) {
6598         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6599         die "mirroring error: $err";
6600     }
6601
6602     qemu_drive_mirror_monitor ($vmid, $vmiddst, $jobs, $skipcomplete, $qga);
6603 }
6604
6605 sub qemu_drive_mirror_monitor {
6606     my ($vmid, $vmiddst, $jobs, $skipcomplete, $qga) = @_;
6607
6608     eval {
6609         my $err_complete = 0;
6610
6611         while (1) {
6612             die "storage migration timed out\n" if $err_complete > 300;
6613
6614             my $stats = vm_mon_cmd($vmid, "query-block-jobs");
6615
6616             my $running_mirror_jobs = {};
6617             foreach my $stat (@$stats) {
6618                 next if $stat->{type} ne 'mirror';
6619                 $running_mirror_jobs->{$stat->{device}} = $stat;
6620             }
6621
6622             my $readycounter = 0;
6623
6624             foreach my $job (keys %$jobs) {
6625
6626                 if(defined($jobs->{$job}->{complete}) && !defined($running_mirror_jobs->{$job})) {
6627                     print "$job : finished\n";
6628                     delete $jobs->{$job};
6629                     next;
6630                 }
6631
6632                 die "$job: mirroring has been cancelled\n" if !defined($running_mirror_jobs->{$job});
6633
6634                 my $busy = $running_mirror_jobs->{$job}->{busy};
6635                 my $ready = $running_mirror_jobs->{$job}->{ready};
6636                 if (my $total = $running_mirror_jobs->{$job}->{len}) {
6637                     my $transferred = $running_mirror_jobs->{$job}->{offset} || 0;
6638                     my $remaining = $total - $transferred;
6639                     my $percent = sprintf "%.2f", ($transferred * 100 / $total);
6640
6641                     print "$job: transferred: $transferred bytes remaining: $remaining bytes total: $total bytes progression: $percent % busy: $busy ready: $ready \n";
6642                 }
6643
6644                 $readycounter++ if $running_mirror_jobs->{$job}->{ready};
6645             }
6646
6647             last if scalar(keys %$jobs) == 0;
6648
6649             if ($readycounter == scalar(keys %$jobs)) {
6650                 print "all mirroring jobs are ready \n";
6651                 last if $skipcomplete; #do the complete later
6652
6653                 if ($vmiddst && $vmiddst != $vmid) {
6654                     my $agent_running = $qga && qga_check_running($vmid);
6655                     if ($agent_running) {
6656                         print "freeze filesystem\n";
6657                         eval { PVE::QemuServer::vm_mon_cmd($vmid, "guest-fsfreeze-freeze"); };
6658                     } else {
6659                         print "suspend vm\n";
6660                         eval { PVE::QemuServer::vm_suspend($vmid, 1); };
6661                     }
6662
6663                     # if we clone a disk for a new target vm, we don't switch the disk
6664                     PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs);
6665
6666                     if ($agent_running) {
6667                         print "unfreeze filesystem\n";
6668                         eval { PVE::QemuServer::vm_mon_cmd($vmid, "guest-fsfreeze-thaw"); };
6669                     } else {
6670                         print "resume vm\n";
6671                         eval {  PVE::QemuServer::vm_resume($vmid, 1, 1); };
6672                     }
6673
6674                     last;
6675                 } else {
6676
6677                     foreach my $job (keys %$jobs) {
6678                         # try to switch the disk if source and destination are on the same guest
6679                         print "$job: Completing block job...\n";
6680
6681                         eval { vm_mon_cmd($vmid, "block-job-complete", device => $job) };
6682                         if ($@ =~ m/cannot be completed/) {
6683                             print "$job: Block job cannot be completed, try again.\n";
6684                             $err_complete++;
6685                         }else {
6686                             print "$job: Completed successfully.\n";
6687                             $jobs->{$job}->{complete} = 1;
6688                         }
6689                     }
6690                 }
6691             }
6692             sleep 1;
6693         }
6694     };
6695     my $err = $@;
6696
6697     if ($err) {
6698         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6699         die "mirroring error: $err";
6700     }
6701
6702 }
6703
6704 sub qemu_blockjobs_cancel {
6705     my ($vmid, $jobs) = @_;
6706
6707     foreach my $job (keys %$jobs) {
6708         print "$job: Cancelling block job\n";
6709         eval { vm_mon_cmd($vmid, "block-job-cancel", device => $job); };
6710         $jobs->{$job}->{cancel} = 1;
6711     }
6712
6713     while (1) {
6714         my $stats = vm_mon_cmd($vmid, "query-block-jobs");
6715
6716         my $running_jobs = {};
6717         foreach my $stat (@$stats) {
6718             $running_jobs->{$stat->{device}} = $stat;
6719         }
6720
6721         foreach my $job (keys %$jobs) {
6722
6723             if (defined($jobs->{$job}->{cancel}) && !defined($running_jobs->{$job})) {
6724                 print "$job: Done.\n";
6725                 delete $jobs->{$job};
6726             }
6727         }
6728
6729         last if scalar(keys %$jobs) == 0;
6730
6731         sleep 1;
6732     }
6733 }
6734
6735 sub clone_disk {
6736     my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
6737         $newvmid, $storage, $format, $full, $newvollist, $jobs, $skipcomplete, $qga) = @_;
6738
6739     my $newvolid;
6740
6741     if (!$full) {
6742         print "create linked clone of drive $drivename ($drive->{file})\n";
6743         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
6744         push @$newvollist, $newvolid;
6745     } else {
6746
6747         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6748         $storeid = $storage if $storage;
6749
6750         my $dst_format = resolve_dst_disk_format($storecfg, $storeid, $volname, $format);
6751         my ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 3);
6752
6753         print "create full clone of drive $drivename ($drive->{file})\n";
6754         my $name = undef;
6755         if (drive_is_cloudinit($drive)) {
6756             $name = "vm-$newvmid-cloudinit";
6757             $snapname = undef;
6758             # cloudinit only supports raw and qcow2 atm:
6759             if ($dst_format eq 'qcow2') {
6760                 $name .= '.qcow2';
6761             } elsif ($dst_format ne 'raw') {
6762                 die "clone: unhandled format for cloudinit image\n";
6763             }
6764         }
6765         $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $newvmid, $dst_format, $name, ($size/1024));
6766         push @$newvollist, $newvolid;
6767
6768         PVE::Storage::activate_volumes($storecfg, [$newvolid]);
6769
6770         my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
6771         if (!$running || $snapname) {
6772             qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit);
6773         } else {
6774
6775             my $kvmver = get_running_qemu_version ($vmid);
6776             if (!qemu_machine_feature_enabled (undef, $kvmver, 2, 7)) {
6777                 die "drive-mirror with iothread requires qemu version 2.7 or higher\n"
6778                     if $drive->{iothread};
6779             }
6780
6781             qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid, $sparseinit, $jobs, $skipcomplete, $qga);
6782         }
6783     }
6784
6785     my ($size) = PVE::Storage::volume_size_info($storecfg, $newvolid, 3);
6786
6787     my $disk = $drive;
6788     $disk->{format} = undef;
6789     $disk->{file} = $newvolid;
6790     $disk->{size} = $size;
6791
6792     return $disk;
6793 }
6794
6795 # this only works if VM is running
6796 sub get_current_qemu_machine {
6797     my ($vmid) = @_;
6798
6799     my $cmd = { execute => 'query-machines', arguments => {} };
6800     my $res = vm_qmp_command($vmid, $cmd);
6801
6802     my ($current, $default);
6803     foreach my $e (@$res) {
6804         $default = $e->{name} if $e->{'is-default'};
6805         $current = $e->{name} if $e->{'is-current'};
6806     }
6807
6808     # fallback to the default machine if current is not supported by qemu
6809     return $current || $default || 'pc';
6810 }
6811
6812 sub get_running_qemu_version {
6813     my ($vmid) = @_;
6814     my $cmd = { execute => 'query-version', arguments => {} };
6815     my $res = vm_qmp_command($vmid, $cmd);
6816     return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
6817 }
6818
6819 sub qemu_machine_feature_enabled {
6820     my ($machine, $kvmver, $version_major, $version_minor) = @_;
6821
6822     my $current_major;
6823     my $current_minor;
6824
6825     if ($machine && $machine =~ m/^((?:pc(-i440fx|-q35)?|virt)-(\d+)\.(\d+))/) {
6826
6827         $current_major = $3;
6828         $current_minor = $4;
6829
6830     } elsif ($kvmver =~ m/^(\d+)\.(\d+)/) {
6831
6832         $current_major = $1;
6833         $current_minor = $2;
6834     }
6835
6836     return 1 if $current_major > $version_major ||
6837                 ($current_major == $version_major &&
6838                  $current_minor >= $version_minor);
6839 }
6840
6841 sub qemu_machine_pxe {
6842     my ($vmid, $conf, $machine) = @_;
6843
6844     $machine =  PVE::QemuServer::get_current_qemu_machine($vmid) if !$machine;
6845
6846     if ($conf->{machine} && $conf->{machine} =~ m/\.pxe$/) {
6847         $machine .= '.pxe';
6848     }
6849
6850     return $machine;
6851 }
6852
6853 sub qemu_use_old_bios_files {
6854     my ($machine_type) = @_;
6855
6856     return if !$machine_type;
6857
6858     my $use_old_bios_files = undef;
6859
6860     if ($machine_type =~ m/^(\S+)\.pxe$/) {
6861         $machine_type = $1;
6862         $use_old_bios_files = 1;
6863     } else {
6864         my $kvmver = kvm_user_version();
6865         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
6866         # load new efi bios files on migration. So this hack is required to allow
6867         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
6868         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
6869         $use_old_bios_files = !qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 4);
6870     }
6871
6872     return ($use_old_bios_files, $machine_type);
6873 }
6874
6875 sub create_efidisk($$$$$) {
6876     my ($storecfg, $storeid, $vmid, $fmt, $arch) = @_;
6877
6878     my (undef, $ovmf_vars) = get_ovmf_files($arch);
6879     die "EFI vars default image not found\n" if ! -f $ovmf_vars;
6880
6881     my $vars_size = PVE::Tools::convert_size(-s $ovmf_vars, 'b' => 'kb');
6882     my $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $fmt, undef, $vars_size);
6883     PVE::Storage::activate_volumes($storecfg, [$volid]);
6884
6885     my $path = PVE::Storage::path($storecfg, $volid);
6886     eval {
6887         run_command(['/usr/bin/qemu-img', 'convert', '-n', '-f', 'raw', '-O', $fmt, $ovmf_vars, $path]);
6888     };
6889     die "Copying EFI vars image failed: $@" if $@;
6890
6891     return ($volid, $vars_size);
6892 }
6893
6894 sub vm_iothreads_list {
6895     my ($vmid) = @_;
6896
6897     my $res = vm_mon_cmd($vmid, 'query-iothreads');
6898
6899     my $iothreads = {};
6900     foreach my $iothread (@$res) {
6901         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
6902     }
6903
6904     return $iothreads;
6905 }
6906
6907 sub scsihw_infos {
6908     my ($conf, $drive) = @_;
6909
6910     my $maxdev = 0;
6911
6912     if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)) {
6913         $maxdev = 7;
6914     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
6915         $maxdev = 1;
6916     } else {
6917         $maxdev = 256;
6918     }
6919
6920     my $controller = int($drive->{index} / $maxdev);
6921     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single') ? "virtioscsi" : "scsihw";
6922
6923     return ($maxdev, $controller, $controller_prefix);
6924 }
6925
6926 sub add_hyperv_enlightenments {
6927     my ($cpuFlags, $winversion, $machine_type, $kvmver, $bios, $gpu_passthrough, $hv_vendor_id) = @_;
6928
6929     return if $winversion < 6;
6930     return if $bios && $bios eq 'ovmf' && $winversion < 8;
6931
6932     if ($gpu_passthrough || defined($hv_vendor_id)) {
6933         $hv_vendor_id //= 'proxmox';
6934         push @$cpuFlags , "hv_vendor_id=$hv_vendor_id";
6935     }
6936
6937     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
6938         push @$cpuFlags , 'hv_spinlocks=0x1fff';
6939         push @$cpuFlags , 'hv_vapic';
6940         push @$cpuFlags , 'hv_time';
6941     } else {
6942         push @$cpuFlags , 'hv_spinlocks=0xffff';
6943     }
6944
6945     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 6)) {
6946         push @$cpuFlags , 'hv_reset';
6947         push @$cpuFlags , 'hv_vpindex';
6948         push @$cpuFlags , 'hv_runtime';
6949     }
6950
6951     if ($winversion >= 7) {
6952         push @$cpuFlags , 'hv_relaxed';
6953
6954         if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 12)) {
6955             push @$cpuFlags , 'hv_synic';
6956             push @$cpuFlags , 'hv_stimer';
6957         }
6958     }
6959 }
6960
6961 sub windows_version {
6962     my ($ostype) = @_;
6963
6964     return 0 if !$ostype;
6965
6966     my $winversion = 0;
6967
6968     if($ostype eq 'wxp' || $ostype eq 'w2k3' || $ostype eq 'w2k') {
6969         $winversion = 5;
6970     } elsif($ostype eq 'w2k8' || $ostype eq 'wvista') {
6971         $winversion = 6;
6972     } elsif ($ostype =~ m/^win(\d+)$/) {
6973         $winversion = $1;
6974     }
6975
6976     return $winversion;
6977 }
6978
6979 sub resolve_dst_disk_format {
6980         my ($storecfg, $storeid, $src_volname, $format) = @_;
6981         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
6982
6983         if (!$format) {
6984             # if no target format is specified, use the source disk format as hint
6985             if ($src_volname) {
6986                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6987                 $format = qemu_img_format($scfg, $src_volname);
6988             } else {
6989                 return $defFormat;
6990             }
6991         }
6992
6993         # test if requested format is supported - else use default
6994         my $supported = grep { $_ eq $format } @$validFormats;
6995         $format = $defFormat if !$supported;
6996         return $format;
6997 }
6998
6999 sub resolve_first_disk {
7000     my $conf = shift;
7001     my @disks = PVE::QemuServer::valid_drive_names();
7002     my $firstdisk;
7003     foreach my $ds (reverse @disks) {
7004         next if !$conf->{$ds};
7005         my $disk = PVE::QemuServer::parse_drive($ds, $conf->{$ds});
7006         next if PVE::QemuServer::drive_is_cdrom($disk);
7007         $firstdisk = $ds;
7008     }
7009     return $firstdisk;
7010 }
7011
7012 sub generate_uuid {
7013     my ($uuid, $uuid_str);
7014     UUID::generate($uuid);
7015     UUID::unparse($uuid, $uuid_str);
7016     return $uuid_str;
7017 }
7018
7019 sub generate_smbios1_uuid {
7020     return "uuid=".generate_uuid();
7021 }
7022
7023 sub nbd_stop {
7024     my ($vmid) = @_;
7025
7026     vm_mon_cmd($vmid, 'nbd-server-stop');
7027 }
7028
7029 # bash completion helper
7030
7031 sub complete_backup_archives {
7032     my ($cmdname, $pname, $cvalue) = @_;
7033
7034     my $cfg = PVE::Storage::config();
7035
7036     my $storeid;
7037
7038     if ($cvalue =~ m/^([^:]+):/) {
7039         $storeid = $1;
7040     }
7041
7042     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
7043
7044     my $res = [];
7045     foreach my $id (keys %$data) {
7046         foreach my $item (@{$data->{$id}}) {
7047             next if $item->{format} !~ m/^vma\.(gz|lzo)$/;
7048             push @$res, $item->{volid} if defined($item->{volid});
7049         }
7050     }
7051
7052     return $res;
7053 }
7054
7055 my $complete_vmid_full = sub {
7056     my ($running) = @_;
7057
7058     my $idlist = vmstatus();
7059
7060     my $res = [];
7061
7062     foreach my $id (keys %$idlist) {
7063         my $d = $idlist->{$id};
7064         if (defined($running)) {
7065             next if $d->{template};
7066             next if $running && $d->{status} ne 'running';
7067             next if !$running && $d->{status} eq 'running';
7068         }
7069         push @$res, $id;
7070
7071     }
7072     return $res;
7073 };
7074
7075 sub complete_vmid {
7076     return &$complete_vmid_full();
7077 }
7078
7079 sub complete_vmid_stopped {
7080     return &$complete_vmid_full(0);
7081 }
7082
7083 sub complete_vmid_running {
7084     return &$complete_vmid_full(1);
7085 }
7086
7087 sub complete_storage {
7088
7089     my $cfg = PVE::Storage::config();
7090     my $ids = $cfg->{ids};
7091
7092     my $res = [];
7093     foreach my $sid (keys %$ids) {
7094         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
7095         next if !$ids->{$sid}->{content}->{images};
7096         push @$res, $sid;
7097     }
7098
7099     return $res;
7100 }
7101
7102 1;