PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5 use POSIX;
   6 use IO::Handle;
   7 use IO::Select;
   8 use IO::File;
   9 use IO::Dir;
  10 use IO::Socket::UNIX;
  11 use File::Basename;
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use Digest::SHA;
  16 use Fcntl ':flock';
  17 use Cwd 'abs_path';
  18 use IPC::Open3;
  19 use JSON;
  20 use Fcntl;
  21 use PVE::SafeSyslog;
  22 use Storable qw(dclone);
  23 use PVE::Exception qw(raise raise_param_exc);
  24 use PVE::Storage;
  25 use PVE::Tools qw(run_command lock_file lock_file_full file_read_firstline dir_glob_foreach $IPV6RE);
  26 use PVE::JSONSchema qw(get_standard_option);
  27 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file cfs_lock_file);
  28 use PVE::INotify;
  29 use PVE::ProcFSTools;
  30 use PVE::QemuConfig;
  31 use PVE::QMPClient;
  32 use PVE::RPCEnvironment;
  33 use PVE::GuestHelpers;
  34 use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr);
  35 use PVE::QemuServer::Memory;
  36 use PVE::QemuServer::USB qw(parse_usb_device);
  37 use PVE::QemuServer::Cloudinit;
  38 use PVE::SysFSTools;
  39 use PVE::Systemd;
  40 use Time::HiRes qw(gettimeofday);
  41 use File::Copy qw(copy);
  42 use URI::Escape;
  43
  44 my $EDK2_FW_BASE = '/usr/share/pve-edk2-firmware/';
  45 my $OVMF = {
  46     x86_64 => [
  47         "$EDK2_FW_BASE/OVMF_CODE.fd",
  48         "$EDK2_FW_BASE/OVMF_VARS.fd"
  49     ],
  50     aarch64 => [
  51         "$EDK2_FW_BASE/AAVMF_CODE.fd",
  52         "$EDK2_FW_BASE/AAVMF_VARS.fd"
  53     ],
  54 };
  55
  56 my $qemu_snap_storage = {rbd => 1, sheepdog => 1};
  57
  58 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  59
  60 my $QEMU_FORMAT_RE = qr/raw|cow|qcow|qcow2|qed|vmdk|cloop/;
  61
  62 # Note about locking: we use flock on the config file protect
  63 # against concurent actions.
  64 # Aditionaly, we have a 'lock' setting in the config file. This
  65 # can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
  66 # allowed when such lock is set. But you can ignore this kind of
  67 # lock with the --skiplock flag.
  68
  69 cfs_register_file('/qemu-server/',
  70                   \&parse_vm_config,
  71                   \&write_vm_config);
  72
  73 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
  74     description => "Some command save/restore state from this location.",
  75     type => 'string',
  76     maxLength => 128,
  77     optional => 1,
  78 });
  79
  80 PVE::JSONSchema::register_standard_option('pve-snapshot-name', {
  81     description => "The name of the snapshot.",
  82     type => 'string', format => 'pve-configid',
  83     maxLength => 40,
  84 });
  85
  86 PVE::JSONSchema::register_standard_option('pve-qm-image-format', {
  87     type => 'string',
  88     enum => [qw(raw cow qcow qed qcow2 vmdk cloop)],
  89     description => "The drive's backing file's data format.",
  90     optional => 1,
  91 });
  92
  93 PVE::JSONSchema::register_standard_option('pve-qemu-machine', {
  94         description => "Specifies the Qemu machine type.",
  95         type => 'string',
  96         pattern => '(pc|pc(-i440fx)?-\d+\.\d+(\.pxe)?|q35|pc-q35-\d+\.\d+(\.pxe)?|virt(?:-\d+\.\d+)?)',
  97         maxLength => 40,
  98         optional => 1,
  99 });
 100
 101 #no warnings 'redefine';
 102
 103 sub cgroups_write {
 104    my ($controller, $vmid, $option, $value) = @_;
 105
 106    my $path = "/sys/fs/cgroup/$controller/qemu.slice/$vmid.scope/$option";
 107    PVE::ProcFSTools::write_proc_entry($path, $value);
 108
 109 }
 110
 111 my $nodename = PVE::INotify::nodename();
 112
 113 mkdir "/etc/pve/nodes/$nodename";
 114 my $confdir = "/etc/pve/nodes/$nodename/qemu-server";
 115 mkdir $confdir;
 116
 117 my $var_run_tmpdir = "/var/run/qemu-server";
 118 mkdir $var_run_tmpdir;
 119
 120 my $lock_dir = "/var/lock/qemu-server";
 121 mkdir $lock_dir;
 122
 123 my $cpu_vendor_list = {
 124     # Intel CPUs
 125     486 => 'GenuineIntel',
 126     pentium => 'GenuineIntel',
 127     pentium2  => 'GenuineIntel',
 128     pentium3  => 'GenuineIntel',
 129     coreduo => 'GenuineIntel',
 130     core2duo => 'GenuineIntel',
 131     Conroe  => 'GenuineIntel',
 132     Penryn  => 'GenuineIntel',
 133     Nehalem  => 'GenuineIntel',
 134     'Nehalem-IBRS'  => 'GenuineIntel',
 135     Westmere => 'GenuineIntel',
 136     'Westmere-IBRS' => 'GenuineIntel',
 137     SandyBridge => 'GenuineIntel',
 138     'SandyBridge-IBRS' => 'GenuineIntel',
 139     IvyBridge => 'GenuineIntel',
 140     'IvyBridge-IBRS' => 'GenuineIntel',
 141     Haswell => 'GenuineIntel',
 142     'Haswell-IBRS' => 'GenuineIntel',
 143     'Haswell-noTSX' => 'GenuineIntel',
 144     'Haswell-noTSX-IBRS' => 'GenuineIntel',
 145     Broadwell => 'GenuineIntel',
 146     'Broadwell-IBRS' => 'GenuineIntel',
 147     'Broadwell-noTSX' => 'GenuineIntel',
 148     'Broadwell-noTSX-IBRS' => 'GenuineIntel',
 149     'Skylake-Client' => 'GenuineIntel',
 150     'Skylake-Client-IBRS' => 'GenuineIntel',
 151     'Skylake-Server' => 'GenuineIntel',
 152     'Skylake-Server-IBRS' => 'GenuineIntel',
 153
 154     # AMD CPUs
 155     athlon => 'AuthenticAMD',
 156     phenom  => 'AuthenticAMD',
 157     Opteron_G1  => 'AuthenticAMD',
 158     Opteron_G2  => 'AuthenticAMD',
 159     Opteron_G3  => 'AuthenticAMD',
 160     Opteron_G4  => 'AuthenticAMD',
 161     Opteron_G5  => 'AuthenticAMD',
 162     EPYC => 'AuthenticAMD',
 163     'EPYC-IBPB' => 'AuthenticAMD',
 164
 165     # generic types, use vendor from host node
 166     host => 'default',
 167     kvm32 => 'default',
 168     kvm64 => 'default',
 169     qemu32 => 'default',
 170     qemu64 => 'default',
 171     max => 'default',
 172 };
 173
 174 my $cpu_flag = qr/[+-](pcid|spec-ctrl|ibpb|ssbd|virt-ssbd|amd-ssbd|amd-no-ssb|pdpe1gb)/;
 175
 176 my $cpu_fmt = {
 177     cputype => {
 178         description => "Emulated CPU type.",
 179         type => 'string',
 180         enum => [ sort { "\L$a" cmp "\L$b" } keys %$cpu_vendor_list ],
 181         default => 'kvm64',
 182         default_key => 1,
 183     },
 184     hidden => {
 185         description => "Do not identify as a KVM virtual machine.",
 186         type => 'boolean',
 187         optional => 1,
 188         default => 0
 189     },
 190     'hv-vendor-id' => {
 191         type => 'string',
 192         pattern => qr/[a-zA-Z0-9]{1,12}/,
 193         format_description => 'vendor-id',
 194         description => 'The Hyper-V vendor ID. Some drivers or programs inside Windows guests need a specific ID.',
 195         optional => 1,
 196     },
 197     flags => {
 198         description => "List of additional CPU flags separated by ';'."
 199                      . " Use '+FLAG' to enable, '-FLAG' to disable a flag."
 200                      . " Currently supported flags: 'pcid', 'spec-ctrl', 'ibpb', 'ssbd', 'virt-ssbd', 'amd-ssbd', 'amd-no-ssb', 'pdpe1gb'.",
 201         format_description => '+FLAG[;-FLAG...]',
 202         type => 'string',
 203         pattern => qr/$cpu_flag(;$cpu_flag)*/,
 204         optional => 1,
 205     },
 206 };
 207
 208 my $watchdog_fmt = {
 209     model => {
 210         default_key => 1,
 211         type => 'string',
 212         enum => [qw(i6300esb ib700)],
 213         description => "Watchdog type to emulate.",
 214         default => 'i6300esb',
 215         optional => 1,
 216     },
 217     action => {
 218         type => 'string',
 219         enum => [qw(reset shutdown poweroff pause debug none)],
 220         description => "The action to perform if after activation the guest fails to poll the watchdog in time.",
 221         optional => 1,
 222     },
 223 };
 224 PVE::JSONSchema::register_format('pve-qm-watchdog', $watchdog_fmt);
 225
 226 my $agent_fmt = {
 227     enabled => {
 228         description => "Enable/disable Qemu GuestAgent.",
 229         type => 'boolean',
 230         default => 0,
 231         default_key => 1,
 232     },
 233     fstrim_cloned_disks => {
 234         description => "Run fstrim after cloning/moving a disk.",
 235         type => 'boolean',
 236         optional => 1,
 237         default => 0
 238     },
 239 };
 240
 241 my $vga_fmt = {
 242     type => {
 243         description => "Select the VGA type.",
 244         type => 'string',
 245         default => 'std',
 246         optional => 1,
 247         default_key => 1,
 248         enum => [qw(cirrus qxl qxl2 qxl3 qxl4 none serial0 serial1 serial2 serial3 std virtio vmware)],
 249     },
 250     memory => {
 251         description => "Sets the VGA memory (in MiB). Has no effect with serial display.",
 252         type => 'integer',
 253         optional => 1,
 254         minimum => 4,
 255         maximum => 512,
 256     },
 257 };
 258
 259 my $ivshmem_fmt = {
 260     size => {
 261         type => 'integer',
 262         minimum => 1,
 263         description => "The size of the file in MB.",
 264     },
 265     name => {
 266         type => 'string',
 267         pattern => '[a-zA-Z0-9\-]+',
 268         optional => 1,
 269         format_description => 'string',
 270         description => "The name of the file. Will be prefixed with 'pve-shm-'. Default is the VMID. Will be deleted when the VM is stopped.",
 271     },
 272 };
 273
 274 my $confdesc = {
 275     onboot => {
 276         optional => 1,
 277         type => 'boolean',
 278         description => "Specifies whether a VM will be started during system bootup.",
 279         default => 0,
 280     },
 281     autostart => {
 282         optional => 1,
 283         type => 'boolean',
 284         description => "Automatic restart after crash (currently ignored).",
 285         default => 0,
 286     },
 287     hotplug => {
 288         optional => 1,
 289         type => 'string', format => 'pve-hotplug-features',
 290         description => "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
 291         default => 'network,disk,usb',
 292     },
 293     reboot => {
 294         optional => 1,
 295         type => 'boolean',
 296         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 297         default => 1,
 298     },
 299     lock => {
 300         optional => 1,
 301         type => 'string',
 302         description => "Lock/unlock the VM.",
 303         enum => [qw(backup clone create migrate rollback snapshot snapshot-delete)],
 304     },
 305     cpulimit => {
 306         optional => 1,
 307         type => 'number',
 308         description => "Limit of CPU usage.",
 309         verbose_description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
 310         minimum => 0,
 311         maximum => 128,
 312         default => 0,
 313     },
 314     cpuunits => {
 315         optional => 1,
 316         type => 'integer',
 317         description => "CPU weight for a VM.",
 318         verbose_description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.",
 319         minimum => 2,
 320         maximum => 262144,
 321         default => 1024,
 322     },
 323     memory => {
 324         optional => 1,
 325         type => 'integer',
 326         description => "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
 327         minimum => 16,
 328         default => 512,
 329     },
 330     balloon => {
 331         optional => 1,
 332         type => 'integer',
 333         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 334         minimum => 0,
 335     },
 336     shares => {
 337         optional => 1,
 338         type => 'integer',
 339         description => "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning. Auto-ballooning is done by pvestatd.",
 340         minimum => 0,
 341         maximum => 50000,
 342         default => 1000,
 343     },
 344     keyboard => {
 345         optional => 1,
 346         type => 'string',
 347         description => "Keybord layout for vnc server. Default is read from the '/etc/pve/datacenter.cfg' configuration file.".
 348                        "It should not be necessary to set it.",
 349         enum => PVE::Tools::kvmkeymaplist(),
 350         default => undef,
 351     },
 352     name => {
 353         optional => 1,
 354         type => 'string', format => 'dns-name',
 355         description => "Set a name for the VM. Only used on the configuration web interface.",
 356     },
 357     scsihw => {
 358         optional => 1,
 359         type => 'string',
 360         description => "SCSI controller model",
 361         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 362         default => 'lsi',
 363     },
 364     description => {
 365         optional => 1,
 366         type => 'string',
 367         description => "Description for the VM. Only used on the configuration web interface. This is saved as comment inside the configuration file.",
 368     },
 369     ostype => {
 370         optional => 1,
 371         type => 'string',
 372         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 l24 l26 solaris)],
 373         description => "Specify guest operating system.",
 374         verbose_description => <<EODESC,
 375 Specify guest operating system. This is used to enable special
 376 optimization/features for specific operating systems:
 377
 378 [horizontal]
 379 other;; unspecified OS
 380 wxp;; Microsoft Windows XP
 381 w2k;; Microsoft Windows 2000
 382 w2k3;; Microsoft Windows 2003
 383 w2k8;; Microsoft Windows 2008
 384 wvista;; Microsoft Windows Vista
 385 win7;; Microsoft Windows 7
 386 win8;; Microsoft Windows 8/2012/2012r2
 387 win10;; Microsoft Windows 10/2016
 388 l24;; Linux 2.4 Kernel
 389 l26;; Linux 2.6/3.X Kernel
 390 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
 391 EODESC
 392     },
 393     boot => {
 394         optional => 1,
 395         type => 'string',
 396         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n).",
 397         pattern => '[acdn]{1,4}',
 398         default => 'cdn',
 399     },
 400     bootdisk => {
 401         optional => 1,
 402         type => 'string', format => 'pve-qm-bootdisk',
 403         description => "Enable booting from specified disk.",
 404         pattern => '(ide|sata|scsi|virtio)\d+',
 405     },
 406     smp => {
 407         optional => 1,
 408         type => 'integer',
 409         description => "The number of CPUs. Please use option -sockets instead.",
 410         minimum => 1,
 411         default => 1,
 412     },
 413     sockets => {
 414         optional => 1,
 415         type => 'integer',
 416         description => "The number of CPU sockets.",
 417         minimum => 1,
 418         default => 1,
 419     },
 420     cores => {
 421         optional => 1,
 422         type => 'integer',
 423         description => "The number of cores per socket.",
 424         minimum => 1,
 425         default => 1,
 426     },
 427     numa => {
 428         optional => 1,
 429         type => 'boolean',
 430         description => "Enable/disable NUMA.",
 431         default => 0,
 432     },
 433     hugepages => {
 434         optional => 1,
 435         type => 'string',
 436         description => "Enable/disable hugepages memory.",
 437         enum => [qw(any 2 1024)],
 438     },
 439     vcpus => {
 440         optional => 1,
 441         type => 'integer',
 442         description => "Number of hotplugged vcpus.",
 443         minimum => 1,
 444         default => 0,
 445     },
 446     acpi => {
 447         optional => 1,
 448         type => 'boolean',
 449         description => "Enable/disable ACPI.",
 450         default => 1,
 451     },
 452     agent => {
 453         optional => 1,
 454         description => "Enable/disable Qemu GuestAgent and its properties.",
 455         type => 'string',
 456         format => $agent_fmt,
 457     },
 458     kvm => {
 459         optional => 1,
 460         type => 'boolean',
 461         description => "Enable/disable KVM hardware virtualization.",
 462         default => 1,
 463     },
 464     tdf => {
 465         optional => 1,
 466         type => 'boolean',
 467         description => "Enable/disable time drift fix.",
 468         default => 0,
 469     },
 470     localtime => {
 471         optional => 1,
 472         type => 'boolean',
 473         description => "Set the real time clock to local time. This is enabled by default if ostype indicates a Microsoft OS.",
 474     },
 475     freeze => {
 476         optional => 1,
 477         type => 'boolean',
 478         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 479     },
 480     vga => {
 481         optional => 1,
 482         type => 'string', format => $vga_fmt,
 483         description => "Configure the VGA hardware.",
 484         verbose_description => "Configure the VGA Hardware. If you want to use ".
 485             "high resolution modes (>= 1280x1024x16) you may need to increase " .
 486             "the vga memory option. Since QEMU 2.9 the default VGA display type " .
 487             "is 'std' for all OS types besides some Windows versions (XP and " .
 488             "older) which use 'cirrus'. The 'qxl' option enables the SPICE " .
 489             "display server. For win* OS you can select how many independent " .
 490             "displays you want, Linux guests can add displays them self.\n".
 491             "You can also run without any graphic card, using a serial device as terminal.",
 492     },
 493     watchdog => {
 494         optional => 1,
 495         type => 'string', format => 'pve-qm-watchdog',
 496         description => "Create a virtual hardware watchdog device.",
 497         verbose_description => "Create a virtual hardware watchdog device. Once enabled" .
 498             " (by a guest action), the watchdog must be periodically polled " .
 499             "by an agent inside the guest or else the watchdog will reset " .
 500             "the guest (or execute the respective action specified)",
 501     },
 502     startdate => {
 503         optional => 1,
 504         type => 'string',
 505         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 506         description => "Set the initial date of the real time clock. Valid format for date are: 'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 507         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 508         default => 'now',
 509     },
 510     startup =>  get_standard_option('pve-startup-order'),
 511     template => {
 512         optional => 1,
 513         type => 'boolean',
 514         description => "Enable/disable Template.",
 515         default => 0,
 516     },
 517     args => {
 518         optional => 1,
 519         type => 'string',
 520         description => "Arbitrary arguments passed to kvm.",
 521         verbose_description => <<EODESCR,
 522 Arbitrary arguments passed to kvm, for example:
 523
 524 args: -no-reboot -no-hpet
 525
 526 NOTE: this option is for experts only.
 527 EODESCR
 528     },
 529     tablet => {
 530         optional => 1,
 531         type => 'boolean',
 532         default => 1,
 533         description => "Enable/disable the USB tablet device.",
 534         verbose_description => "Enable/disable the USB tablet device. This device is " .
 535             "usually needed to allow absolute mouse positioning with VNC. " .
 536             "Else the mouse runs out of sync with normal VNC clients. " .
 537             "If you're running lots of console-only guests on one host, " .
 538             "you may consider disabling this to save some context switches. " .
 539             "This is turned off by default if you use spice (-vga=qxl).",
 540     },
 541     migrate_speed => {
 542         optional => 1,
 543         type => 'integer',
 544         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 545         minimum => 0,
 546         default => 0,
 547     },
 548     migrate_downtime => {
 549         optional => 1,
 550         type => 'number',
 551         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 552         minimum => 0,
 553         default => 0.1,
 554     },
 555     cdrom => {
 556         optional => 1,
 557         type => 'string', format => 'pve-qm-ide',
 558         typetext => '<volume>',
 559         description => "This is an alias for option -ide2",
 560     },
 561     cpu => {
 562         optional => 1,
 563         description => "Emulated CPU type.",
 564         type => 'string',
 565         format => $cpu_fmt,
 566     },
 567     parent => get_standard_option('pve-snapshot-name', {
 568         optional => 1,
 569         description => "Parent snapshot name. This is used internally, and should not be modified.",
 570     }),
 571     snaptime => {
 572         optional => 1,
 573         description => "Timestamp for snapshots.",
 574         type => 'integer',
 575         minimum => 0,
 576     },
 577     vmstate => {
 578         optional => 1,
 579         type => 'string', format => 'pve-volume-id',
 580         description => "Reference to a volume which stores the VM state. This is used internally for snapshots.",
 581     },
 582     vmstatestorage => get_standard_option('pve-storage-id', {
 583         description => "Default storage for VM state volumes/files.",
 584         optional => 1,
 585     }),
 586     runningmachine => get_standard_option('pve-qemu-machine', {
 587         description => "Specifies the Qemu machine type of the running vm. This is used internally for snapshots.",
 588     }),
 589     machine => get_standard_option('pve-qemu-machine'),
 590     arch => {
 591         description => "Virtual processor architecture. Defaults to the host.",
 592         optional => 1,
 593         type => 'string',
 594         enum => [qw(x86_64 aarch64)],
 595     },
 596     smbios1 => {
 597         description => "Specify SMBIOS type 1 fields.",
 598         type => 'string', format => 'pve-qm-smbios1',
 599         maxLength => 256,
 600         optional => 1,
 601     },
 602     protection => {
 603         optional => 1,
 604         type => 'boolean',
 605         description => "Sets the protection flag of the VM. This will disable the remove VM and remove disk operations.",
 606         default => 0,
 607     },
 608     bios => {
 609         optional => 1,
 610         type => 'string',
 611         enum => [ qw(seabios ovmf) ],
 612         description => "Select BIOS implementation.",
 613         default => 'seabios',
 614     },
 615     vmgenid => {
 616         type => 'string',
 617         pattern => '(?:[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}|[01])',
 618         format_description => 'UUID',
 619         description => "Set VM Generation ID. Use '1' to autogenerate on create or update, pass '0' to disable explicitly.",
 620         verbose_description => "The VM generation ID (vmgenid) device exposes a".
 621             " 128-bit integer value identifier to the guest OS. This allows to".
 622             " notify the guest operating system when the virtual machine is".
 623             " executed with a different configuration (e.g. snapshot execution".
 624             " or creation from a template). The guest operating system notices".
 625             " the change, and is then able to react as appropriate by marking".
 626             " its copies of distributed databases as dirty, re-initializing its".
 627             " random number generator, etc.\n".
 628             "Note that auto-creation only works when done throug API/CLI create".
 629             " or update methods, but not when manually editing the config file.",
 630         default => "1 (autogenerated)",
 631         optional => 1,
 632     },
 633     hookscript => {
 634         type => 'string',
 635         format => 'pve-volume-id',
 636         optional => 1,
 637         description => "Script that will be executed during various steps in the vms lifetime.",
 638     },
 639     ivshmem => {
 640         type => 'string',
 641         format => $ivshmem_fmt,
 642         description => "Inter-VM shared memory. Useful for direct communication between VMs, or to the host.",
 643         optional => 1,
 644     }
 645 };
 646
 647 my $confdesc_cloudinit = {
 648     citype => {
 649         optional => 1,
 650         type => 'string',
 651         description => 'Specifies the cloud-init configuration format. The default depends on the configured operating system type (`ostype`. We use the `nocloud` format for Linux, and `configdrive2` for windows.',
 652         enum => ['configdrive2', 'nocloud'],
 653     },
 654     ciuser => {
 655         optional => 1,
 656         type => 'string',
 657         description => "cloud-init: User name to change ssh keys and password for instead of the image's configured default user.",
 658     },
 659     cipassword => {
 660         optional => 1,
 661         type => 'string',
 662         description => 'cloud-init: Password to assign the user. Using this is generally not recommended. Use ssh keys instead. Also note that older cloud-init versions do not support hashed passwords.',
 663     },
 664     searchdomain => {
 665         optional => 1,
 666         type => 'string',
 667         description => "cloud-init: Sets DNS search domains for a container. Create will automatically use the setting from the host if neither searchdomain nor nameserver are set.",
 668     },
 669     nameserver => {
 670         optional => 1,
 671         type => 'string', format => 'address-list',
 672         description => "cloud-init: Sets DNS server IP address for a container. Create will automatically use the setting from the host if neither searchdomain nor nameserver are set.",
 673     },
 674     sshkeys => {
 675         optional => 1,
 676         type => 'string',
 677         format => 'urlencoded',
 678         description => "cloud-init: Setup public SSH keys (one key per line, OpenSSH format).",
 679     },
 680 };
 681
 682 # what about other qemu settings ?
 683 #cpu => 'string',
 684 #machine => 'string',
 685 #fda => 'file',
 686 #fdb => 'file',
 687 #mtdblock => 'file',
 688 #sd => 'file',
 689 #pflash => 'file',
 690 #snapshot => 'bool',
 691 #bootp => 'file',
 692 ##tftp => 'dir',
 693 ##smb => 'dir',
 694 #kernel => 'file',
 695 #append => 'string',
 696 #initrd => 'file',
 697 ##soundhw => 'string',
 698
 699 while (my ($k, $v) = each %$confdesc) {
 700     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 701 }
 702
 703 my $MAX_IDE_DISKS = 4;
 704 my $MAX_SCSI_DISKS = 14;
 705 my $MAX_VIRTIO_DISKS = 16;
 706 my $MAX_SATA_DISKS = 6;
 707 my $MAX_USB_DEVICES = 5;
 708 my $MAX_NETS = 32;
 709 my $MAX_UNUSED_DISKS = 256;
 710 my $MAX_HOSTPCI_DEVICES = 4;
 711 my $MAX_SERIAL_PORTS = 4;
 712 my $MAX_PARALLEL_PORTS = 3;
 713 my $MAX_NUMA = 8;
 714
 715 my $numa_fmt = {
 716     cpus => {
 717         type => "string",
 718         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 719         description => "CPUs accessing this NUMA node.",
 720         format_description => "id[-id];...",
 721     },
 722     memory => {
 723         type => "number",
 724         description => "Amount of memory this NUMA node provides.",
 725         optional => 1,
 726     },
 727     hostnodes => {
 728         type => "string",
 729         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 730         description => "Host NUMA nodes to use.",
 731         format_description => "id[-id];...",
 732         optional => 1,
 733     },
 734     policy => {
 735         type => 'string',
 736         enum => [qw(preferred bind interleave)],
 737         description => "NUMA allocation policy.",
 738         optional => 1,
 739     },
 740 };
 741 PVE::JSONSchema::register_format('pve-qm-numanode', $numa_fmt);
 742 my $numadesc = {
 743     optional => 1,
 744     type => 'string', format => $numa_fmt,
 745     description => "NUMA topology.",
 746 };
 747 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 748
 749 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 750     $confdesc->{"numa$i"} = $numadesc;
 751 }
 752
 753 my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000',  'pcnet',  'virtio',
 754                       'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3',
 755                       'e1000-82540em', 'e1000-82544gc', 'e1000-82545em'];
 756 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 757
 758 my $net_fmt_bridge_descr = <<__EOD__;
 759 Bridge to attach the network device to. The Proxmox VE standard bridge
 760 is called 'vmbr0'.
 761
 762 If you do not specify a bridge, we create a kvm user (NATed) network
 763 device, which provides DHCP and DNS services. The following addresses
 764 are used:
 765
 766  10.0.2.2   Gateway
 767  10.0.2.3   DNS Server
 768  10.0.2.4   SMB Server
 769
 770 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 771 __EOD__
 772
 773 my $net_fmt = {
 774     macaddr => {
 775         type => 'string',
 776         pattern => qr/[0-9a-f]{2}(?::[0-9a-f]{2}){5}/i,
 777         description => "MAC address. That address must be unique withing your network. This is automatically generated if not specified.",
 778         format_description => "XX:XX:XX:XX:XX:XX",
 779         optional => 1,
 780     },
 781     model => {
 782         type => 'string',
 783         description => "Network Card Model. The 'virtio' model provides the best performance with very low CPU overhead. If your guest does not support this driver, it is usually best to use 'e1000'.",
 784         enum => $nic_model_list,
 785         default_key => 1,
 786     },
 787     (map { $_ => { keyAlias => 'model', alias => 'macaddr' }} @$nic_model_list),
 788     bridge => {
 789         type => 'string',
 790         description => $net_fmt_bridge_descr,
 791         format_description => 'bridge',
 792         optional => 1,
 793     },
 794     queues => {
 795         type => 'integer',
 796         minimum => 0, maximum => 16,
 797         description => 'Number of packet queues to be used on the device.',
 798         optional => 1,
 799     },
 800     rate => {
 801         type => 'number',
 802         minimum => 0,
 803         description => "Rate limit in mbps (megabytes per second) as floating point number.",
 804         optional => 1,
 805     },
 806     tag => {
 807         type => 'integer',
 808         minimum => 1, maximum => 4094,
 809         description => 'VLAN tag to apply to packets on this interface.',
 810         optional => 1,
 811     },
 812     trunks => {
 813         type => 'string',
 814         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 815         description => 'VLAN trunks to pass through this interface.',
 816         format_description => 'vlanid[;vlanid...]',
 817         optional => 1,
 818     },
 819     firewall => {
 820         type => 'boolean',
 821         description => 'Whether this interface should be protected by the firewall.',
 822         optional => 1,
 823     },
 824     link_down => {
 825         type => 'boolean',
 826         description => 'Whether this interface should be disconnected (like pulling the plug).',
 827         optional => 1,
 828     },
 829 };
 830
 831 my $netdesc = {
 832     optional => 1,
 833     type => 'string', format => $net_fmt,
 834     description => "Specify network devices.",
 835 };
 836
 837 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 838
 839 my $ipconfig_fmt = {
 840     ip => {
 841         type => 'string',
 842         format => 'pve-ipv4-config',
 843         format_description => 'IPv4Format/CIDR',
 844         description => 'IPv4 address in CIDR format.',
 845         optional => 1,
 846         default => 'dhcp',
 847     },
 848     gw => {
 849         type => 'string',
 850         format => 'ipv4',
 851         format_description => 'GatewayIPv4',
 852         description => 'Default gateway for IPv4 traffic.',
 853         optional => 1,
 854         requires => 'ip',
 855     },
 856     ip6 => {
 857         type => 'string',
 858         format => 'pve-ipv6-config',
 859         format_description => 'IPv6Format/CIDR',
 860         description => 'IPv6 address in CIDR format.',
 861         optional => 1,
 862         default => 'dhcp',
 863     },
 864     gw6 => {
 865         type => 'string',
 866         format => 'ipv6',
 867         format_description => 'GatewayIPv6',
 868         description => 'Default gateway for IPv6 traffic.',
 869         optional => 1,
 870         requires => 'ip6',
 871     },
 872 };
 873 PVE::JSONSchema::register_format('pve-qm-ipconfig', $ipconfig_fmt);
 874 my $ipconfigdesc = {
 875     optional => 1,
 876     type => 'string', format => 'pve-qm-ipconfig',
 877     description => <<'EODESCR',
 878 cloud-init: Specify IP addresses and gateways for the corresponding interface.
 879
 880 IP addresses use CIDR notation, gateways are optional but need an IP of the same type specified.
 881
 882 The special string 'dhcp' can be used for IP addresses to use DHCP, in which case no explicit gateway should be provided.
 883 For IPv6 the special string 'auto' can be used to use stateless autoconfiguration.
 884
 885 If cloud-init is enabled and neither an IPv4 nor an IPv6 address is specified, it defaults to using dhcp on IPv4.
 886 EODESCR
 887 };
 888 PVE::JSONSchema::register_standard_option("pve-qm-ipconfig", $netdesc);
 889
 890 for (my $i = 0; $i < $MAX_NETS; $i++)  {
 891     $confdesc->{"net$i"} = $netdesc;
 892     $confdesc_cloudinit->{"ipconfig$i"} = $ipconfigdesc;
 893 }
 894
 895 foreach my $key (keys %$confdesc_cloudinit) {
 896     $confdesc->{$key} = $confdesc_cloudinit->{$key};
 897 }
 898
 899 PVE::JSONSchema::register_format('pve-volume-id-or-qm-path', \&verify_volume_id_or_qm_path);
 900 sub verify_volume_id_or_qm_path {
 901     my ($volid, $noerr) = @_;
 902
 903     if ($volid eq 'none' || $volid eq 'cdrom' || $volid =~ m|^/|) {
 904         return $volid;
 905     }
 906
 907     # if its neither 'none' nor 'cdrom' nor a path, check if its a volume-id
 908     $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') };
 909     if ($@) {
 910         return undef if $noerr;
 911         die $@;
 912     }
 913     return $volid;
 914 }
 915
 916 my $drivename_hash;
 917
 918 my %drivedesc_base = (
 919     volume => { alias => 'file' },
 920     file => {
 921         type => 'string',
 922         format => 'pve-volume-id-or-qm-path',
 923         default_key => 1,
 924         format_description => 'volume',
 925         description => "The drive's backing volume.",
 926     },
 927     media => {
 928         type => 'string',
 929         enum => [qw(cdrom disk)],
 930         description => "The drive's media type.",
 931         default => 'disk',
 932         optional => 1
 933     },
 934     cyls => {
 935         type => 'integer',
 936         description => "Force the drive's physical geometry to have a specific cylinder count.",
 937         optional => 1
 938     },
 939     heads => {
 940         type => 'integer',
 941         description => "Force the drive's physical geometry to have a specific head count.",
 942         optional => 1
 943     },
 944     secs => {
 945         type => 'integer',
 946         description => "Force the drive's physical geometry to have a specific sector count.",
 947         optional => 1
 948     },
 949     trans => {
 950         type => 'string',
 951         enum => [qw(none lba auto)],
 952         description => "Force disk geometry bios translation mode.",
 953         optional => 1,
 954     },
 955     snapshot => {
 956         type => 'boolean',
 957         description => "Controls qemu's snapshot mode feature."
 958             . " If activated, changes made to the disk are temporary and will"
 959             . " be discarded when the VM is shutdown.",
 960         optional => 1,
 961     },
 962     cache => {
 963         type => 'string',
 964         enum => [qw(none writethrough writeback unsafe directsync)],
 965         description => "The drive's cache mode",
 966         optional => 1,
 967     },
 968     format => get_standard_option('pve-qm-image-format'),
 969     size => {
 970         type => 'string',
 971         format => 'disk-size',
 972         format_description => 'DiskSize',
 973         description => "Disk size. This is purely informational and has no effect.",
 974         optional => 1,
 975     },
 976     backup => {
 977         type => 'boolean',
 978         description => "Whether the drive should be included when making backups.",
 979         optional => 1,
 980     },
 981     replicate => {
 982         type => 'boolean',
 983         description => 'Whether the drive should considered for replication jobs.',
 984         optional => 1,
 985         default => 1,
 986     },
 987     rerror => {
 988         type => 'string',
 989         enum => [qw(ignore report stop)],
 990         description => 'Read error action.',
 991         optional => 1,
 992     },
 993     werror => {
 994         type => 'string',
 995         enum => [qw(enospc ignore report stop)],
 996         description => 'Write error action.',
 997         optional => 1,
 998     },
 999     aio => {
1000         type => 'string',
1001         enum => [qw(native threads)],
1002         description => 'AIO type to use.',
1003         optional => 1,
1004     },
1005     discard => {
1006         type => 'string',
1007         enum => [qw(ignore on)],
1008         description => 'Controls whether to pass discard/trim requests to the underlying storage.',
1009         optional => 1,
1010     },
1011     detect_zeroes => {
1012         type => 'boolean',
1013         description => 'Controls whether to detect and try to optimize writes of zeroes.',
1014         optional => 1,
1015     },
1016     serial => {
1017         type => 'string',
1018         format => 'urlencoded',
1019         format_description => 'serial',
1020         maxLength => 20*3, # *3 since it's %xx url enoded
1021         description => "The drive's reported serial number, url-encoded, up to 20 bytes long.",
1022         optional => 1,
1023     },
1024     shared => {
1025         type => 'boolean',
1026         description => 'Mark this locally-managed volume as available on all nodes',
1027         verbose_description => "Mark this locally-managed volume as available on all nodes.\n\nWARNING: This option does not share the volume automatically, it assumes it is shared already!",
1028         optional => 1,
1029         default => 0,
1030     }
1031 );
1032
1033 my %iothread_fmt = ( iothread => {
1034         type => 'boolean',
1035         description => "Whether to use iothreads for this drive",
1036         optional => 1,
1037 });
1038
1039 my %model_fmt = (
1040     model => {
1041         type => 'string',
1042         format => 'urlencoded',
1043         format_description => 'model',
1044         maxLength => 40*3, # *3 since it's %xx url enoded
1045         description => "The drive's reported model name, url-encoded, up to 40 bytes long.",
1046         optional => 1,
1047     },
1048 );
1049
1050 my %queues_fmt = (
1051     queues => {
1052         type => 'integer',
1053         description => "Number of queues.",
1054         minimum => 2,
1055         optional => 1
1056     }
1057 );
1058
1059 my %scsiblock_fmt = (
1060     scsiblock => {
1061         type => 'boolean',
1062         description => "whether to use scsi-block for full passthrough of host block device\n\nWARNING: can lead to I/O errors in combination with low memory or high memory fragmentation on host",
1063         optional => 1,
1064         default => 0,
1065     },
1066 );
1067
1068 my %ssd_fmt = (
1069     ssd => {
1070         type => 'boolean',
1071         description => "Whether to expose this drive as an SSD, rather than a rotational hard disk.",
1072         optional => 1,
1073     },
1074 );
1075
1076 my %wwn_fmt = (
1077     wwn => {
1078         type => 'string',
1079         pattern => qr/^(0x)[0-9a-fA-F]{16}/,
1080         format_description => 'wwn',
1081         description => "The drive's worldwide name, encoded as 16 bytes hex string, prefixed by '0x'.",
1082         optional => 1,
1083     },
1084 );
1085
1086 my $add_throttle_desc = sub {
1087     my ($key, $type, $what, $unit, $longunit, $minimum) = @_;
1088     my $d = {
1089         type => $type,
1090         format_description => $unit,
1091         description => "Maximum $what in $longunit.",
1092         optional => 1,
1093     };
1094     $d->{minimum} = $minimum if defined($minimum);
1095     $drivedesc_base{$key} = $d;
1096 };
1097 # throughput: (leaky bucket)
1098 $add_throttle_desc->('bps',     'integer', 'r/w speed',   'bps',  'bytes per second');
1099 $add_throttle_desc->('bps_rd',  'integer', 'read speed',  'bps',  'bytes per second');
1100 $add_throttle_desc->('bps_wr',  'integer', 'write speed', 'bps',  'bytes per second');
1101 $add_throttle_desc->('mbps',    'number',  'r/w speed',   'mbps', 'megabytes per second');
1102 $add_throttle_desc->('mbps_rd', 'number',  'read speed',  'mbps', 'megabytes per second');
1103 $add_throttle_desc->('mbps_wr', 'number',  'write speed', 'mbps', 'megabytes per second');
1104 $add_throttle_desc->('iops',    'integer', 'r/w I/O',     'iops', 'operations per second');
1105 $add_throttle_desc->('iops_rd', 'integer', 'read I/O',    'iops', 'operations per second');
1106 $add_throttle_desc->('iops_wr', 'integer', 'write I/O',   'iops', 'operations per second');
1107
1108 # pools: (pool of IO before throttling starts taking effect)
1109 $add_throttle_desc->('mbps_max',    'number',  'unthrottled r/w pool',       'mbps', 'megabytes per second');
1110 $add_throttle_desc->('mbps_rd_max', 'number',  'unthrottled read pool',      'mbps', 'megabytes per second');
1111 $add_throttle_desc->('mbps_wr_max', 'number',  'unthrottled write pool',     'mbps', 'megabytes per second');
1112 $add_throttle_desc->('iops_max',    'integer', 'unthrottled r/w I/O pool',   'iops', 'operations per second');
1113 $add_throttle_desc->('iops_rd_max', 'integer', 'unthrottled read I/O pool',  'iops', 'operations per second');
1114 $add_throttle_desc->('iops_wr_max', 'integer', 'unthrottled write I/O pool', 'iops', 'operations per second');
1115
1116 # burst lengths
1117 $add_throttle_desc->('bps_max_length',     'integer', 'length of I/O bursts',       'seconds', 'seconds', 1);
1118 $add_throttle_desc->('bps_rd_max_length',  'integer', 'length of read I/O bursts',  'seconds', 'seconds', 1);
1119 $add_throttle_desc->('bps_wr_max_length',  'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
1120 $add_throttle_desc->('iops_max_length',    'integer', 'length of I/O bursts',       'seconds', 'seconds', 1);
1121 $add_throttle_desc->('iops_rd_max_length', 'integer', 'length of read I/O bursts',  'seconds', 'seconds', 1);
1122 $add_throttle_desc->('iops_wr_max_length', 'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
1123
1124 # legacy support
1125 $drivedesc_base{'bps_rd_length'} = { alias => 'bps_rd_max_length' };
1126 $drivedesc_base{'bps_wr_length'} = { alias => 'bps_wr_max_length' };
1127 $drivedesc_base{'iops_rd_length'} = { alias => 'iops_rd_max_length' };
1128 $drivedesc_base{'iops_wr_length'} = { alias => 'iops_wr_max_length' };
1129
1130 my $ide_fmt = {
1131     %drivedesc_base,
1132     %model_fmt,
1133     %ssd_fmt,
1134     %wwn_fmt,
1135 };
1136 PVE::JSONSchema::register_format("pve-qm-ide", $ide_fmt);
1137
1138 my $idedesc = {
1139     optional => 1,
1140     type => 'string', format => $ide_fmt,
1141     description => "Use volume as IDE hard disk or CD-ROM (n is 0 to " .($MAX_IDE_DISKS -1) . ").",
1142 };
1143 PVE::JSONSchema::register_standard_option("pve-qm-ide", $idedesc);
1144
1145 my $scsi_fmt = {
1146     %drivedesc_base,
1147     %iothread_fmt,
1148     %queues_fmt,
1149     %scsiblock_fmt,
1150     %ssd_fmt,
1151     %wwn_fmt,
1152 };
1153 my $scsidesc = {
1154     optional => 1,
1155     type => 'string', format => $scsi_fmt,
1156     description => "Use volume as SCSI hard disk or CD-ROM (n is 0 to " . ($MAX_SCSI_DISKS - 1) . ").",
1157 };
1158 PVE::JSONSchema::register_standard_option("pve-qm-scsi", $scsidesc);
1159
1160 my $sata_fmt = {
1161     %drivedesc_base,
1162     %ssd_fmt,
1163     %wwn_fmt,
1164 };
1165 my $satadesc = {
1166     optional => 1,
1167     type => 'string', format => $sata_fmt,
1168     description => "Use volume as SATA hard disk or CD-ROM (n is 0 to " . ($MAX_SATA_DISKS - 1). ").",
1169 };
1170 PVE::JSONSchema::register_standard_option("pve-qm-sata", $satadesc);
1171
1172 my $virtio_fmt = {
1173     %drivedesc_base,
1174     %iothread_fmt,
1175 };
1176 my $virtiodesc = {
1177     optional => 1,
1178     type => 'string', format => $virtio_fmt,
1179     description => "Use volume as VIRTIO hard disk (n is 0 to " . ($MAX_VIRTIO_DISKS - 1) . ").",
1180 };
1181 PVE::JSONSchema::register_standard_option("pve-qm-virtio", $virtiodesc);
1182
1183 my $alldrive_fmt = {
1184     %drivedesc_base,
1185     %iothread_fmt,
1186     %model_fmt,
1187     %queues_fmt,
1188     %scsiblock_fmt,
1189     %ssd_fmt,
1190     %wwn_fmt,
1191 };
1192
1193 my $efidisk_fmt = {
1194     volume => { alias => 'file' },
1195     file => {
1196         type => 'string',
1197         format => 'pve-volume-id-or-qm-path',
1198         default_key => 1,
1199         format_description => 'volume',
1200         description => "The drive's backing volume.",
1201     },
1202     format => get_standard_option('pve-qm-image-format'),
1203     size => {
1204         type => 'string',
1205         format => 'disk-size',
1206         format_description => 'DiskSize',
1207         description => "Disk size. This is purely informational and has no effect.",
1208         optional => 1,
1209     },
1210 };
1211
1212 my $efidisk_desc = {
1213     optional => 1,
1214     type => 'string', format => $efidisk_fmt,
1215     description => "Configure a Disk for storing EFI vars",
1216 };
1217
1218 PVE::JSONSchema::register_standard_option("pve-qm-efidisk", $efidisk_desc);
1219
1220 my $usb_fmt = {
1221     host => {
1222         default_key => 1,
1223         type => 'string', format => 'pve-qm-usb-device',
1224         format_description => 'HOSTUSBDEVICE|spice',
1225         description => <<EODESCR,
1226 The Host USB device or port or the value 'spice'. HOSTUSBDEVICE syntax is:
1227
1228  'bus-port(.port)*' (decimal numbers) or
1229  'vendor_id:product_id' (hexadeciaml numbers) or
1230  'spice'
1231
1232 You can use the 'lsusb -t' command to list existing usb devices.
1233
1234 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1235
1236 The value 'spice' can be used to add a usb redirection devices for spice.
1237 EODESCR
1238     },
1239     usb3 => {
1240         optional => 1,
1241         type => 'boolean',
1242         description => "Specifies whether if given host option is a USB3 device or port (this does currently not work reliably with spice redirection and is then ignored).",
1243         default => 0,
1244     },
1245 };
1246
1247 my $usbdesc = {
1248     optional => 1,
1249     type => 'string', format => $usb_fmt,
1250     description => "Configure an USB device (n is 0 to 4).",
1251 };
1252 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
1253
1254 my $PCIRE = qr/[a-f0-9]{2}:[a-f0-9]{2}(?:\.[a-f0-9])?/;
1255 my $hostpci_fmt = {
1256     host => {
1257         default_key => 1,
1258         type => 'string',
1259         pattern => qr/$PCIRE(;$PCIRE)*/,
1260         format_description => 'HOSTPCIID[;HOSTPCIID2...]',
1261         description => <<EODESCR,
1262 Host PCI device pass through. The PCI ID of a host's PCI device or a list
1263 of PCI virtual functions of the host. HOSTPCIID syntax is:
1264
1265 'bus:dev.func' (hexadecimal numbers)
1266
1267 You can us the 'lspci' command to list existing PCI devices.
1268 EODESCR
1269     },
1270     rombar => {
1271         type => 'boolean',
1272         description =>  "Specify whether or not the device's ROM will be visible in the guest's memory map.",
1273         optional => 1,
1274         default => 1,
1275     },
1276     romfile => {
1277         type => 'string',
1278         pattern => '[^,;]+',
1279         format_description => 'string',
1280         description => "Custom pci device rom filename (must be located in /usr/share/kvm/).",
1281         optional => 1,
1282     },
1283     pcie => {
1284         type => 'boolean',
1285         description =>  "Choose the PCI-express bus (needs the 'q35' machine model).",
1286         optional => 1,
1287         default => 0,
1288     },
1289     'x-vga' => {
1290         type => 'boolean',
1291         description =>  "Enable vfio-vga device support.",
1292         optional => 1,
1293         default => 0,
1294     },
1295     'mdev' => {
1296         type => 'string',
1297         format_description => 'string',
1298         pattern => '[^/\.:]+',
1299         optional => 1,
1300         description => <<EODESCR
1301 The type of mediated device to use.
1302 An instance of this type will be created on startup of the VM and
1303 will be cleaned up when the VM stops.
1304 EODESCR
1305     }
1306 };
1307 PVE::JSONSchema::register_format('pve-qm-hostpci', $hostpci_fmt);
1308
1309 my $hostpcidesc = {
1310         optional => 1,
1311         type => 'string', format => 'pve-qm-hostpci',
1312         description => "Map host PCI devices into guest.",
1313         verbose_description =>  <<EODESCR,
1314 Map host PCI devices into guest.
1315
1316 NOTE: This option allows direct access to host hardware. So it is no longer
1317 possible to migrate such machines - use with special care.
1318
1319 CAUTION: Experimental! User reported problems with this option.
1320 EODESCR
1321 };
1322 PVE::JSONSchema::register_standard_option("pve-qm-hostpci", $hostpcidesc);
1323
1324 my $serialdesc = {
1325         optional => 1,
1326         type => 'string',
1327         pattern => '(/dev/.+|socket)',
1328         description =>  "Create a serial device inside the VM (n is 0 to 3)",
1329         verbose_description =>  <<EODESCR,
1330 Create a serial device inside the VM (n is 0 to 3), and pass through a
1331 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1332 host side (use 'qm terminal' to open a terminal connection).
1333
1334 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines - use with special care.
1335
1336 CAUTION: Experimental! User reported problems with this option.
1337 EODESCR
1338 };
1339
1340 my $paralleldesc= {
1341         optional => 1,
1342         type => 'string',
1343         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
1344         description =>  "Map host parallel devices (n is 0 to 2).",
1345         verbose_description =>  <<EODESCR,
1346 Map host parallel devices (n is 0 to 2).
1347
1348 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1349
1350 CAUTION: Experimental! User reported problems with this option.
1351 EODESCR
1352 };
1353
1354 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
1355     $confdesc->{"parallel$i"} = $paralleldesc;
1356 }
1357
1358 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
1359     $confdesc->{"serial$i"} = $serialdesc;
1360 }
1361
1362 for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
1363     $confdesc->{"hostpci$i"} = $hostpcidesc;
1364 }
1365
1366 for (my $i = 0; $i < $MAX_IDE_DISKS; $i++)  {
1367     $drivename_hash->{"ide$i"} = 1;
1368     $confdesc->{"ide$i"} = $idedesc;
1369 }
1370
1371 for (my $i = 0; $i < $MAX_SATA_DISKS; $i++)  {
1372     $drivename_hash->{"sata$i"} = 1;
1373     $confdesc->{"sata$i"} = $satadesc;
1374 }
1375
1376 for (my $i = 0; $i < $MAX_SCSI_DISKS; $i++)  {
1377     $drivename_hash->{"scsi$i"} = 1;
1378     $confdesc->{"scsi$i"} = $scsidesc ;
1379 }
1380
1381 for (my $i = 0; $i < $MAX_VIRTIO_DISKS; $i++)  {
1382     $drivename_hash->{"virtio$i"} = 1;
1383     $confdesc->{"virtio$i"} = $virtiodesc;
1384 }
1385
1386 $drivename_hash->{efidisk0} = 1;
1387 $confdesc->{efidisk0} = $efidisk_desc;
1388
1389 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
1390     $confdesc->{"usb$i"} = $usbdesc;
1391 }
1392
1393 my $unuseddesc = {
1394     optional => 1,
1395     type => 'string', format => 'pve-volume-id',
1396     description => "Reference to unused volumes. This is used internally, and should not be modified manually.",
1397 };
1398
1399 for (my $i = 0; $i < $MAX_UNUSED_DISKS; $i++)  {
1400     $confdesc->{"unused$i"} = $unuseddesc;
1401 }
1402
1403 my $kvm_api_version = 0;
1404
1405 sub kvm_version {
1406     return $kvm_api_version if $kvm_api_version;
1407
1408     open my $fh, '<', '/dev/kvm'
1409         or return undef;
1410
1411     # 0xae00 => KVM_GET_API_VERSION
1412     $kvm_api_version = ioctl($fh, 0xae00, 0);
1413
1414     return $kvm_api_version;
1415 }
1416
1417 my $kvm_user_version;
1418
1419 sub kvm_user_version {
1420
1421     return $kvm_user_version if $kvm_user_version;
1422
1423     $kvm_user_version = 'unknown';
1424
1425     my $code = sub {
1426         my $line = shift;
1427         if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1428             $kvm_user_version = $2;
1429         }
1430     };
1431
1432     eval { run_command("kvm -version", outfunc => $code); };
1433     warn $@ if $@;
1434
1435     return $kvm_user_version;
1436
1437 }
1438
1439 sub kernel_has_vhost_net {
1440     return -c '/dev/vhost-net';
1441 }
1442
1443 sub valid_drive_names {
1444     # order is important - used to autoselect boot disk
1445     return ((map { "ide$_" } (0 .. ($MAX_IDE_DISKS - 1))),
1446             (map { "scsi$_" } (0 .. ($MAX_SCSI_DISKS - 1))),
1447             (map { "virtio$_" } (0 .. ($MAX_VIRTIO_DISKS - 1))),
1448             (map { "sata$_" } (0 .. ($MAX_SATA_DISKS - 1))),
1449             'efidisk0');
1450 }
1451
1452 sub is_valid_drivename {
1453     my $dev = shift;
1454
1455     return defined($drivename_hash->{$dev});
1456 }
1457
1458 sub option_exists {
1459     my $key = shift;
1460     return defined($confdesc->{$key});
1461 }
1462
1463 sub nic_models {
1464     return $nic_model_list;
1465 }
1466
1467 sub os_list_description {
1468
1469     return {
1470         other => 'Other',
1471         wxp => 'Windows XP',
1472         w2k => 'Windows 2000',
1473         w2k3 =>, 'Windows 2003',
1474         w2k8 => 'Windows 2008',
1475         wvista => 'Windows Vista',
1476         win7 => 'Windows 7',
1477         win8 => 'Windows 8/2012',
1478         win10 => 'Windows 10/2016',
1479         l24 => 'Linux 2.4',
1480         l26 => 'Linux 2.6',
1481     };
1482 }
1483
1484 my $cdrom_path;
1485
1486 sub get_cdrom_path {
1487
1488     return  $cdrom_path if $cdrom_path;
1489
1490     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
1491     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
1492     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
1493 }
1494
1495 sub get_iso_path {
1496     my ($storecfg, $vmid, $cdrom) = @_;
1497
1498     if ($cdrom eq 'cdrom') {
1499         return get_cdrom_path();
1500     } elsif ($cdrom eq 'none') {
1501         return '';
1502     } elsif ($cdrom =~ m|^/|) {
1503         return $cdrom;
1504     } else {
1505         return PVE::Storage::path($storecfg, $cdrom);
1506     }
1507 }
1508
1509 # try to convert old style file names to volume IDs
1510 sub filename_to_volume_id {
1511     my ($vmid, $file, $media) = @_;
1512
1513      if (!($file eq 'none' || $file eq 'cdrom' ||
1514           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1515
1516         return undef if $file =~ m|/|;
1517
1518         if ($media && $media eq 'cdrom') {
1519             $file = "local:iso/$file";
1520         } else {
1521             $file = "local:$vmid/$file";
1522         }
1523     }
1524
1525     return $file;
1526 }
1527
1528 sub verify_media_type {
1529     my ($opt, $vtype, $media) = @_;
1530
1531     return if !$media;
1532
1533     my $etype;
1534     if ($media eq 'disk') {
1535         $etype = 'images';
1536     } elsif ($media eq 'cdrom') {
1537         $etype = 'iso';
1538     } else {
1539         die "internal error";
1540     }
1541
1542     return if ($vtype eq $etype);
1543
1544     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1545 }
1546
1547 sub cleanup_drive_path {
1548     my ($opt, $storecfg, $drive) = @_;
1549
1550     # try to convert filesystem paths to volume IDs
1551
1552     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1553         ($drive->{file} !~ m|^/dev/.+|) &&
1554         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1555         ($drive->{file} !~ m/^\d+$/)) {
1556         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1557         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"}) if !$vtype;
1558         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1559         verify_media_type($opt, $vtype, $drive->{media});
1560         $drive->{file} = $volid;
1561     }
1562
1563     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1564 }
1565
1566 sub parse_hotplug_features {
1567     my ($data) = @_;
1568
1569     my $res = {};
1570
1571     return $res if $data eq '0';
1572
1573     $data = $confdesc->{hotplug}->{default} if $data eq '1';
1574
1575     foreach my $feature (PVE::Tools::split_list($data)) {
1576         if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
1577             $res->{$1} = 1;
1578         } else {
1579             die "invalid hotplug feature '$feature'\n";
1580         }
1581     }
1582     return $res;
1583 }
1584
1585 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1586 sub pve_verify_hotplug_features {
1587     my ($value, $noerr) = @_;
1588
1589     return $value if parse_hotplug_features($value);
1590
1591     return undef if $noerr;
1592
1593     die "unable to parse hotplug option\n";
1594 }
1595
1596 # ideX = [volume=]volume-id[,media=d][,cyls=c,heads=h,secs=s[,trans=t]]
1597 #        [,snapshot=on|off][,cache=on|off][,format=f][,backup=yes|no]
1598 #        [,rerror=ignore|report|stop][,werror=enospc|ignore|report|stop]
1599 #        [,aio=native|threads][,discard=ignore|on][,detect_zeroes=on|off]
1600 #        [,iothread=on][,serial=serial][,model=model]
1601
1602 sub parse_drive {
1603     my ($key, $data) = @_;
1604
1605     my ($interface, $index);
1606
1607     if ($key =~ m/^([^\d]+)(\d+)$/) {
1608         $interface = $1;
1609         $index = $2;
1610     } else {
1611         return undef;
1612     }
1613
1614     my $desc = $key =~ /^unused\d+$/ ? $alldrive_fmt
1615                                      : $confdesc->{$key}->{format};
1616     if (!$desc) {
1617         warn "invalid drive key: $key\n";
1618         return undef;
1619     }
1620     my $res = eval { PVE::JSONSchema::parse_property_string($desc, $data) };
1621     return undef if !$res;
1622     $res->{interface} = $interface;
1623     $res->{index} = $index;
1624
1625     my $error = 0;
1626     foreach my $opt (qw(bps bps_rd bps_wr)) {
1627         if (my $bps = defined(delete $res->{$opt})) {
1628             if (defined($res->{"m$opt"})) {
1629                 warn "both $opt and m$opt specified\n";
1630                 ++$error;
1631                 next;
1632             }
1633             $res->{"m$opt"} = sprintf("%.3f", $bps / (1024*1024.0));
1634         }
1635     }
1636
1637     # can't use the schema's 'requires' because of the mbps* => bps* "transforming aliases"
1638     for my $requirement (
1639         [mbps_max => 'mbps'],
1640         [mbps_rd_max => 'mbps_rd'],
1641         [mbps_wr_max => 'mbps_wr'],
1642         [miops_max => 'miops'],
1643         [miops_rd_max => 'miops_rd'],
1644         [miops_wr_max => 'miops_wr'],
1645         [bps_max_length => 'mbps_max'],
1646         [bps_rd_max_length => 'mbps_rd_max'],
1647         [bps_wr_max_length => 'mbps_wr_max'],
1648         [iops_max_length => 'iops_max'],
1649         [iops_rd_max_length => 'iops_rd_max'],
1650         [iops_wr_max_length => 'iops_wr_max']) {
1651         my ($option, $requires) = @$requirement;
1652         if ($res->{$option} && !$res->{$requires}) {
1653             warn "$option requires $requires\n";
1654             ++$error;
1655         }
1656     }
1657
1658     return undef if $error;
1659
1660     return undef if $res->{mbps_rd} && $res->{mbps};
1661     return undef if $res->{mbps_wr} && $res->{mbps};
1662     return undef if $res->{iops_rd} && $res->{iops};
1663     return undef if $res->{iops_wr} && $res->{iops};
1664
1665     if ($res->{media} && ($res->{media} eq 'cdrom')) {
1666         return undef if $res->{snapshot} || $res->{trans} || $res->{format};
1667         return undef if $res->{heads} || $res->{secs} || $res->{cyls};
1668         return undef if $res->{interface} eq 'virtio';
1669     }
1670
1671     if (my $size = $res->{size}) {
1672         return undef if !defined($res->{size} = PVE::JSONSchema::parse_size($size));
1673     }
1674
1675     return $res;
1676 }
1677
1678 sub print_drive {
1679     my ($vmid, $drive) = @_;
1680     my $data = { %$drive };
1681     delete $data->{$_} for qw(index interface);
1682     return PVE::JSONSchema::print_property_string($data, $alldrive_fmt);
1683 }
1684
1685 sub scsi_inquiry {
1686     my($fh, $noerr) = @_;
1687
1688     my $SG_IO = 0x2285;
1689     my $SG_GET_VERSION_NUM = 0x2282;
1690
1691     my $versionbuf = "\x00" x 8;
1692     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1693     if (!$ret) {
1694         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1695         return undef;
1696     }
1697     my $version = unpack("I", $versionbuf);
1698     if ($version < 30000) {
1699         die "scsi generic interface too old\n"  if !$noerr;
1700         return undef;
1701     }
1702
1703     my $buf = "\x00" x 36;
1704     my $sensebuf = "\x00" x 8;
1705     my $cmd = pack("C x3 C x1", 0x12, 36);
1706
1707     # see /usr/include/scsi/sg.h
1708     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1709
1710     my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1711                       length($sensebuf), 0, length($buf), $buf,
1712                       $cmd, $sensebuf, 6000);
1713
1714     $ret = ioctl($fh, $SG_IO, $packet);
1715     if (!$ret) {
1716         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1717         return undef;
1718     }
1719
1720     my @res = unpack($sg_io_hdr_t, $packet);
1721     if ($res[17] || $res[18]) {
1722         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1723         return undef;
1724     }
1725
1726     my $res = {};
1727     (my $byte0, my $byte1, $res->{vendor},
1728      $res->{product}, $res->{revision}) = unpack("C C x6 A8 A16 A4", $buf);
1729
1730     $res->{removable} = $byte1 & 128 ? 1 : 0;
1731     $res->{type} = $byte0 & 31;
1732
1733     return $res;
1734 }
1735
1736 sub path_is_scsi {
1737     my ($path) = @_;
1738
1739     my $fh = IO::File->new("+<$path") || return undef;
1740     my $res = scsi_inquiry($fh, 1);
1741     close($fh);
1742
1743     return $res;
1744 }
1745
1746 sub machine_type_is_q35 {
1747     my ($conf) = @_;
1748
1749     return $conf->{machine} && ($conf->{machine} =~ m/q35/) ? 1 : 0;
1750 }
1751
1752 sub print_tabletdevice_full {
1753     my ($conf, $arch) = @_;
1754
1755     my $q35 = machine_type_is_q35($conf);
1756
1757     # we use uhci for old VMs because tablet driver was buggy in older qemu
1758     my $usbbus;
1759     if (machine_type_is_q35($conf) || $arch eq 'aarch64') {
1760         $usbbus = 'ehci';
1761     } else {
1762         $usbbus = 'uhci';
1763     }
1764
1765     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1766 }
1767
1768 sub print_keyboarddevice_full {
1769     my ($conf, $arch, $machine) = @_;
1770
1771     return undef if $arch ne 'aarch64';
1772
1773     return "usb-kbd,id=keyboard,bus=ehci.0,port=2";
1774 }
1775
1776 sub print_drivedevice_full {
1777     my ($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type) = @_;
1778
1779     my $device = '';
1780     my $maxdev = 0;
1781
1782     if ($drive->{interface} eq 'virtio') {
1783         my $pciaddr = print_pci_addr("$drive->{interface}$drive->{index}", $bridges, $arch, $machine_type);
1784         $device = "virtio-blk-pci,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}$pciaddr";
1785         $device .= ",iothread=iothread-$drive->{interface}$drive->{index}" if $drive->{iothread};
1786     } elsif ($drive->{interface} eq 'scsi') {
1787
1788         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1789         my $unit = $drive->{index} % $maxdev;
1790         my $devicetype = 'hd';
1791         my $path = '';
1792         if (drive_is_cdrom($drive)) {
1793             $devicetype = 'cd';
1794         } else {
1795             if ($drive->{file} =~ m|^/|) {
1796                 $path = $drive->{file};
1797                 if (my $info = path_is_scsi($path)) {
1798                     if ($info->{type} == 0 && $drive->{scsiblock}) {
1799                         $devicetype = 'block';
1800                     } elsif ($info->{type} == 1) { # tape
1801                         $devicetype = 'generic';
1802                     }
1803                 }
1804             } else {
1805                  $path = PVE::Storage::path($storecfg, $drive->{file});
1806             }
1807
1808             if($path =~ m/^iscsi\:\/\//){
1809                 $devicetype = 'generic';
1810             }
1811         }
1812
1813         if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)){
1814             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1815         } else {
1816             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0,lun=$drive->{index},drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1817         }
1818
1819         if ($drive->{ssd} && ($devicetype eq 'block' || $devicetype eq 'hd')) {
1820             $device .= ",rotation_rate=1";
1821         }
1822         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1823
1824     } elsif ($drive->{interface} eq 'ide' || $drive->{interface} eq 'sata') {
1825         my $maxdev = ($drive->{interface} eq 'sata') ? $MAX_SATA_DISKS : 2;
1826         my $controller = int($drive->{index} / $maxdev);
1827         my $unit = $drive->{index} % $maxdev;
1828         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1829
1830         $device = "ide-$devicetype";
1831         if ($drive->{interface} eq 'ide') {
1832             $device .= ",bus=ide.$controller,unit=$unit";
1833         } else {
1834             $device .= ",bus=ahci$controller.$unit";
1835         }
1836         $device .= ",drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1837
1838         if ($devicetype eq 'hd') {
1839             if (my $model = $drive->{model}) {
1840                 $model = URI::Escape::uri_unescape($model);
1841                 $device .= ",model=$model";
1842             }
1843             if ($drive->{ssd}) {
1844                 $device .= ",rotation_rate=1";
1845             }
1846         }
1847         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1848     } elsif ($drive->{interface} eq 'usb') {
1849         die "implement me";
1850         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1851     } else {
1852         die "unsupported interface type";
1853     }
1854
1855     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1856
1857     if (my $serial = $drive->{serial}) {
1858         $serial = URI::Escape::uri_unescape($serial);
1859         $device .= ",serial=$serial";
1860     }
1861
1862
1863     return $device;
1864 }
1865
1866 sub get_initiator_name {
1867     my $initiator;
1868
1869     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return undef;
1870     while (defined(my $line = <$fh>)) {
1871         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1872         $initiator = $1;
1873         last;
1874     }
1875     $fh->close();
1876
1877     return $initiator;
1878 }
1879
1880 sub print_drive_full {
1881     my ($storecfg, $vmid, $drive) = @_;
1882
1883     my $path;
1884     my $volid = $drive->{file};
1885     my $format;
1886
1887     if (drive_is_cdrom($drive)) {
1888         $path = get_iso_path($storecfg, $vmid, $volid);
1889     } else {
1890         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1891         if ($storeid) {
1892             $path = PVE::Storage::path($storecfg, $volid);
1893             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
1894             $format = qemu_img_format($scfg, $volname);
1895         } else {
1896             $path = $volid;
1897             $format = "raw";
1898         }
1899    }
1900
1901     my $opts = '';
1902     my @qemu_drive_options = qw(heads secs cyls trans media format cache rerror werror aio discard);
1903     foreach my $o (@qemu_drive_options) {
1904         $opts .= ",$o=$drive->{$o}" if defined($drive->{$o});
1905     }
1906
1907     # snapshot only accepts on|off
1908     if (defined($drive->{snapshot})) {
1909         my $v = $drive->{snapshot} ? 'on' : 'off';
1910         $opts .= ",snapshot=$v";
1911     }
1912
1913     foreach my $type (['', '-total'], [_rd => '-read'], [_wr => '-write']) {
1914         my ($dir, $qmpname) = @$type;
1915         if (my $v = $drive->{"mbps$dir"}) {
1916             $opts .= ",throttling.bps$qmpname=".int($v*1024*1024);
1917         }
1918         if (my $v = $drive->{"mbps${dir}_max"}) {
1919             $opts .= ",throttling.bps$qmpname-max=".int($v*1024*1024);
1920         }
1921         if (my $v = $drive->{"bps${dir}_max_length"}) {
1922             $opts .= ",throttling.bps$qmpname-max-length=$v";
1923         }
1924         if (my $v = $drive->{"iops${dir}"}) {
1925             $opts .= ",throttling.iops$qmpname=$v";
1926         }
1927         if (my $v = $drive->{"iops${dir}_max"}) {
1928             $opts .= ",throttling.iops$qmpname-max=$v";
1929         }
1930         if (my $v = $drive->{"iops${dir}_max_length"}) {
1931             $opts .= ",throttling.iops$qmpname-max-length=$v";
1932         }
1933     }
1934
1935     $opts .= ",format=$format" if $format && !$drive->{format};
1936
1937     my $cache_direct = 0;
1938
1939     if (my $cache = $drive->{cache}) {
1940         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1941     } elsif (!drive_is_cdrom($drive)) {
1942         $opts .= ",cache=none";
1943         $cache_direct = 1;
1944     }
1945
1946     # aio native works only with O_DIRECT
1947     if (!$drive->{aio}) {
1948         if($cache_direct) {
1949             $opts .= ",aio=native";
1950         } else {
1951             $opts .= ",aio=threads";
1952         }
1953     }
1954
1955     if (!drive_is_cdrom($drive)) {
1956         my $detectzeroes;
1957         if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
1958             $detectzeroes = 'off';
1959         } elsif ($drive->{discard}) {
1960             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
1961         } else {
1962             # This used to be our default with discard not being specified:
1963             $detectzeroes = 'on';
1964         }
1965         $opts .= ",detect-zeroes=$detectzeroes" if $detectzeroes;
1966     }
1967
1968     my $pathinfo = $path ? "file=$path," : '';
1969
1970     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1971 }
1972
1973 sub print_netdevice_full {
1974     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files, $arch, $machine_type) = @_;
1975
1976     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
1977
1978     my $device = $net->{model};
1979     if ($net->{model} eq 'virtio') {
1980          $device = 'virtio-net-pci';
1981      };
1982
1983     my $pciaddr = print_pci_addr("$netid", $bridges, $arch, $machine_type);
1984     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1985     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
1986         #Consider we have N queues, the number of vectors needed is 2*N + 2 (plus one config interrupt and control vq)
1987         my $vectors = $net->{queues} * 2 + 2;
1988         $tmpstr .= ",vectors=$vectors,mq=on";
1989     }
1990     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
1991
1992     if ($use_old_bios_files) {
1993         my $romfile;
1994         if ($device eq 'virtio-net-pci') {
1995             $romfile = 'pxe-virtio.rom';
1996         } elsif ($device eq 'e1000') {
1997             $romfile = 'pxe-e1000.rom';
1998         } elsif ($device eq 'ne2k') {
1999             $romfile = 'pxe-ne2k_pci.rom';
2000         } elsif ($device eq 'pcnet') {
2001             $romfile = 'pxe-pcnet.rom';
2002         } elsif ($device eq 'rtl8139') {
2003             $romfile = 'pxe-rtl8139.rom';
2004         }
2005         $tmpstr .= ",romfile=$romfile" if $romfile;
2006     }
2007
2008     return $tmpstr;
2009 }
2010
2011 sub print_netdev_full {
2012     my ($vmid, $conf, $arch, $net, $netid, $hotplug) = @_;
2013
2014     my $i = '';
2015     if ($netid =~ m/^net(\d+)$/) {
2016         $i = int($1);
2017     }
2018
2019     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
2020
2021     my $ifname = "tap${vmid}i$i";
2022
2023     # kvm uses TUNSETIFF ioctl, and that limits ifname length
2024     die "interface name '$ifname' is too long (max 15 character)\n"
2025         if length($ifname) >= 16;
2026
2027     my $vhostparam = '';
2028     if (is_native($arch)) {
2029         $vhostparam = ',vhost=on' if kernel_has_vhost_net() && $net->{model} eq 'virtio';
2030     }
2031
2032     my $vmname = $conf->{name} || "vm$vmid";
2033
2034     my $netdev = "";
2035     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
2036
2037     if ($net->{bridge}) {
2038         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script,downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
2039     } else {
2040         $netdev = "type=user,id=$netid,hostname=$vmname";
2041     }
2042
2043     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
2044
2045     return $netdev;
2046 }
2047
2048
2049 sub print_cpu_device {
2050     my ($conf, $id) = @_;
2051
2052     my $kvm = $conf->{kvm} // 1;
2053     my $cpu = $kvm ? "kvm64" : "qemu64";
2054     if (my $cputype = $conf->{cpu}) {
2055         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
2056             or die "Cannot parse cpu description: $cputype\n";
2057         $cpu = $cpuconf->{cputype};
2058     }
2059
2060     my $cores = $conf->{cores} || 1;
2061
2062     my $current_core = ($id - 1) % $cores;
2063     my $current_socket = int(($id - 1 - $current_core)/$cores);
2064
2065     return "$cpu-x86_64-cpu,id=cpu$id,socket-id=$current_socket,core-id=$current_core,thread-id=0";
2066 }
2067
2068 my $vga_map = {
2069     'cirrus' => 'cirrus-vga',
2070     'std' => 'VGA',
2071     'vmware' => 'vmware-svga',
2072     'virtio' => 'virtio-vga',
2073 };
2074
2075 sub print_vga_device {
2076     my ($conf, $vga, $arch, $machine, $id, $qxlnum, $bridges) = @_;
2077
2078     my $type = $vga_map->{$vga->{type}};
2079     if ($arch eq 'aarch64' && defined($type) && $type eq 'virtio-vga') {
2080         $type = 'virtio-gpu';
2081     }
2082     my $vgamem_mb = $vga->{memory};
2083     if ($qxlnum) {
2084         $type = $id ? 'qxl' : 'qxl-vga';
2085     }
2086     die "no devicetype for $vga->{type}\n" if !$type;
2087
2088     my $memory = "";
2089     if ($vgamem_mb) {
2090         if ($vga->{type} eq 'virtio') {
2091             my $bytes = PVE::Tools::convert_size($vgamem_mb, "mb" => "b");
2092             $memory = ",max_hostmem=$bytes";
2093         } elsif ($qxlnum) {
2094             # from https://www.spice-space.org/multiple-monitors.html
2095             $memory = ",vgamem_mb=$vga->{memory}";
2096             my $ram = $vgamem_mb * 4;
2097             my $vram = $vgamem_mb * 2;
2098             $memory .= ",ram_size_mb=$ram,vram_size_mb=$vram";
2099         } else {
2100             $memory = ",vgamem_mb=$vga->{memory}";
2101         }
2102     } elsif ($qxlnum && $id) {
2103         $memory = ",ram_size=67108864,vram_size=33554432";
2104     }
2105
2106     my $q35 = machine_type_is_q35($conf);
2107     my $vgaid = "vga" . ($id // '');
2108     my $pciaddr;
2109
2110     if ($q35 && $vgaid eq 'vga') {
2111         # the first display uses pcie.0 bus on q35 machines
2112         $pciaddr = print_pcie_addr($vgaid, $bridges, $arch, $machine);
2113     } else {
2114         $pciaddr = print_pci_addr($vgaid, $bridges, $arch, $machine);
2115     }
2116
2117     return "$type,id=${vgaid}${memory}${pciaddr}";
2118 }
2119
2120 sub drive_is_cloudinit {
2121     my ($drive) = @_;
2122     return $drive->{file} =~ m@[:/]vm-\d+-cloudinit(?:\.$QEMU_FORMAT_RE)?$@;
2123 }
2124
2125 sub drive_is_cdrom {
2126     my ($drive, $exclude_cloudinit) = @_;
2127
2128     return 0 if $exclude_cloudinit && drive_is_cloudinit($drive);
2129
2130     return $drive && $drive->{media} && ($drive->{media} eq 'cdrom');
2131
2132 }
2133
2134 sub parse_number_sets {
2135     my ($set) = @_;
2136     my $res = [];
2137     foreach my $part (split(/;/, $set)) {
2138         if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
2139             die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
2140             push @$res, [ $1, $2 ];
2141         } else {
2142             die "invalid range: $part\n";
2143         }
2144     }
2145     return $res;
2146 }
2147
2148 sub parse_numa {
2149     my ($data) = @_;
2150
2151     my $res = PVE::JSONSchema::parse_property_string($numa_fmt, $data);
2152     $res->{cpus} = parse_number_sets($res->{cpus}) if defined($res->{cpus});
2153     $res->{hostnodes} = parse_number_sets($res->{hostnodes}) if defined($res->{hostnodes});
2154     return $res;
2155 }
2156
2157 sub parse_hostpci {
2158     my ($value) = @_;
2159
2160     return undef if !$value;
2161
2162     my $res = PVE::JSONSchema::parse_property_string($hostpci_fmt, $value);
2163
2164     my @idlist = split(/;/, $res->{host});
2165     delete $res->{host};
2166     foreach my $id (@idlist) {
2167         if ($id =~ m/\./) { # full id 00:00.1
2168             push @{$res->{pciid}}, {
2169                 id => $id,
2170             };
2171         } else { # partial id 00:00
2172             $res->{pciid} = PVE::SysFSTools::lspci($id);
2173         }
2174     }
2175     return $res;
2176 }
2177
2178 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
2179 sub parse_net {
2180     my ($data) = @_;
2181
2182     my $res = eval { PVE::JSONSchema::parse_property_string($net_fmt, $data) };
2183     if ($@) {
2184         warn $@;
2185         return undef;
2186     }
2187     if (!defined($res->{macaddr})) {
2188         my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
2189         $res->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix});
2190     }
2191     return $res;
2192 }
2193
2194 # ipconfigX ip=cidr,gw=ip,ip6=cidr,gw6=ip
2195 sub parse_ipconfig {
2196     my ($data) = @_;
2197
2198     my $res = eval { PVE::JSONSchema::parse_property_string($ipconfig_fmt, $data) };
2199     if ($@) {
2200         warn $@;
2201         return undef;
2202     }
2203
2204     if ($res->{gw} && !$res->{ip}) {
2205         warn 'gateway specified without specifying an IP address';
2206         return undef;
2207     }
2208     if ($res->{gw6} && !$res->{ip6}) {
2209         warn 'IPv6 gateway specified without specifying an IPv6 address';
2210         return undef;
2211     }
2212     if ($res->{gw} && $res->{ip} eq 'dhcp') {
2213         warn 'gateway specified together with DHCP';
2214         return undef;
2215     }
2216     if ($res->{gw6} && $res->{ip6} !~ /^$IPV6RE/) {
2217         # gw6 + auto/dhcp
2218         warn "IPv6 gateway specified together with $res->{ip6} address";
2219         return undef;
2220     }
2221
2222     if (!$res->{ip} && !$res->{ip6}) {
2223         return { ip => 'dhcp', ip6 => 'dhcp' };
2224     }
2225
2226     return $res;
2227 }
2228
2229 sub print_net {
2230     my $net = shift;
2231
2232     return PVE::JSONSchema::print_property_string($net, $net_fmt);
2233 }
2234
2235 sub add_random_macs {
2236     my ($settings) = @_;
2237
2238     foreach my $opt (keys %$settings) {
2239         next if $opt !~ m/^net(\d+)$/;
2240         my $net = parse_net($settings->{$opt});
2241         next if !$net;
2242         $settings->{$opt} = print_net($net);
2243     }
2244 }
2245
2246 sub vm_is_volid_owner {
2247     my ($storecfg, $vmid, $volid) = @_;
2248
2249     if ($volid !~  m|^/|) {
2250         my ($path, $owner);
2251         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
2252         if ($owner && ($owner == $vmid)) {
2253             return 1;
2254         }
2255     }
2256
2257     return undef;
2258 }
2259
2260 sub split_flagged_list {
2261     my $text = shift || '';
2262     $text =~ s/[,;]/ /g;
2263     $text =~ s/^\s+//;
2264     return { map { /^(!?)(.*)$/ && ($2, $1) } ($text =~ /\S+/g) };
2265 }
2266
2267 sub join_flagged_list {
2268     my ($how, $lst) = @_;
2269     join $how, map { $lst->{$_} . $_ } keys %$lst;
2270 }
2271
2272 sub vmconfig_delete_pending_option {
2273     my ($conf, $key, $force) = @_;
2274
2275     delete $conf->{pending}->{$key};
2276     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
2277     $pending_delete_hash->{$key} = $force ? '!' : '';
2278     $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
2279 }
2280
2281 sub vmconfig_undelete_pending_option {
2282     my ($conf, $key) = @_;
2283
2284     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
2285     delete $pending_delete_hash->{$key};
2286
2287     if (%$pending_delete_hash) {
2288         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
2289     } else {
2290         delete $conf->{pending}->{delete};
2291     }
2292 }
2293
2294 sub vmconfig_register_unused_drive {
2295     my ($storecfg, $vmid, $conf, $drive) = @_;
2296
2297     if (drive_is_cloudinit($drive)) {
2298         eval { PVE::Storage::vdisk_free($storecfg, $drive->{file}) };
2299         warn $@ if $@;
2300     } elsif (!drive_is_cdrom($drive)) {
2301         my $volid = $drive->{file};
2302         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
2303             PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
2304         }
2305     }
2306 }
2307
2308 sub vmconfig_cleanup_pending {
2309     my ($conf) = @_;
2310
2311     # remove pending changes when nothing changed
2312     my $changes;
2313     foreach my $opt (keys %{$conf->{pending}}) {
2314         if (defined($conf->{$opt}) && ($conf->{pending}->{$opt} eq  $conf->{$opt})) {
2315             $changes = 1;
2316             delete $conf->{pending}->{$opt};
2317         }
2318     }
2319
2320     my $current_delete_hash = split_flagged_list($conf->{pending}->{delete});
2321     my $pending_delete_hash = {};
2322     while (my ($opt, $force) = each %$current_delete_hash) {
2323         if (defined($conf->{$opt})) {
2324             $pending_delete_hash->{$opt} = $force;
2325         } else {
2326             $changes = 1;
2327         }
2328     }
2329
2330     if (%$pending_delete_hash) {
2331         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
2332     } else {
2333         delete $conf->{pending}->{delete};
2334     }
2335
2336     return $changes;
2337 }
2338
2339 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str]
2340 my $smbios1_fmt = {
2341     uuid => {
2342         type => 'string',
2343         pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
2344         format_description => 'UUID',
2345         description => "Set SMBIOS1 UUID.",
2346         optional => 1,
2347     },
2348     version => {
2349         type => 'string',
2350         pattern => '\S+',
2351         format_description => 'string',
2352         description => "Set SMBIOS1 version.",
2353         optional => 1,
2354     },
2355     serial => {
2356         type => 'string',
2357         pattern => '\S+',
2358         format_description => 'string',
2359         description => "Set SMBIOS1 serial number.",
2360         optional => 1,
2361     },
2362     manufacturer => {
2363         type => 'string',
2364         pattern => '\S+',
2365         format_description => 'string',
2366         description => "Set SMBIOS1 manufacturer.",
2367         optional => 1,
2368     },
2369     product => {
2370         type => 'string',
2371         pattern => '\S+',
2372         format_description => 'string',
2373         description => "Set SMBIOS1 product ID.",
2374         optional => 1,
2375     },
2376     sku => {
2377         type => 'string',
2378         pattern => '\S+',
2379         format_description => 'string',
2380         description => "Set SMBIOS1 SKU string.",
2381         optional => 1,
2382     },
2383     family => {
2384         type => 'string',
2385         pattern => '\S+',
2386         format_description => 'string',
2387         description => "Set SMBIOS1 family string.",
2388         optional => 1,
2389     },
2390 };
2391
2392 sub parse_smbios1 {
2393     my ($data) = @_;
2394
2395     my $res = eval { PVE::JSONSchema::parse_property_string($smbios1_fmt, $data) };
2396     warn $@ if $@;
2397     return $res;
2398 }
2399
2400 sub print_smbios1 {
2401     my ($smbios1) = @_;
2402     return PVE::JSONSchema::print_property_string($smbios1, $smbios1_fmt);
2403 }
2404
2405 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_fmt);
2406
2407 PVE::JSONSchema::register_format('pve-qm-bootdisk', \&verify_bootdisk);
2408 sub verify_bootdisk {
2409     my ($value, $noerr) = @_;
2410
2411     return $value if is_valid_drivename($value);
2412
2413     return undef if $noerr;
2414
2415     die "invalid boot disk '$value'\n";
2416 }
2417
2418 sub parse_watchdog {
2419     my ($value) = @_;
2420
2421     return undef if !$value;
2422
2423     my $res = eval { PVE::JSONSchema::parse_property_string($watchdog_fmt, $value) };
2424     warn $@ if $@;
2425     return $res;
2426 }
2427
2428 sub parse_guest_agent {
2429     my ($value) = @_;
2430
2431     return {} if !defined($value->{agent});
2432
2433     my $res = eval { PVE::JSONSchema::parse_property_string($agent_fmt, $value->{agent}) };
2434     warn $@ if $@;
2435
2436     # if the agent is disabled ignore the other potentially set properties
2437     return {} if !$res->{enabled};
2438     return $res;
2439 }
2440
2441 sub parse_vga {
2442     my ($value) = @_;
2443
2444     return {} if !$value;
2445     my $res = eval { PVE::JSONSchema::parse_property_string($vga_fmt, $value) };
2446     warn $@ if $@;
2447     return $res;
2448 }
2449
2450 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
2451 sub verify_usb_device {
2452     my ($value, $noerr) = @_;
2453
2454     return $value if parse_usb_device($value);
2455
2456     return undef if $noerr;
2457
2458     die "unable to parse usb device\n";
2459 }
2460
2461 # add JSON properties for create and set function
2462 sub json_config_properties {
2463     my $prop = shift;
2464
2465     foreach my $opt (keys %$confdesc) {
2466         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate' || $opt eq 'runningmachine';
2467         $prop->{$opt} = $confdesc->{$opt};
2468     }
2469
2470     return $prop;
2471 }
2472
2473 # return copy of $confdesc_cloudinit to generate documentation
2474 sub cloudinit_config_properties {
2475
2476     return dclone($confdesc_cloudinit);
2477 }
2478
2479 sub check_type {
2480     my ($key, $value) = @_;
2481
2482     die "unknown setting '$key'\n" if !$confdesc->{$key};
2483
2484     my $type = $confdesc->{$key}->{type};
2485
2486     if (!defined($value)) {
2487         die "got undefined value\n";
2488     }
2489
2490     if ($value =~ m/[\n\r]/) {
2491         die "property contains a line feed\n";
2492     }
2493
2494     if ($type eq 'boolean') {
2495         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
2496         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
2497         die "type check ('boolean') failed - got '$value'\n";
2498     } elsif ($type eq 'integer') {
2499         return int($1) if $value =~ m/^(\d+)$/;
2500         die "type check ('integer') failed - got '$value'\n";
2501     } elsif ($type eq 'number') {
2502         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
2503         die "type check ('number') failed - got '$value'\n";
2504     } elsif ($type eq 'string') {
2505         if (my $fmt = $confdesc->{$key}->{format}) {
2506             PVE::JSONSchema::check_format($fmt, $value);
2507             return $value;
2508         }
2509         $value =~ s/^\"(.*)\"$/$1/;
2510         return $value;
2511     } else {
2512         die "internal error"
2513     }
2514 }
2515
2516 sub touch_config {
2517     my ($vmid) = @_;
2518
2519     my $conf = PVE::QemuConfig->config_file($vmid);
2520     utime undef, undef, $conf;
2521 }
2522
2523 sub destroy_vm {
2524     my ($storecfg, $vmid, $keep_empty_config, $skiplock) = @_;
2525
2526     my $conffile = PVE::QemuConfig->config_file($vmid);
2527
2528     my $conf = PVE::QemuConfig->load_config($vmid);
2529
2530     PVE::QemuConfig->check_lock($conf) if !$skiplock;
2531
2532     if ($conf->{template}) {
2533         # check if any base image is still used by a linked clone
2534         foreach_drive($conf, sub {
2535                 my ($ds, $drive) = @_;
2536
2537                 return if drive_is_cdrom($drive);
2538
2539                 my $volid = $drive->{file};
2540
2541                 return if !$volid || $volid =~ m|^/|;
2542
2543                 die "base volume '$volid' is still in use by linked cloned\n"
2544                     if PVE::Storage::volume_is_base_and_used($storecfg, $volid);
2545
2546         });
2547     }
2548
2549     # only remove disks owned by this VM
2550     foreach_drive($conf, sub {
2551         my ($ds, $drive) = @_;
2552
2553         return if drive_is_cdrom($drive, 1);
2554
2555         my $volid = $drive->{file};
2556
2557         return if !$volid || $volid =~ m|^/|;
2558
2559         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
2560         return if !$path || !$owner || ($owner != $vmid);
2561
2562         eval {
2563             PVE::Storage::vdisk_free($storecfg, $volid);
2564         };
2565         warn "Could not remove disk '$volid', check manually: $@" if $@;
2566
2567     });
2568
2569     if ($keep_empty_config) {
2570         PVE::Tools::file_set_contents($conffile, "memory: 128\n");
2571     } else {
2572         unlink $conffile;
2573     }
2574
2575     # also remove unused disk
2576     eval {
2577         my $dl = PVE::Storage::vdisk_list($storecfg, undef, $vmid);
2578
2579         eval {
2580             PVE::Storage::foreach_volid($dl, sub {
2581                 my ($volid, $sid, $volname, $d) = @_;
2582                 PVE::Storage::vdisk_free($storecfg, $volid);
2583             });
2584         };
2585         warn $@ if $@;
2586
2587     };
2588     warn $@ if $@;
2589 }
2590
2591 sub parse_vm_config {
2592     my ($filename, $raw) = @_;
2593
2594     return undef if !defined($raw);
2595
2596     my $res = {
2597         digest => Digest::SHA::sha1_hex($raw),
2598         snapshots => {},
2599         pending => {},
2600     };
2601
2602     $filename =~ m|/qemu-server/(\d+)\.conf$|
2603         || die "got strange filename '$filename'";
2604
2605     my $vmid = $1;
2606
2607     my $conf = $res;
2608     my $descr;
2609     my $section = '';
2610
2611     my @lines = split(/\n/, $raw);
2612     foreach my $line (@lines) {
2613         next if $line =~ m/^\s*$/;
2614
2615         if ($line =~ m/^\[PENDING\]\s*$/i) {
2616             $section = 'pending';
2617             if (defined($descr)) {
2618                 $descr =~ s/\s+$//;
2619                 $conf->{description} = $descr;
2620             }
2621             $descr = undef;
2622             $conf = $res->{$section} = {};
2623             next;
2624
2625         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2626             $section = $1;
2627             if (defined($descr)) {
2628                 $descr =~ s/\s+$//;
2629                 $conf->{description} = $descr;
2630             }
2631             $descr = undef;
2632             $conf = $res->{snapshots}->{$section} = {};
2633             next;
2634         }
2635
2636         if ($line =~ m/^\#(.*)\s*$/) {
2637             $descr = '' if !defined($descr);
2638             $descr .= PVE::Tools::decode_text($1) . "\n";
2639             next;
2640         }
2641
2642         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2643             $descr = '' if !defined($descr);
2644             $descr .= PVE::Tools::decode_text($2);
2645         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2646             $conf->{snapstate} = $1;
2647         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2648             my $key = $1;
2649             my $value = $2;
2650             $conf->{$key} = $value;
2651         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2652             my $value = $1;
2653             if ($section eq 'pending') {
2654                 $conf->{delete} = $value; # we parse this later
2655             } else {
2656                 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
2657             }
2658         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(.+?)\s*$/) {
2659             my $key = $1;
2660             my $value = $2;
2661             eval { $value = check_type($key, $value); };
2662             if ($@) {
2663                 warn "vm $vmid - unable to parse value of '$key' - $@";
2664             } else {
2665                 $key = 'ide2' if $key eq 'cdrom';
2666                 my $fmt = $confdesc->{$key}->{format};
2667                 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2668                     my $v = parse_drive($key, $value);
2669                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2670                         $v->{file} = $volid;
2671                         $value = print_drive($vmid, $v);
2672                     } else {
2673                         warn "vm $vmid - unable to parse value of '$key'\n";
2674                         next;
2675                     }
2676                 }
2677
2678                 $conf->{$key} = $value;
2679             }
2680         }
2681     }
2682
2683     if (defined($descr)) {
2684         $descr =~ s/\s+$//;
2685         $conf->{description} = $descr;
2686     }
2687     delete $res->{snapstate}; # just to be sure
2688
2689     return $res;
2690 }
2691
2692 sub write_vm_config {
2693     my ($filename, $conf) = @_;
2694
2695     delete $conf->{snapstate}; # just to be sure
2696
2697     if ($conf->{cdrom}) {
2698         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2699         $conf->{ide2} = $conf->{cdrom};
2700         delete $conf->{cdrom};
2701     }
2702
2703     # we do not use 'smp' any longer
2704     if ($conf->{sockets}) {
2705         delete $conf->{smp};
2706     } elsif ($conf->{smp}) {
2707         $conf->{sockets} = $conf->{smp};
2708         delete $conf->{cores};
2709         delete $conf->{smp};
2710     }
2711
2712     my $used_volids = {};
2713
2714     my $cleanup_config = sub {
2715         my ($cref, $pending, $snapname) = @_;
2716
2717         foreach my $key (keys %$cref) {
2718             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2719                 $key eq 'snapstate' || $key eq 'pending';
2720             my $value = $cref->{$key};
2721             if ($key eq 'delete') {
2722                 die "propertry 'delete' is only allowed in [PENDING]\n"
2723                     if !$pending;
2724                 # fixme: check syntax?
2725                 next;
2726             }
2727             eval { $value = check_type($key, $value); };
2728             die "unable to parse value of '$key' - $@" if $@;
2729
2730             $cref->{$key} = $value;
2731
2732             if (!$snapname && is_valid_drivename($key)) {
2733                 my $drive = parse_drive($key, $value);
2734                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2735             }
2736         }
2737     };
2738
2739     &$cleanup_config($conf);
2740
2741     &$cleanup_config($conf->{pending}, 1);
2742
2743     foreach my $snapname (keys %{$conf->{snapshots}}) {
2744         die "internal error" if $snapname eq 'pending';
2745         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2746     }
2747
2748     # remove 'unusedX' settings if we re-add a volume
2749     foreach my $key (keys %$conf) {
2750         my $value = $conf->{$key};
2751         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2752             delete $conf->{$key};
2753         }
2754     }
2755
2756     my $generate_raw_config = sub {
2757         my ($conf, $pending) = @_;
2758
2759         my $raw = '';
2760
2761         # add description as comment to top of file
2762         if (defined(my $descr = $conf->{description})) {
2763             if ($descr) {
2764                 foreach my $cl (split(/\n/, $descr)) {
2765                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2766                 }
2767             } else {
2768                 $raw .= "#\n" if $pending;
2769             }
2770         }
2771
2772         foreach my $key (sort keys %$conf) {
2773             next if $key eq 'digest' || $key eq 'description' || $key eq 'pending' || $key eq 'snapshots';
2774             $raw .= "$key: $conf->{$key}\n";
2775         }
2776         return $raw;
2777     };
2778
2779     my $raw = &$generate_raw_config($conf);
2780
2781     if (scalar(keys %{$conf->{pending}})){
2782         $raw .= "\n[PENDING]\n";
2783         $raw .= &$generate_raw_config($conf->{pending}, 1);
2784     }
2785
2786     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2787         $raw .= "\n[$snapname]\n";
2788         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2789     }
2790
2791     return $raw;
2792 }
2793
2794 sub load_defaults {
2795
2796     my $res = {};
2797
2798     # we use static defaults from our JSON schema configuration
2799     foreach my $key (keys %$confdesc) {
2800         if (defined(my $default = $confdesc->{$key}->{default})) {
2801             $res->{$key} = $default;
2802         }
2803     }
2804
2805     return $res;
2806 }
2807
2808 sub config_list {
2809     my $vmlist = PVE::Cluster::get_vmlist();
2810     my $res = {};
2811     return $res if !$vmlist || !$vmlist->{ids};
2812     my $ids = $vmlist->{ids};
2813
2814     foreach my $vmid (keys %$ids) {
2815         my $d = $ids->{$vmid};
2816         next if !$d->{node} || $d->{node} ne $nodename;
2817         next if !$d->{type} || $d->{type} ne 'qemu';
2818         $res->{$vmid}->{exists} = 1;
2819     }
2820     return $res;
2821 }
2822
2823 # test if VM uses local resources (to prevent migration)
2824 sub check_local_resources {
2825     my ($conf, $noerr) = @_;
2826
2827     my $loc_res = 0;
2828
2829     $loc_res = 1 if $conf->{hostusb}; # old syntax
2830     $loc_res = 1 if $conf->{hostpci}; # old syntax
2831
2832     $loc_res = 1 if $conf->{ivshmem};
2833
2834     foreach my $k (keys %$conf) {
2835         next if $k =~ m/^usb/ && ($conf->{$k} eq 'spice');
2836         # sockets are safe: they will recreated be on the target side post-migrate
2837         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2838         $loc_res = 1 if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2839     }
2840
2841     die "VM uses local resources\n" if $loc_res && !$noerr;
2842
2843     return $loc_res;
2844 }
2845
2846 # check if used storages are available on all nodes (use by migrate)
2847 sub check_storage_availability {
2848     my ($storecfg, $conf, $node) = @_;
2849
2850     foreach_drive($conf, sub {
2851         my ($ds, $drive) = @_;
2852
2853         my $volid = $drive->{file};
2854         return if !$volid;
2855
2856         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2857         return if !$sid;
2858
2859         # check if storage is available on both nodes
2860         my $scfg = PVE::Storage::storage_check_node($storecfg, $sid);
2861         PVE::Storage::storage_check_node($storecfg, $sid, $node);
2862    });
2863 }
2864
2865 # list nodes where all VM images are available (used by has_feature API)
2866 sub shared_nodes {
2867     my ($conf, $storecfg) = @_;
2868
2869     my $nodelist = PVE::Cluster::get_nodelist();
2870     my $nodehash = { map { $_ => 1 } @$nodelist };
2871     my $nodename = PVE::INotify::nodename();
2872
2873     foreach_drive($conf, sub {
2874         my ($ds, $drive) = @_;
2875
2876         my $volid = $drive->{file};
2877         return if !$volid;
2878
2879         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2880         if ($storeid) {
2881             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2882             if ($scfg->{disable}) {
2883                 $nodehash = {};
2884             } elsif (my $avail = $scfg->{nodes}) {
2885                 foreach my $node (keys %$nodehash) {
2886                     delete $nodehash->{$node} if !$avail->{$node};
2887                 }
2888             } elsif (!$scfg->{shared}) {
2889                 foreach my $node (keys %$nodehash) {
2890                     delete $nodehash->{$node} if $node ne $nodename
2891                 }
2892             }
2893         }
2894     });
2895
2896     return $nodehash
2897 }
2898
2899 sub check_cmdline {
2900     my ($pidfile, $pid) = @_;
2901
2902     my $fh = IO::File->new("/proc/$pid/cmdline", "r");
2903     if (defined($fh)) {
2904         my $line = <$fh>;
2905         $fh->close;
2906         return undef if !$line;
2907         my @param = split(/\0/, $line);
2908
2909         my $cmd = $param[0];
2910         return if !$cmd || ($cmd !~ m|kvm$| && $cmd !~ m@(?:^|/)qemu-system-[^/]+$@);
2911
2912         for (my $i = 0; $i < scalar (@param); $i++) {
2913             my $p = $param[$i];
2914             next if !$p;
2915             if (($p eq '-pidfile') || ($p eq '--pidfile')) {
2916                 my $p = $param[$i+1];
2917                 return 1 if $p && ($p eq $pidfile);
2918                 return undef;
2919             }
2920         }
2921     }
2922     return undef;
2923 }
2924
2925 sub check_running {
2926     my ($vmid, $nocheck, $node) = @_;
2927
2928     my $filename = PVE::QemuConfig->config_file($vmid, $node);
2929
2930     die "unable to find configuration file for VM $vmid - no such machine\n"
2931         if !$nocheck && ! -f $filename;
2932
2933     my $pidfile = pidfile_name($vmid);
2934
2935     if (my $fd = IO::File->new("<$pidfile")) {
2936         my $st = stat($fd);
2937         my $line = <$fd>;
2938         close($fd);
2939
2940         my $mtime = $st->mtime;
2941         if ($mtime > time()) {
2942             warn "file '$filename' modified in future\n";
2943         }
2944
2945         if ($line =~ m/^(\d+)$/) {
2946             my $pid = $1;
2947             if (check_cmdline($pidfile, $pid)) {
2948                 if (my $pinfo = PVE::ProcFSTools::check_process_running($pid)) {
2949                     return $pid;
2950                 }
2951             }
2952         }
2953     }
2954
2955     return undef;
2956 }
2957
2958 sub vzlist {
2959
2960     my $vzlist = config_list();
2961
2962     my $fd = IO::Dir->new($var_run_tmpdir) || return $vzlist;
2963
2964     while (defined(my $de = $fd->read)) {
2965         next if $de !~ m/^(\d+)\.pid$/;
2966         my $vmid = $1;
2967         next if !defined($vzlist->{$vmid});
2968         if (my $pid = check_running($vmid)) {
2969             $vzlist->{$vmid}->{pid} = $pid;
2970         }
2971     }
2972
2973     return $vzlist;
2974 }
2975
2976 sub disksize {
2977     my ($storecfg, $conf) = @_;
2978
2979     my $bootdisk = $conf->{bootdisk};
2980     return undef if !$bootdisk;
2981     return undef if !is_valid_drivename($bootdisk);
2982
2983     return undef if !$conf->{$bootdisk};
2984
2985     my $drive = parse_drive($bootdisk, $conf->{$bootdisk});
2986     return undef if !defined($drive);
2987
2988     return undef if drive_is_cdrom($drive);
2989
2990     my $volid = $drive->{file};
2991     return undef if !$volid;
2992
2993     return $drive->{size};
2994 }
2995
2996 our $vmstatus_return_properties = {
2997     vmid => get_standard_option('pve-vmid'),
2998     status => {
2999         description => "Qemu process status.",
3000         type => 'string',
3001         enum => ['stopped', 'running'],
3002     },
3003     maxmem => {
3004         description => "Maximum memory in bytes.",
3005         type => 'integer',
3006         optional => 1,
3007         renderer => 'bytes',
3008     },
3009     maxdisk => {
3010         description => "Root disk size in bytes.",
3011         type => 'integer',
3012         optional => 1,
3013         renderer => 'bytes',
3014     },
3015     name => {
3016         description => "VM name.",
3017         type => 'string',
3018         optional => 1,
3019     },
3020     qmpstatus => {
3021         description => "Qemu QMP agent status.",
3022         type => 'string',
3023         optional => 1,
3024     },
3025     pid => {
3026         description => "PID of running qemu process.",
3027         type => 'integer',
3028         optional => 1,
3029     },
3030     uptime => {
3031         description => "Uptime.",
3032         type => 'integer',
3033         optional => 1,
3034         renderer => 'duration',
3035     },
3036     cpus => {
3037         description => "Maximum usable CPUs.",
3038         type => 'number',
3039         optional => 1,
3040     },
3041 };
3042
3043 my $last_proc_pid_stat;
3044
3045 # get VM status information
3046 # This must be fast and should not block ($full == false)
3047 # We only query KVM using QMP if $full == true (this can be slow)
3048 sub vmstatus {
3049     my ($opt_vmid, $full) = @_;
3050
3051     my $res = {};
3052
3053     my $storecfg = PVE::Storage::config();
3054
3055     my $list = vzlist();
3056     my $defaults = load_defaults();
3057
3058     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
3059
3060     my $cpucount = $cpuinfo->{cpus} || 1;
3061
3062     foreach my $vmid (keys %$list) {
3063         next if $opt_vmid && ($vmid ne $opt_vmid);
3064
3065         my $cfspath = PVE::QemuConfig->cfs_config_path($vmid);
3066         my $conf = PVE::Cluster::cfs_read_file($cfspath) || {};
3067
3068         my $d = { vmid => $vmid };
3069         $d->{pid} = $list->{$vmid}->{pid};
3070
3071         # fixme: better status?
3072         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
3073
3074         my $size = disksize($storecfg, $conf);
3075         if (defined($size)) {
3076             $d->{disk} = 0; # no info available
3077             $d->{maxdisk} = $size;
3078         } else {
3079             $d->{disk} = 0;
3080             $d->{maxdisk} = 0;
3081         }
3082
3083         $d->{cpus} = ($conf->{sockets} || $defaults->{sockets})
3084             * ($conf->{cores} || $defaults->{cores});
3085         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
3086         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
3087
3088         $d->{name} = $conf->{name} || "VM $vmid";
3089         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024)
3090             : $defaults->{memory}*(1024*1024);
3091
3092         if ($conf->{balloon}) {
3093             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
3094             $d->{shares} = defined($conf->{shares}) ? $conf->{shares}
3095                 : $defaults->{shares};
3096         }
3097
3098         $d->{uptime} = 0;
3099         $d->{cpu} = 0;
3100         $d->{mem} = 0;
3101
3102         $d->{netout} = 0;
3103         $d->{netin} = 0;
3104
3105         $d->{diskread} = 0;
3106         $d->{diskwrite} = 0;
3107
3108         $d->{template} = PVE::QemuConfig->is_template($conf);
3109
3110         $d->{serial} = 1 if conf_has_serial($conf);
3111
3112         $res->{$vmid} = $d;
3113     }
3114
3115     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
3116     foreach my $dev (keys %$netdev) {
3117         next if $dev !~ m/^tap([1-9]\d*)i/;
3118         my $vmid = $1;
3119         my $d = $res->{$vmid};
3120         next if !$d;
3121
3122         $d->{netout} += $netdev->{$dev}->{receive};
3123         $d->{netin} += $netdev->{$dev}->{transmit};
3124
3125         if ($full) {
3126             $d->{nics}->{$dev}->{netout} = $netdev->{$dev}->{receive};
3127             $d->{nics}->{$dev}->{netin} = $netdev->{$dev}->{transmit};
3128         }
3129
3130     }
3131
3132     my $ctime = gettimeofday;
3133
3134     foreach my $vmid (keys %$list) {
3135
3136         my $d = $res->{$vmid};
3137         my $pid = $d->{pid};
3138         next if !$pid;
3139
3140         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
3141         next if !$pstat; # not running
3142
3143         my $used = $pstat->{utime} + $pstat->{stime};
3144
3145         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
3146
3147         if ($pstat->{vsize}) {
3148             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
3149         }
3150
3151         my $old = $last_proc_pid_stat->{$pid};
3152         if (!$old) {
3153             $last_proc_pid_stat->{$pid} = {
3154                 time => $ctime,
3155                 used => $used,
3156                 cpu => 0,
3157             };
3158             next;
3159         }
3160
3161         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
3162
3163         if ($dtime > 1000) {
3164             my $dutime = $used -  $old->{used};
3165
3166             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
3167             $last_proc_pid_stat->{$pid} = {
3168                 time => $ctime,
3169                 used => $used,
3170                 cpu => $d->{cpu},
3171             };
3172         } else {
3173             $d->{cpu} = $old->{cpu};
3174         }
3175     }
3176
3177     return $res if !$full;
3178
3179     my $qmpclient = PVE::QMPClient->new();
3180
3181     my $ballooncb = sub {
3182         my ($vmid, $resp) = @_;
3183
3184         my $info = $resp->{'return'};
3185         return if !$info->{max_mem};
3186
3187         my $d = $res->{$vmid};
3188
3189         # use memory assigned to VM
3190         $d->{maxmem} = $info->{max_mem};
3191         $d->{balloon} = $info->{actual};
3192
3193         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
3194             $d->{mem} = $info->{total_mem} - $info->{free_mem};
3195             $d->{freemem} = $info->{free_mem};
3196         }
3197
3198         $d->{ballooninfo} = $info;
3199     };
3200
3201     my $blockstatscb = sub {
3202         my ($vmid, $resp) = @_;
3203         my $data = $resp->{'return'} || [];
3204         my $totalrdbytes = 0;
3205         my $totalwrbytes = 0;
3206
3207         for my $blockstat (@$data) {
3208             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
3209             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
3210
3211             $blockstat->{device} =~ s/drive-//;
3212             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
3213         }
3214         $res->{$vmid}->{diskread} = $totalrdbytes;
3215         $res->{$vmid}->{diskwrite} = $totalwrbytes;
3216     };
3217
3218     my $statuscb = sub {
3219         my ($vmid, $resp) = @_;
3220
3221         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
3222         # this fails if ballon driver is not loaded, so this must be
3223         # the last commnand (following command are aborted if this fails).
3224         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
3225
3226         my $status = 'unknown';
3227         if (!defined($status = $resp->{'return'}->{status})) {
3228             warn "unable to get VM status\n";
3229             return;
3230         }
3231
3232         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
3233     };
3234
3235     foreach my $vmid (keys %$list) {
3236         next if $opt_vmid && ($vmid ne $opt_vmid);
3237         next if !$res->{$vmid}->{pid}; # not running
3238         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
3239     }
3240
3241     $qmpclient->queue_execute(undef, 2);
3242
3243     foreach my $vmid (keys %$list) {
3244         next if $opt_vmid && ($vmid ne $opt_vmid);
3245         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
3246     }
3247
3248     return $res;
3249 }
3250
3251 sub foreach_drive {
3252     my ($conf, $func, @param) = @_;
3253
3254     foreach my $ds (valid_drive_names()) {
3255         next if !defined($conf->{$ds});
3256
3257         my $drive = parse_drive($ds, $conf->{$ds});
3258         next if !$drive;
3259
3260         &$func($ds, $drive, @param);
3261     }
3262 }
3263
3264 sub foreach_volid {
3265     my ($conf, $func, @param) = @_;
3266
3267     my $volhash = {};
3268
3269     my $test_volid = sub {
3270         my ($volid, $is_cdrom, $replicate, $shared, $snapname) = @_;
3271
3272         return if !$volid;
3273
3274         $volhash->{$volid}->{cdrom} //= 1;
3275         $volhash->{$volid}->{cdrom} = 0 if !$is_cdrom;
3276
3277         $volhash->{$volid}->{replicate} //= 0;
3278         $volhash->{$volid}->{replicate} = 1 if $replicate;
3279
3280         $volhash->{$volid}->{shared} //= 0;
3281         $volhash->{$volid}->{shared} = 1 if $shared;
3282
3283         $volhash->{$volid}->{referenced_in_config} //= 0;
3284         $volhash->{$volid}->{referenced_in_config} = 1 if !defined($snapname);
3285
3286         $volhash->{$volid}->{referenced_in_snapshot}->{$snapname} = 1
3287             if defined($snapname);
3288     };
3289
3290     foreach_drive($conf, sub {
3291         my ($ds, $drive) = @_;
3292         $test_volid->($drive->{file}, drive_is_cdrom($drive), $drive->{replicate} // 1, $drive->{shared}, undef);
3293     });
3294
3295     foreach my $snapname (keys %{$conf->{snapshots}}) {
3296         my $snap = $conf->{snapshots}->{$snapname};
3297         $test_volid->($snap->{vmstate}, 0, 1, $snapname);
3298         foreach_drive($snap, sub {
3299             my ($ds, $drive) = @_;
3300             $test_volid->($drive->{file}, drive_is_cdrom($drive), $drive->{replicate} // 1, $drive->{shared}, $snapname);
3301         });
3302     }
3303
3304     foreach my $volid (keys %$volhash) {
3305         &$func($volid, $volhash->{$volid}, @param);
3306     }
3307 }
3308
3309 sub conf_has_serial {
3310     my ($conf) = @_;
3311
3312     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3313         if ($conf->{"serial$i"}) {
3314             return 1;
3315         }
3316     }
3317
3318     return 0;
3319 }
3320
3321 sub vga_conf_has_spice {
3322     my ($vga) = @_;
3323
3324     my $vgaconf = parse_vga($vga);
3325     my $vgatype = $vgaconf->{type};
3326     return 0 if !$vgatype || $vgatype !~ m/^qxl([234])?$/;
3327
3328     return $1 || 1;
3329 }
3330
3331 my $host_arch; # FIXME: fix PVE::Tools::get_host_arch
3332 sub get_host_arch() {
3333     $host_arch = (POSIX::uname())[4] if !$host_arch;
3334     return $host_arch;
3335 }
3336
3337 sub is_native($) {
3338     my ($arch) = @_;
3339     return get_host_arch() eq $arch;
3340 }
3341
3342 my $default_machines = {
3343     x86_64 => 'pc',
3344     aarch64 => 'virt',
3345 };
3346
3347 sub get_basic_machine_info {
3348     my ($conf, $forcemachine) = @_;
3349
3350     my $arch = $conf->{arch} // get_host_arch();
3351     my $machine = $forcemachine || $conf->{machine} || $default_machines->{$arch};
3352     return ($arch, $machine);
3353 }
3354
3355 sub get_ovmf_files($) {
3356     my ($arch) = @_;
3357
3358     my $ovmf = $OVMF->{$arch}
3359         or die "no OVMF images known for architecture '$arch'\n";
3360
3361     return @$ovmf;
3362 }
3363
3364 my $Arch2Qemu = {
3365     aarch64 => '/usr/bin/qemu-system-aarch64',
3366     x86_64 => '/usr/bin/qemu-system-x86_64',
3367 };
3368 sub get_command_for_arch($) {
3369     my ($arch) = @_;
3370     return '/usr/bin/kvm' if is_native($arch);
3371
3372     my $cmd = $Arch2Qemu->{$arch}
3373         or die "don't know how to emulate architecture '$arch'\n";
3374     return $cmd;
3375 }
3376
3377 sub get_cpu_options {
3378     my ($conf, $arch, $kvm, $machine_type, $kvm_off, $kvmver, $winversion, $gpu_passthrough) = @_;
3379
3380     my $cpuFlags = [];
3381     my $ostype = $conf->{ostype};
3382
3383     my $cpu = $kvm ? "kvm64" : "qemu64";
3384     if ($arch eq 'aarch64') {
3385         $cpu = 'cortex-a57';
3386     }
3387     my $hv_vendor_id;
3388     if (my $cputype = $conf->{cpu}) {
3389         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
3390             or die "Cannot parse cpu description: $cputype\n";
3391         $cpu = $cpuconf->{cputype};
3392         $kvm_off = 1 if $cpuconf->{hidden};
3393         $hv_vendor_id = $cpuconf->{'hv-vendor-id'};
3394
3395         if (defined(my $flags = $cpuconf->{flags})) {
3396             push @$cpuFlags, split(";", $flags);
3397         }
3398     }
3399
3400     push @$cpuFlags , '+lahf_lm' if $cpu eq 'kvm64' && $arch eq 'x86_64';
3401
3402     push @$cpuFlags , '-x2apic'
3403         if $conf->{ostype} && $conf->{ostype} eq 'solaris';
3404
3405     push @$cpuFlags, '+sep' if $cpu eq 'kvm64' || $cpu eq 'kvm32';
3406
3407     push @$cpuFlags, '-rdtscp' if $cpu =~ m/^Opteron/;
3408
3409     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3) && $arch eq 'x86_64') {
3410
3411         push @$cpuFlags , '+kvm_pv_unhalt' if $kvm;
3412         push @$cpuFlags , '+kvm_pv_eoi' if $kvm;
3413     }
3414
3415     add_hyperv_enlightenments($cpuFlags, $winversion, $machine_type, $kvmver, $conf->{bios}, $gpu_passthrough, $hv_vendor_id) if $kvm;
3416
3417     push @$cpuFlags, 'enforce' if $cpu ne 'host' && $kvm && $arch eq 'x86_64';
3418
3419     push @$cpuFlags, 'kvm=off' if $kvm_off;
3420
3421     if (my $cpu_vendor = $cpu_vendor_list->{$cpu}) {
3422         push @$cpuFlags, "vendor=${cpu_vendor}"
3423             if $cpu_vendor ne 'default';
3424     } elsif ($arch ne 'aarch64') {
3425         die "internal error"; # should not happen
3426     }
3427
3428     $cpu .= "," . join(',', @$cpuFlags) if scalar(@$cpuFlags);
3429
3430     return ('-cpu', $cpu);
3431 }
3432
3433 sub config_to_command {
3434     my ($storecfg, $vmid, $conf, $defaults, $forcemachine) = @_;
3435
3436     my $cmd = [];
3437     my $globalFlags = [];
3438     my $machineFlags = [];
3439     my $rtcFlags = [];
3440     my $devices = [];
3441     my $pciaddr = '';
3442     my $bridges = {};
3443     my $kvmver = kvm_user_version();
3444     my $vernum = 0; # unknown
3445     my $ostype = $conf->{ostype};
3446     my $winversion = windows_version($ostype);
3447     my $kvm = $conf->{kvm};
3448
3449     my ($arch, $machine_type) = get_basic_machine_info($conf, $forcemachine);
3450     $kvm //= 1 if is_native($arch);
3451
3452     if ($kvm) {
3453         die "KVM virtualisation configured, but not available. Either disable in VM configuration or enable in BIOS.\n"
3454             if !defined kvm_version();
3455     }
3456
3457     if ($kvmver =~ m/^(\d+)\.(\d+)$/) {
3458         $vernum = $1*1000000+$2*1000;
3459     } elsif ($kvmver =~ m/^(\d+)\.(\d+)\.(\d+)$/) {
3460         $vernum = $1*1000000+$2*1000+$3;
3461     }
3462
3463     die "detected old qemu-kvm binary ($kvmver)\n" if $vernum < 15000;
3464
3465     my $have_ovz = -f '/proc/vz/vestat';
3466
3467     my $q35 = machine_type_is_q35($conf);
3468     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
3469     my $use_old_bios_files = undef;
3470     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3471
3472     my $cpuunits = defined($conf->{cpuunits}) ?
3473             $conf->{cpuunits} : $defaults->{cpuunits};
3474
3475     push @$cmd, get_command_for_arch($arch);
3476
3477     push @$cmd, '-id', $vmid;
3478
3479     my $vmname = $conf->{name} || "vm$vmid";
3480
3481     push @$cmd, '-name', $vmname;
3482
3483     my $use_virtio = 0;
3484
3485     my $qmpsocket = qmp_socket($vmid);
3486     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server,nowait";
3487     push @$cmd, '-mon', "chardev=qmp,mode=control";
3488
3489     if (qemu_machine_feature_enabled($machine_type, $kvmver, 2, 12)) {
3490         push @$cmd, '-chardev', "socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5";
3491         push @$cmd, '-mon', "chardev=qmp-event,mode=control";
3492     }
3493
3494     push @$cmd, '-pidfile' , pidfile_name($vmid);
3495
3496     push @$cmd, '-daemonize';
3497
3498     if ($conf->{smbios1}) {
3499         push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
3500     }
3501
3502     if ($conf->{vmgenid}) {
3503         push @$devices, '-device', 'vmgenid,guid='.$conf->{vmgenid};
3504     }
3505
3506     my ($ovmf_code, $ovmf_vars) = get_ovmf_files($arch);
3507     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3508         die "uefi base image not found\n" if ! -f $ovmf_code;
3509
3510         my $path;
3511         my $format;
3512         if (my $efidisk = $conf->{efidisk0}) {
3513             my $d = PVE::JSONSchema::parse_property_string($efidisk_fmt, $efidisk);
3514             my ($storeid, $volname) = PVE::Storage::parse_volume_id($d->{file}, 1);
3515             $format = $d->{format};
3516             if ($storeid) {
3517                 $path = PVE::Storage::path($storecfg, $d->{file});
3518                 if (!defined($format)) {
3519                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
3520                     $format = qemu_img_format($scfg, $volname);
3521                 }
3522             } else {
3523                 $path = $d->{file};
3524                 die "efidisk format must be specified\n"
3525                     if !defined($format);
3526             }
3527         } else {
3528             warn "no efidisk configured! Using temporary efivars disk.\n";
3529             $path = "/tmp/$vmid-ovmf.fd";
3530             PVE::Tools::file_copy($ovmf_vars, $path, -s $ovmf_vars);
3531             $format = 'raw';
3532         }
3533
3534         push @$cmd, '-drive', "if=pflash,unit=0,format=raw,readonly,file=$ovmf_code";
3535         push @$cmd, '-drive', "if=pflash,unit=1,format=$format,id=drive-efidisk0,file=$path";
3536     }
3537
3538
3539     # add usb controllers
3540     my @usbcontrollers = PVE::QemuServer::USB::get_usb_controllers($conf, $bridges, $arch, $machine_type, $usbdesc->{format}, $MAX_USB_DEVICES);
3541     push @$devices, @usbcontrollers if @usbcontrollers;
3542     my $vga = parse_vga($conf->{vga});
3543
3544     my $qxlnum = vga_conf_has_spice($conf->{vga});
3545     $vga->{type} = 'qxl' if $qxlnum;
3546
3547     if (!$vga->{type}) {
3548         if ($arch eq 'aarch64') {
3549             $vga->{type} = 'virtio';
3550         } elsif (qemu_machine_feature_enabled($machine_type, $kvmver, 2, 9)) {
3551             $vga->{type} = (!$winversion || $winversion >= 6) ? 'std' : 'cirrus';
3552         } else {
3553             $vga->{type} = ($winversion >= 6) ? 'std' : 'cirrus';
3554         }
3555     }
3556
3557     # enable absolute mouse coordinates (needed by vnc)
3558     my $tablet;
3559     if (defined($conf->{tablet})) {
3560         $tablet = $conf->{tablet};
3561     } else {
3562         $tablet = $defaults->{tablet};
3563         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
3564         $tablet = 0 if $vga->{type} =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
3565     }
3566
3567     if ($tablet) {
3568         push @$devices, '-device', print_tabletdevice_full($conf, $arch) if $tablet;
3569         my $kbd = print_keyboarddevice_full($conf, $arch);
3570         push @$devices, '-device', $kbd if defined($kbd);
3571     }
3572
3573     my $kvm_off = 0;
3574     my $gpu_passthrough;
3575
3576     # host pci devices
3577     for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
3578         my $d = parse_hostpci($conf->{"hostpci$i"});
3579         next if !$d;
3580
3581         my $pcie = $d->{pcie};
3582         if ($pcie) {
3583             die "q35 machine model is not enabled" if !$q35;
3584             # win7 wants to have the pcie devices directly on the pcie bus
3585             # instead of in the root port
3586             if ($winversion == 7) {
3587                 $pciaddr = print_pcie_addr("hostpci${i}bus0");
3588             } else {
3589                 $pciaddr = print_pcie_addr("hostpci$i");
3590             }
3591         } else {
3592             $pciaddr = print_pci_addr("hostpci$i", $bridges, $arch, $machine_type);
3593         }
3594
3595         my $rombar = defined($d->{rombar}) && !$d->{rombar} ? ',rombar=0' : '';
3596         my $romfile = $d->{romfile};
3597
3598         my $xvga = '';
3599         if ($d->{'x-vga'}) {
3600             $xvga = ',x-vga=on';
3601             $kvm_off = 1;
3602             $vga->{type} = 'none' if !defined($conf->{vga});
3603             $gpu_passthrough = 1;
3604
3605             if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3606                 $xvga = "";
3607             }
3608         }
3609         my $pcidevices = $d->{pciid};
3610         my $multifunction = 1 if @$pcidevices > 1;
3611         my $sysfspath;
3612         if ($d->{mdev} && scalar(@$pcidevices) == 1) {
3613             my $id = $pcidevices->[0]->{id};
3614             my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $i);
3615             $sysfspath = "/sys/bus/pci/devices/0000:$id/$uuid";
3616         } elsif ($d->{mdev}) {
3617             warn "ignoring mediated device with multifunction device\n";
3618         }
3619
3620         my $j=0;
3621         foreach my $pcidevice (@$pcidevices) {
3622
3623             my $id = "hostpci$i";
3624             $id .= ".$j" if $multifunction;
3625             my $addr = $pciaddr;
3626             $addr .= ".$j" if $multifunction;
3627             my $devicestr = "vfio-pci";
3628             if ($sysfspath) {
3629                 $devicestr .= ",sysfsdev=$sysfspath";
3630             } else {
3631                 $devicestr .= ",host=$pcidevice->{id}";
3632             }
3633             $devicestr .= ",id=$id$addr";
3634
3635             if($j == 0){
3636                 $devicestr .= "$rombar$xvga";
3637                 $devicestr .= ",multifunction=on" if $multifunction;
3638                 $devicestr .= ",romfile=/usr/share/kvm/$romfile" if $romfile;
3639             }
3640
3641             push @$devices, '-device', $devicestr;
3642             $j++;
3643         }
3644     }
3645
3646     # usb devices
3647     my @usbdevices = PVE::QemuServer::USB::get_usb_devices($conf, $usbdesc->{format}, $MAX_USB_DEVICES);
3648     push @$devices, @usbdevices if @usbdevices;
3649     # serial devices
3650     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3651         if (my $path = $conf->{"serial$i"}) {
3652             if ($path eq 'socket') {
3653                 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
3654                 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server,nowait";
3655                 # On aarch64, serial0 is the UART device. Qemu only allows
3656                 # connecting UART devices via the '-serial' command line, as
3657                 # the device has a fixed slot on the hardware...
3658                 if ($arch eq 'aarch64' && $i == 0) {
3659                     push @$devices, '-serial', "chardev:serial$i";
3660                 } else {
3661                     push @$devices, '-device', "isa-serial,chardev=serial$i";
3662                 }
3663             } else {
3664                 die "no such serial device\n" if ! -c $path;
3665                 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
3666                 push @$devices, '-device', "isa-serial,chardev=serial$i";
3667             }
3668         }
3669     }
3670
3671     # parallel devices
3672     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
3673         if (my $path = $conf->{"parallel$i"}) {
3674             die "no such parallel device\n" if ! -c $path;
3675             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
3676             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
3677             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
3678         }
3679     }
3680
3681
3682     my $sockets = 1;
3683     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3684     $sockets = $conf->{sockets} if  $conf->{sockets};
3685
3686     my $cores = $conf->{cores} || 1;
3687
3688     my $maxcpus = $sockets * $cores;
3689
3690     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
3691
3692     my $allowed_vcpus = $cpuinfo->{cpus};
3693
3694     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n"
3695         if ($allowed_vcpus < $maxcpus);
3696
3697     if($hotplug_features->{cpu} && qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 7)) {
3698
3699         push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3700         for (my $i = 2; $i <= $vcpus; $i++)  {
3701             my $cpustr = print_cpu_device($conf,$i);
3702             push @$cmd, '-device', $cpustr;
3703         }
3704
3705     } else {
3706
3707         push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3708     }
3709     push @$cmd, '-nodefaults';
3710
3711     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
3712
3713     my $bootindex_hash = {};
3714     my $i = 1;
3715     foreach my $o (split(//, $bootorder)) {
3716         $bootindex_hash->{$o} = $i*100;
3717         $i++;
3718     }
3719
3720     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
3721
3722     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
3723
3724     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
3725
3726     if ($vga->{type} && $vga->{type} !~ m/^serial\d+$/ && $vga->{type} ne 'none'){
3727         push @$devices, '-device', print_vga_device($conf, $vga, $arch, $machine_type, undef, $qxlnum, $bridges);
3728         my $socket = vnc_socket($vmid);
3729         push @$cmd,  '-vnc', "unix:$socket,x509,password";
3730     } else {
3731         push @$cmd, '-vga', 'none' if $vga->{type} eq 'none';
3732         push @$cmd, '-nographic';
3733     }
3734
3735     # time drift fix
3736     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
3737
3738     my $useLocaltime = $conf->{localtime};
3739
3740     if ($winversion >= 5) { # windows
3741         $useLocaltime = 1 if !defined($conf->{localtime});
3742
3743         # use time drift fix when acpi is enabled
3744         if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
3745             $tdf = 1 if !defined($conf->{tdf});
3746         }
3747     }
3748
3749     if ($winversion >= 6) {
3750         push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3751         push @$cmd, '-no-hpet';
3752     }
3753
3754     push @$rtcFlags, 'driftfix=slew' if $tdf;
3755
3756     if (!$kvm) {
3757         push @$machineFlags, 'accel=tcg';
3758     }
3759
3760     if ($machine_type) {
3761         push @$machineFlags, "type=${machine_type}";
3762     }
3763
3764     if ($conf->{startdate}) {
3765         push @$rtcFlags, "base=$conf->{startdate}";
3766     } elsif ($useLocaltime) {
3767         push @$rtcFlags, 'base=localtime';
3768     }
3769
3770     push @$cmd, get_cpu_options($conf, $arch, $kvm, $machine_type, $kvm_off, $kvmver, $winversion, $gpu_passthrough);
3771
3772     PVE::QemuServer::Memory::config($conf, $vmid, $sockets, $cores, $defaults, $hotplug_features, $cmd);
3773
3774     push @$cmd, '-S' if $conf->{freeze};
3775
3776     push @$cmd, '-k', $conf->{keyboard} if defined($conf->{keyboard});
3777
3778     # enable sound
3779     #my $soundhw = $conf->{soundhw} || $defaults->{soundhw};
3780     #push @$cmd, '-soundhw', 'es1370';
3781     #push @$cmd, '-soundhw', $soundhw if $soundhw;
3782
3783     if (parse_guest_agent($conf)->{enabled}) {
3784         my $qgasocket = qmp_socket($vmid, 1);
3785         my $pciaddr = print_pci_addr("qga0", $bridges, $arch, $machine_type);
3786         push @$devices, '-chardev', "socket,path=$qgasocket,server,nowait,id=qga0";
3787         push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3788         push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3789     }
3790
3791     my $spice_port;
3792
3793     if ($qxlnum) {
3794         if ($qxlnum > 1) {
3795             if ($winversion){
3796                 for(my $i = 1; $i < $qxlnum; $i++){
3797                     push @$devices, '-device', print_vga_device($conf, $vga, $arch, $machine_type, $i, $qxlnum, $bridges);
3798                 }
3799             } else {
3800                 # assume other OS works like Linux
3801                 my ($ram, $vram) = ("134217728", "67108864");
3802                 if ($vga->{memory}) {
3803                     $ram = PVE::Tools::convert_size($qxlnum*4*$vga->{memory}, 'mb' => 'b');
3804                     $vram = PVE::Tools::convert_size($qxlnum*2*$vga->{memory}, 'mb' => 'b');
3805                 }
3806                 push @$cmd, '-global', "qxl-vga.ram_size=$ram";
3807                 push @$cmd, '-global', "qxl-vga.vram_size=$vram";
3808             }
3809         }
3810
3811         my $pciaddr = print_pci_addr("spice", $bridges, $arch, $machine_type);
3812
3813         my $nodename = PVE::INotify::nodename();
3814         my $pfamily = PVE::Tools::get_host_address_family($nodename);
3815         my @nodeaddrs = PVE::Tools::getaddrinfo_all('localhost', family => $pfamily);
3816         die "failed to get an ip address of type $pfamily for 'localhost'\n" if !@nodeaddrs;
3817         my $localhost = PVE::Network::addr_to_ip($nodeaddrs[0]->{addr});
3818         $spice_port = PVE::Tools::next_spice_port($pfamily, $localhost);
3819
3820         push @$devices, '-spice', "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";
3821
3822         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3823         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3824         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3825     }
3826
3827     # enable balloon by default, unless explicitly disabled
3828     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3829         $pciaddr = print_pci_addr("balloon0", $bridges, $arch, $machine_type);
3830         push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
3831     }
3832
3833     if ($conf->{watchdog}) {
3834         my $wdopts = parse_watchdog($conf->{watchdog});
3835         $pciaddr = print_pci_addr("watchdog", $bridges, $arch, $machine_type);
3836         my $watchdog = $wdopts->{model} || 'i6300esb';
3837         push @$devices, '-device', "$watchdog$pciaddr";
3838         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3839     }
3840
3841     my $vollist = [];
3842     my $scsicontroller = {};
3843     my $ahcicontroller = {};
3844     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3845
3846     # Add iscsi initiator name if available
3847     if (my $initiator = get_initiator_name()) {
3848         push @$devices, '-iscsi', "initiator-name=$initiator";
3849     }
3850
3851     foreach_drive($conf, sub {
3852         my ($ds, $drive) = @_;
3853
3854         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3855             push @$vollist, $drive->{file};
3856         }
3857
3858         # ignore efidisk here, already added in bios/fw handling code above
3859         return if $drive->{interface} eq 'efidisk';
3860
3861         $use_virtio = 1 if $ds =~ m/^virtio/;
3862
3863         if (drive_is_cdrom ($drive)) {
3864             if ($bootindex_hash->{d}) {
3865                 $drive->{bootindex} = $bootindex_hash->{d};
3866                 $bootindex_hash->{d} += 1;
3867             }
3868         } else {
3869             if ($bootindex_hash->{c}) {
3870                 $drive->{bootindex} = $bootindex_hash->{c} if $conf->{bootdisk} && ($conf->{bootdisk} eq $ds);
3871                 $bootindex_hash->{c} += 1;
3872             }
3873         }
3874
3875         if($drive->{interface} eq 'virtio'){
3876            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3877         }
3878
3879         if ($drive->{interface} eq 'scsi') {
3880
3881             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
3882
3883             $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges, $arch, $machine_type);
3884             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
3885
3886             my $iothread = '';
3887             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
3888                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3889                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3890             } elsif ($drive->{iothread}) {
3891                 warn "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n";
3892             }
3893
3894             my $queues = '';
3895             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
3896                 $queues = ",num_queues=$drive->{queues}";
3897             }
3898
3899             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues" if !$scsicontroller->{$controller};
3900             $scsicontroller->{$controller}=1;
3901         }
3902
3903         if ($drive->{interface} eq 'sata') {
3904            my $controller = int($drive->{index} / $MAX_SATA_DISKS);
3905            $pciaddr = print_pci_addr("ahci$controller", $bridges, $arch, $machine_type);
3906            push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr" if !$ahcicontroller->{$controller};
3907            $ahcicontroller->{$controller}=1;
3908         }
3909
3910         my $drive_cmd = print_drive_full($storecfg, $vmid, $drive);
3911         push @$devices, '-drive',$drive_cmd;
3912         push @$devices, '-device', print_drivedevice_full($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type);
3913     });
3914
3915     for (my $i = 0; $i < $MAX_NETS; $i++) {
3916          next if !$conf->{"net$i"};
3917          my $d = parse_net($conf->{"net$i"});
3918          next if !$d;
3919
3920          $use_virtio = 1 if $d->{model} eq 'virtio';
3921
3922          if ($bootindex_hash->{n}) {
3923             $d->{bootindex} = $bootindex_hash->{n};
3924             $bootindex_hash->{n} += 1;
3925          }
3926
3927          my $netdevfull = print_netdev_full($vmid, $conf, $arch, $d, "net$i");
3928          push @$devices, '-netdev', $netdevfull;
3929
3930          my $netdevicefull = print_netdevice_full($vmid, $conf, $d, "net$i", $bridges, $use_old_bios_files, $arch, $machine_type);
3931          push @$devices, '-device', $netdevicefull;
3932     }
3933
3934     if ($conf->{ivshmem}) {
3935         my $ivshmem = PVE::JSONSchema::parse_property_string($ivshmem_fmt, $conf->{ivshmem});
3936
3937         my $bus;
3938         if ($q35) {
3939             $bus = print_pcie_addr("ivshmem");
3940         } else {
3941             $bus = print_pci_addr("ivshmem", $bridges, $arch, $machine_type);
3942         }
3943
3944         my $ivshmem_name = $ivshmem->{name} // $vmid;
3945         my $path = '/dev/shm/pve-shm-' . $ivshmem_name;
3946
3947         push @$devices, '-device', "ivshmem-plain,memdev=ivshmem$bus,";
3948         push @$devices, '-object', "memory-backend-file,id=ivshmem,share=on,mem-path=$path,size=$ivshmem->{size}M";
3949     }
3950
3951     if (!$q35) {
3952         # add pci bridges
3953         if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3954            $bridges->{1} = 1;
3955            $bridges->{2} = 1;
3956         }
3957
3958         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
3959
3960         while (my ($k, $v) = each %$bridges) {
3961             $pciaddr = print_pci_addr("pci.$k", undef, $arch, $machine_type);
3962             unshift @$devices, '-device', "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr" if $k > 0;
3963         }
3964     }
3965
3966     push @$cmd, @$devices;
3967     push @$cmd, '-rtc', join(',', @$rtcFlags)
3968         if scalar(@$rtcFlags);
3969     push @$cmd, '-machine', join(',', @$machineFlags)
3970         if scalar(@$machineFlags);
3971     push @$cmd, '-global', join(',', @$globalFlags)
3972         if scalar(@$globalFlags);
3973
3974     # add custom args
3975     if ($conf->{args}) {
3976         my $aa = PVE::Tools::split_args($conf->{args});
3977         push @$cmd, @$aa;
3978     }
3979
3980     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
3981 }
3982
3983 sub vnc_socket {
3984     my ($vmid) = @_;
3985     return "${var_run_tmpdir}/$vmid.vnc";
3986 }
3987
3988 sub spice_port {
3989     my ($vmid) = @_;
3990
3991     my $res = vm_mon_cmd($vmid, 'query-spice');
3992
3993     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
3994 }
3995
3996 sub qmp_socket {
3997     my ($vmid, $qga, $name) = @_;
3998     my $sockettype = $qga ? 'qga' : 'qmp';
3999     my $ext = $name ? '-'.$name : '';
4000     return "${var_run_tmpdir}/$vmid$ext.$sockettype";
4001 }
4002
4003 sub pidfile_name {
4004     my ($vmid) = @_;
4005     return "${var_run_tmpdir}/$vmid.pid";
4006 }
4007
4008 sub vm_devices_list {
4009     my ($vmid) = @_;
4010
4011     my $res = vm_mon_cmd($vmid, 'query-pci');
4012     my $devices_to_check = [];
4013     my $devices = {};
4014     foreach my $pcibus (@$res) {
4015         push @$devices_to_check, @{$pcibus->{devices}},
4016     }
4017
4018     while (@$devices_to_check) {
4019         my $to_check = [];
4020         for my $d (@$devices_to_check) {
4021             $devices->{$d->{'qdev_id'}} = 1 if $d->{'qdev_id'};
4022             next if !$d->{'pci_bridge'};
4023
4024             $devices->{$d->{'qdev_id'}} += scalar(@{$d->{'pci_bridge'}->{devices}});
4025             push @$to_check, @{$d->{'pci_bridge'}->{devices}};
4026         }
4027         $devices_to_check = $to_check;
4028     }
4029
4030     my $resblock = vm_mon_cmd($vmid, 'query-block');
4031     foreach my $block (@$resblock) {
4032         if($block->{device} =~ m/^drive-(\S+)/){
4033                 $devices->{$1} = 1;
4034         }
4035     }
4036
4037     my $resmice = vm_mon_cmd($vmid, 'query-mice');
4038     foreach my $mice (@$resmice) {
4039         if ($mice->{name} eq 'QEMU HID Tablet') {
4040             $devices->{tablet} = 1;
4041             last;
4042         }
4043     }
4044
4045     # for usb devices there is no query-usb
4046     # but we can iterate over the entries in
4047     # qom-list path=/machine/peripheral
4048     my $resperipheral = vm_mon_cmd($vmid, 'qom-list', path => '/machine/peripheral');
4049     foreach my $per (@$resperipheral) {
4050         if ($per->{name} =~ m/^usb\d+$/) {
4051             $devices->{$per->{name}} = 1;
4052         }
4053     }
4054
4055     return $devices;
4056 }
4057
4058 sub vm_deviceplug {
4059     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4060
4061     my $q35 = machine_type_is_q35($conf);
4062
4063     my $devices_list = vm_devices_list($vmid);
4064     return 1 if defined($devices_list->{$deviceid});
4065
4066     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid, $arch, $machine_type); # add PCI bridge if we need it for the device
4067
4068     if ($deviceid eq 'tablet') {
4069
4070         qemu_deviceadd($vmid, print_tabletdevice_full($conf, $arch));
4071
4072     } elsif ($deviceid eq 'keyboard') {
4073
4074         qemu_deviceadd($vmid, print_keyboarddevice_full($conf, $arch));
4075
4076     } elsif ($deviceid =~ m/^usb(\d+)$/) {
4077
4078         die "usb hotplug currently not reliable\n";
4079         # since we can't reliably hot unplug all added usb devices
4080         # and usb passthrough disables live migration
4081         # we disable usb hotplugging for now
4082         qemu_deviceadd($vmid, PVE::QemuServer::USB::print_usbdevice_full($conf, $deviceid, $device));
4083
4084     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4085
4086         qemu_iothread_add($vmid, $deviceid, $device);
4087
4088         qemu_driveadd($storecfg, $vmid, $device);
4089         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, $arch, $machine_type);
4090
4091         qemu_deviceadd($vmid, $devicefull);
4092         eval { qemu_deviceaddverify($vmid, $deviceid); };
4093         if (my $err = $@) {
4094             eval { qemu_drivedel($vmid, $deviceid); };
4095             warn $@ if $@;
4096             die $err;
4097         }
4098
4099     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4100
4101
4102         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
4103         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
4104         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
4105
4106         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
4107
4108         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
4109             qemu_iothread_add($vmid, $deviceid, $device);
4110             $devicefull .= ",iothread=iothread-$deviceid";
4111         }
4112
4113         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
4114             $devicefull .= ",num_queues=$device->{queues}";
4115         }
4116
4117         qemu_deviceadd($vmid, $devicefull);
4118         qemu_deviceaddverify($vmid, $deviceid);
4119
4120     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4121
4122         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device, $arch, $machine_type);
4123         qemu_driveadd($storecfg, $vmid, $device);
4124
4125         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, $arch, $machine_type);
4126         eval { qemu_deviceadd($vmid, $devicefull); };
4127         if (my $err = $@) {
4128             eval { qemu_drivedel($vmid, $deviceid); };
4129             warn $@ if $@;
4130             die $err;
4131         }
4132
4133     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4134
4135         return undef if !qemu_netdevadd($vmid, $conf, $arch, $device, $deviceid);
4136
4137         my $machine_type = PVE::QemuServer::qemu_machine_pxe($vmid, $conf);
4138         my $use_old_bios_files = undef;
4139         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
4140
4141         my $netdevicefull = print_netdevice_full($vmid, $conf, $device, $deviceid, undef, $use_old_bios_files, $arch, $machine_type);
4142         qemu_deviceadd($vmid, $netdevicefull);
4143         eval {
4144             qemu_deviceaddverify($vmid, $deviceid);
4145             qemu_set_link_status($vmid, $deviceid, !$device->{link_down});
4146         };
4147         if (my $err = $@) {
4148             eval { qemu_netdevdel($vmid, $deviceid); };
4149             warn $@ if $@;
4150             die $err;
4151         }
4152
4153     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
4154
4155         my $bridgeid = $2;
4156         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
4157         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
4158
4159         qemu_deviceadd($vmid, $devicefull);
4160         qemu_deviceaddverify($vmid, $deviceid);
4161
4162     } else {
4163         die "can't hotplug device '$deviceid'\n";
4164     }
4165
4166     return 1;
4167 }
4168
4169 # fixme: this should raise exceptions on error!
4170 sub vm_deviceunplug {
4171     my ($vmid, $conf, $deviceid) = @_;
4172
4173     my $devices_list = vm_devices_list($vmid);
4174     return 1 if !defined($devices_list->{$deviceid});
4175
4176     die "can't unplug bootdisk" if $conf->{bootdisk} && $conf->{bootdisk} eq $deviceid;
4177
4178     if ($deviceid eq 'tablet' || $deviceid eq 'keyboard') {
4179
4180         qemu_devicedel($vmid, $deviceid);
4181
4182     } elsif ($deviceid =~ m/^usb\d+$/) {
4183
4184         die "usb hotplug currently not reliable\n";
4185         # when unplugging usb devices this way,
4186         # there may be remaining usb controllers/hubs
4187         # so we disable it for now
4188         qemu_devicedel($vmid, $deviceid);
4189         qemu_devicedelverify($vmid, $deviceid);
4190
4191     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4192
4193         qemu_devicedel($vmid, $deviceid);
4194         qemu_devicedelverify($vmid, $deviceid);
4195         qemu_drivedel($vmid, $deviceid);
4196         qemu_iothread_del($conf, $vmid, $deviceid);
4197
4198     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4199
4200         qemu_devicedel($vmid, $deviceid);
4201         qemu_devicedelverify($vmid, $deviceid);
4202         qemu_iothread_del($conf, $vmid, $deviceid);
4203
4204     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4205
4206         qemu_devicedel($vmid, $deviceid);
4207         qemu_drivedel($vmid, $deviceid);
4208         qemu_deletescsihw($conf, $vmid, $deviceid);
4209
4210     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4211
4212         qemu_devicedel($vmid, $deviceid);
4213         qemu_devicedelverify($vmid, $deviceid);
4214         qemu_netdevdel($vmid, $deviceid);
4215
4216     } else {
4217         die "can't unplug device '$deviceid'\n";
4218     }
4219
4220     return 1;
4221 }
4222
4223 sub qemu_deviceadd {
4224     my ($vmid, $devicefull) = @_;
4225
4226     $devicefull = "driver=".$devicefull;
4227     my %options =  split(/[=,]/, $devicefull);
4228
4229     vm_mon_cmd($vmid, "device_add" , %options);
4230 }
4231
4232 sub qemu_devicedel {
4233     my ($vmid, $deviceid) = @_;
4234
4235     my $ret = vm_mon_cmd($vmid, "device_del", id => $deviceid);
4236 }
4237
4238 sub qemu_iothread_add {
4239     my($vmid, $deviceid, $device) = @_;
4240
4241     if ($device->{iothread}) {
4242         my $iothreads = vm_iothreads_list($vmid);
4243         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
4244     }
4245 }
4246
4247 sub qemu_iothread_del {
4248     my($conf, $vmid, $deviceid) = @_;
4249
4250     my $device = parse_drive($deviceid, $conf->{$deviceid});
4251     if ($device->{iothread}) {
4252         my $iothreads = vm_iothreads_list($vmid);
4253         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
4254     }
4255 }
4256
4257 sub qemu_objectadd {
4258     my($vmid, $objectid, $qomtype) = @_;
4259
4260     vm_mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
4261
4262     return 1;
4263 }
4264
4265 sub qemu_objectdel {
4266     my($vmid, $objectid) = @_;
4267
4268     vm_mon_cmd($vmid, "object-del", id => $objectid);
4269
4270     return 1;
4271 }
4272
4273 sub qemu_driveadd {
4274     my ($storecfg, $vmid, $device) = @_;
4275
4276     my $drive = print_drive_full($storecfg, $vmid, $device);
4277     $drive =~ s/\\/\\\\/g;
4278     my $ret = vm_human_monitor_command($vmid, "drive_add auto \"$drive\"");
4279
4280     # If the command succeeds qemu prints: "OK"
4281     return 1 if $ret =~ m/OK/s;
4282
4283     die "adding drive failed: $ret\n";
4284 }
4285
4286 sub qemu_drivedel {
4287     my($vmid, $deviceid) = @_;
4288
4289     my $ret = vm_human_monitor_command($vmid, "drive_del drive-$deviceid");
4290     $ret =~ s/^\s+//;
4291
4292     return 1 if $ret eq "";
4293
4294     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
4295     return 1 if $ret =~ m/Device \'.*?\' not found/s;
4296
4297     die "deleting drive $deviceid failed : $ret\n";
4298 }
4299
4300 sub qemu_deviceaddverify {
4301     my ($vmid, $deviceid) = @_;
4302
4303     for (my $i = 0; $i <= 5; $i++) {
4304          my $devices_list = vm_devices_list($vmid);
4305          return 1 if defined($devices_list->{$deviceid});
4306          sleep 1;
4307     }
4308
4309     die "error on hotplug device '$deviceid'\n";
4310 }
4311
4312
4313 sub qemu_devicedelverify {
4314     my ($vmid, $deviceid) = @_;
4315
4316     # need to verify that the device is correctly removed as device_del
4317     # is async and empty return is not reliable
4318
4319     for (my $i = 0; $i <= 5; $i++) {
4320          my $devices_list = vm_devices_list($vmid);
4321          return 1 if !defined($devices_list->{$deviceid});
4322          sleep 1;
4323     }
4324
4325     die "error on hot-unplugging device '$deviceid'\n";
4326 }
4327
4328 sub qemu_findorcreatescsihw {
4329     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4330
4331     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4332
4333     my $scsihwid="$controller_prefix$controller";
4334     my $devices_list = vm_devices_list($vmid);
4335
4336     if(!defined($devices_list->{$scsihwid})) {
4337         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device, $arch, $machine_type);
4338     }
4339
4340     return 1;
4341 }
4342
4343 sub qemu_deletescsihw {
4344     my ($conf, $vmid, $opt) = @_;
4345
4346     my $device = parse_drive($opt, $conf->{$opt});
4347
4348     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
4349         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
4350         return 1;
4351     }
4352
4353     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4354
4355     my $devices_list = vm_devices_list($vmid);
4356     foreach my $opt (keys %{$devices_list}) {
4357         if (PVE::QemuServer::is_valid_drivename($opt)) {
4358             my $drive = PVE::QemuServer::parse_drive($opt, $conf->{$opt});
4359             if($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
4360                 return 1;
4361             }
4362         }
4363     }
4364
4365     my $scsihwid="scsihw$controller";
4366
4367     vm_deviceunplug($vmid, $conf, $scsihwid);
4368
4369     return 1;
4370 }
4371
4372 sub qemu_add_pci_bridge {
4373     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4374
4375     my $bridges = {};
4376
4377     my $bridgeid;
4378
4379     print_pci_addr($device, $bridges, $arch, $machine_type);
4380
4381     while (my ($k, $v) = each %$bridges) {
4382         $bridgeid = $k;
4383     }
4384     return 1 if !defined($bridgeid) || $bridgeid < 1;
4385
4386     my $bridge = "pci.$bridgeid";
4387     my $devices_list = vm_devices_list($vmid);
4388
4389     if (!defined($devices_list->{$bridge})) {
4390         vm_deviceplug($storecfg, $conf, $vmid, $bridge, $arch, $machine_type);
4391     }
4392
4393     return 1;
4394 }
4395
4396 sub qemu_set_link_status {
4397     my ($vmid, $device, $up) = @_;
4398
4399     vm_mon_cmd($vmid, "set_link", name => $device,
4400                up => $up ? JSON::true : JSON::false);
4401 }
4402
4403 sub qemu_netdevadd {
4404     my ($vmid, $conf, $arch, $device, $deviceid) = @_;
4405
4406     my $netdev = print_netdev_full($vmid, $conf, $arch, $device, $deviceid, 1);
4407     my %options =  split(/[=,]/, $netdev);
4408
4409     vm_mon_cmd($vmid, "netdev_add",  %options);
4410     return 1;
4411 }
4412
4413 sub qemu_netdevdel {
4414     my ($vmid, $deviceid) = @_;
4415
4416     vm_mon_cmd($vmid, "netdev_del", id => $deviceid);
4417 }
4418
4419 sub qemu_usb_hotplug {
4420     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4421
4422     return if !$device;
4423
4424     # remove the old one first
4425     vm_deviceunplug($vmid, $conf, $deviceid);
4426
4427     # check if xhci controller is necessary and available
4428     if ($device->{usb3}) {
4429
4430         my $devicelist = vm_devices_list($vmid);
4431
4432         if (!$devicelist->{xhci}) {
4433             my $pciaddr = print_pci_addr("xhci", undef, $arch, $machine_type);
4434             qemu_deviceadd($vmid, "nec-usb-xhci,id=xhci$pciaddr");
4435         }
4436     }
4437     my $d = parse_usb_device($device->{host});
4438     $d->{usb3} = $device->{usb3};
4439
4440     # add the new one
4441     vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $d, $arch, $machine_type);
4442 }
4443
4444 sub qemu_cpu_hotplug {
4445     my ($vmid, $conf, $vcpus) = @_;
4446
4447     my $machine_type = PVE::QemuServer::get_current_qemu_machine($vmid);
4448
4449     my $sockets = 1;
4450     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
4451     $sockets = $conf->{sockets} if  $conf->{sockets};
4452     my $cores = $conf->{cores} || 1;
4453     my $maxcpus = $sockets * $cores;
4454
4455     $vcpus = $maxcpus if !$vcpus;
4456
4457     die "you can't add more vcpus than maxcpus\n"
4458         if $vcpus > $maxcpus;
4459
4460     my $currentvcpus = $conf->{vcpus} || $maxcpus;
4461
4462     if ($vcpus < $currentvcpus) {
4463
4464         if (qemu_machine_feature_enabled ($machine_type, undef, 2, 7)) {
4465
4466             for (my $i = $currentvcpus; $i > $vcpus; $i--) {
4467                 qemu_devicedel($vmid, "cpu$i");
4468                 my $retry = 0;
4469                 my $currentrunningvcpus = undef;
4470                 while (1) {
4471                     $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
4472                     last if scalar(@{$currentrunningvcpus}) == $i-1;
4473                     raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
4474                     $retry++;
4475                     sleep 1;
4476                 }
4477                 #update conf after each succesfull cpu unplug
4478                 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4479                 PVE::QemuConfig->write_config($vmid, $conf);
4480             }
4481         } else {
4482             die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
4483         }
4484
4485         return;
4486     }
4487
4488     my $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
4489     die "vcpus in running vm does not match its configuration\n"
4490         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
4491
4492     if (qemu_machine_feature_enabled ($machine_type, undef, 2, 7)) {
4493
4494         for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
4495             my $cpustr = print_cpu_device($conf, $i);
4496             qemu_deviceadd($vmid, $cpustr);
4497
4498             my $retry = 0;
4499             my $currentrunningvcpus = undef;
4500             while (1) {
4501                 $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
4502                 last if scalar(@{$currentrunningvcpus}) == $i;
4503                 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
4504                 sleep 1;
4505                 $retry++;
4506             }
4507             #update conf after each succesfull cpu hotplug
4508             $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4509             PVE::QemuConfig->write_config($vmid, $conf);
4510         }
4511     } else {
4512
4513         for (my $i = $currentvcpus; $i < $vcpus; $i++) {
4514             vm_mon_cmd($vmid, "cpu-add", id => int($i));
4515         }
4516     }
4517 }
4518
4519 sub qemu_block_set_io_throttle {
4520     my ($vmid, $deviceid,
4521         $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
4522         $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max,
4523         $bps_max_length, $bps_rd_max_length, $bps_wr_max_length,
4524         $iops_max_length, $iops_rd_max_length, $iops_wr_max_length) = @_;
4525
4526     return if !check_running($vmid) ;
4527
4528     vm_mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
4529         bps => int($bps),
4530         bps_rd => int($bps_rd),
4531         bps_wr => int($bps_wr),
4532         iops => int($iops),
4533         iops_rd => int($iops_rd),
4534         iops_wr => int($iops_wr),
4535         bps_max => int($bps_max),
4536         bps_rd_max => int($bps_rd_max),
4537         bps_wr_max => int($bps_wr_max),
4538         iops_max => int($iops_max),
4539         iops_rd_max => int($iops_rd_max),
4540         iops_wr_max => int($iops_wr_max),
4541         bps_max_length => int($bps_max_length),
4542         bps_rd_max_length => int($bps_rd_max_length),
4543         bps_wr_max_length => int($bps_wr_max_length),
4544         iops_max_length => int($iops_max_length),
4545         iops_rd_max_length => int($iops_rd_max_length),
4546         iops_wr_max_length => int($iops_wr_max_length),
4547     );
4548
4549 }
4550
4551 # old code, only used to shutdown old VM after update
4552 sub __read_avail {
4553     my ($fh, $timeout) = @_;
4554
4555     my $sel = new IO::Select;
4556     $sel->add($fh);
4557
4558     my $res = '';
4559     my $buf;
4560
4561     my @ready;
4562     while (scalar (@ready = $sel->can_read($timeout))) {
4563         my $count;
4564         if ($count = $fh->sysread($buf, 8192)) {
4565             if ($buf =~ /^(.*)\(qemu\) $/s) {
4566                 $res .= $1;
4567                 last;
4568             } else {
4569                 $res .= $buf;
4570             }
4571         } else {
4572             if (!defined($count)) {
4573                 die "$!\n";
4574             }
4575             last;
4576         }
4577     }
4578
4579     die "monitor read timeout\n" if !scalar(@ready);
4580
4581     return $res;
4582 }
4583
4584 sub qemu_block_resize {
4585     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4586
4587     my $running = check_running($vmid);
4588
4589     $size = 0 if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4590
4591     return if !$running;
4592
4593     vm_mon_cmd($vmid, "block_resize", device => $deviceid, size => int($size));
4594
4595 }
4596
4597 sub qemu_volume_snapshot {
4598     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4599
4600     my $running = check_running($vmid);
4601
4602     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4603         vm_mon_cmd($vmid, 'blockdev-snapshot-internal-sync', device => $deviceid, name => $snap);
4604     } else {
4605         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4606     }
4607 }
4608
4609 sub qemu_volume_snapshot_delete {
4610     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4611
4612     my $running = check_running($vmid);
4613
4614     if($running) {
4615
4616         $running = undef;
4617         my $conf = PVE::QemuConfig->load_config($vmid);
4618         foreach_drive($conf, sub {
4619             my ($ds, $drive) = @_;
4620             $running = 1 if $drive->{file} eq $volid;
4621         });
4622     }
4623
4624     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4625         vm_mon_cmd($vmid, 'blockdev-snapshot-delete-internal-sync', device => $deviceid, name => $snap);
4626     } else {
4627         PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4628     }
4629 }
4630
4631 sub set_migration_caps {
4632     my ($vmid) = @_;
4633
4634     my $cap_ref = [];
4635
4636     my $enabled_cap = {
4637         "auto-converge" => 1,
4638         "xbzrle" => 1,
4639         "x-rdma-pin-all" => 0,
4640         "zero-blocks" => 0,
4641         "compress" => 0
4642     };
4643
4644     my $supported_capabilities = vm_mon_cmd_nocheck($vmid, "query-migrate-capabilities");
4645
4646     for my $supported_capability (@$supported_capabilities) {
4647         push @$cap_ref, {
4648             capability => $supported_capability->{capability},
4649             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4650         };
4651     }
4652
4653     vm_mon_cmd_nocheck($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4654 }
4655
4656 my $fast_plug_option = {
4657     'lock' => 1,
4658     'name' => 1,
4659     'onboot' => 1,
4660     'shares' => 1,
4661     'startup' => 1,
4662     'description' => 1,
4663     'protection' => 1,
4664     'vmstatestorage' => 1,
4665     'hookscript' => 1,
4666 };
4667
4668 # hotplug changes in [PENDING]
4669 # $selection hash can be used to only apply specified options, for
4670 # example: { cores => 1 } (only apply changed 'cores')
4671 # $errors ref is used to return error messages
4672 sub vmconfig_hotplug_pending {
4673     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4674
4675     my $defaults = load_defaults();
4676     my ($arch, $machine_type) = get_basic_machine_info($conf, undef);
4677
4678     # commit values which do not have any impact on running VM first
4679     # Note: those option cannot raise errors, we we do not care about
4680     # $selection and always apply them.
4681
4682     my $add_error = sub {
4683         my ($opt, $msg) = @_;
4684         $errors->{$opt} = "hotplug problem - $msg";
4685     };
4686
4687     my $changes = 0;
4688     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4689         if ($fast_plug_option->{$opt}) {
4690             $conf->{$opt} = $conf->{pending}->{$opt};
4691             delete $conf->{pending}->{$opt};
4692             $changes = 1;
4693         }
4694     }
4695
4696     if ($changes) {
4697         PVE::QemuConfig->write_config($vmid, $conf);
4698         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4699     }
4700
4701     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4702
4703     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4704     while (my ($opt, $force) = each %$pending_delete_hash) {
4705         next if $selection && !$selection->{$opt};
4706         eval {
4707             if ($opt eq 'hotplug') {
4708                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
4709             } elsif ($opt eq 'tablet') {
4710                 die "skip\n" if !$hotplug_features->{usb};
4711                 if ($defaults->{tablet}) {
4712                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4713                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4714                         if $arch eq 'aarch64';
4715                 } else {
4716                     vm_deviceunplug($vmid, $conf, 'tablet');
4717                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4718                 }
4719             } elsif ($opt =~ m/^usb\d+/) {
4720                 die "skip\n";
4721                 # since we cannot reliably hot unplug usb devices
4722                 # we are disabling it
4723                 die "skip\n" if !$hotplug_features->{usb} || $conf->{$opt} =~ m/spice/i;
4724                 vm_deviceunplug($vmid, $conf, $opt);
4725             } elsif ($opt eq 'vcpus') {
4726                 die "skip\n" if !$hotplug_features->{cpu};
4727                 qemu_cpu_hotplug($vmid, $conf, undef);
4728             } elsif ($opt eq 'balloon') {
4729                 # enable balloon device is not hotpluggable
4730                 die "skip\n" if defined($conf->{balloon}) && $conf->{balloon} == 0;
4731                 # here we reset the ballooning value to memory
4732                 my $balloon = $conf->{memory} || $defaults->{memory};
4733                 vm_mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4734             } elsif ($fast_plug_option->{$opt}) {
4735                 # do nothing
4736             } elsif ($opt =~ m/^net(\d+)$/) {
4737                 die "skip\n" if !$hotplug_features->{network};
4738                 vm_deviceunplug($vmid, $conf, $opt);
4739             } elsif (is_valid_drivename($opt)) {
4740                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4741                 vm_deviceunplug($vmid, $conf, $opt);
4742                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4743             } elsif ($opt =~ m/^memory$/) {
4744                 die "skip\n" if !$hotplug_features->{memory};
4745                 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
4746             } elsif ($opt eq 'cpuunits') {
4747                 cgroups_write("cpu", $vmid, "cpu.shares", $defaults->{cpuunits});
4748             } elsif ($opt eq 'cpulimit') {
4749                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", -1);
4750             } else {
4751                 die "skip\n";
4752             }
4753         };
4754         if (my $err = $@) {
4755             &$add_error($opt, $err) if $err ne "skip\n";
4756         } else {
4757             # save new config if hotplug was successful
4758             delete $conf->{$opt};
4759             vmconfig_undelete_pending_option($conf, $opt);
4760             PVE::QemuConfig->write_config($vmid, $conf);
4761             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4762         }
4763     }
4764
4765     my $apply_pending_cloudinit;
4766     $apply_pending_cloudinit = sub {
4767         my ($key, $value) = @_;
4768         $apply_pending_cloudinit = sub {}; # once is enough
4769
4770         my @cloudinit_opts = keys %$confdesc_cloudinit;
4771         foreach my $opt (keys %{$conf->{pending}}) {
4772             next if !grep { $_ eq $opt } @cloudinit_opts;
4773             $conf->{$opt} = delete $conf->{pending}->{$opt};
4774         }
4775
4776         my $new_conf = { %$conf };
4777         $new_conf->{$key} = $value;
4778         PVE::QemuServer::Cloudinit::generate_cloudinitconfig($new_conf, $vmid);
4779     };
4780
4781     foreach my $opt (keys %{$conf->{pending}}) {
4782         next if $selection && !$selection->{$opt};
4783         my $value = $conf->{pending}->{$opt};
4784         eval {
4785             if ($opt eq 'hotplug') {
4786                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
4787             } elsif ($opt eq 'tablet') {
4788                 die "skip\n" if !$hotplug_features->{usb};
4789                 if ($value == 1) {
4790                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4791                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4792                         if $arch eq 'aarch64';
4793                 } elsif ($value == 0) {
4794                     vm_deviceunplug($vmid, $conf, 'tablet');
4795                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4796                 }
4797             } elsif ($opt =~ m/^usb\d+$/) {
4798                 die "skip\n";
4799                 # since we cannot reliably hot unplug usb devices
4800                 # we are disabling it
4801                 die "skip\n" if !$hotplug_features->{usb} || $value =~ m/spice/i;
4802                 my $d = eval { PVE::JSONSchema::parse_property_string($usbdesc->{format}, $value) };
4803                 die "skip\n" if !$d;
4804                 qemu_usb_hotplug($storecfg, $conf, $vmid, $opt, $d, $arch, $machine_type);
4805             } elsif ($opt eq 'vcpus') {
4806                 die "skip\n" if !$hotplug_features->{cpu};
4807                 qemu_cpu_hotplug($vmid, $conf, $value);
4808             } elsif ($opt eq 'balloon') {
4809                 # enable/disable balloning device is not hotpluggable
4810                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
4811                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
4812                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
4813
4814                 # allow manual ballooning if shares is set to zero
4815                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
4816                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
4817                     vm_mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4818                 }
4819             } elsif ($opt =~ m/^net(\d+)$/) {
4820                 # some changes can be done without hotplug
4821                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
4822                                     $vmid, $opt, $value, $arch, $machine_type);
4823             } elsif (is_valid_drivename($opt)) {
4824                 # some changes can be done without hotplug
4825                 my $drive = parse_drive($opt, $value);
4826                 if (drive_is_cloudinit($drive)) {
4827                     &$apply_pending_cloudinit($opt, $value);
4828                 }
4829                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
4830                                      $vmid, $opt, $value, 1, $arch, $machine_type);
4831             } elsif ($opt =~ m/^memory$/) { #dimms
4832                 die "skip\n" if !$hotplug_features->{memory};
4833                 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
4834             } elsif ($opt eq 'cpuunits') {
4835                 cgroups_write("cpu", $vmid, "cpu.shares", $conf->{pending}->{$opt});
4836             } elsif ($opt eq 'cpulimit') {
4837                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
4838                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", $cpulimit);
4839             } else {
4840                 die "skip\n";  # skip non-hot-pluggable options
4841             }
4842         };
4843         if (my $err = $@) {
4844             &$add_error($opt, $err) if $err ne "skip\n";
4845         } else {
4846             # save new config if hotplug was successful
4847             $conf->{$opt} = $value;
4848             delete $conf->{pending}->{$opt};
4849             PVE::QemuConfig->write_config($vmid, $conf);
4850             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4851         }
4852     }
4853 }
4854
4855 sub try_deallocate_drive {
4856     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
4857
4858     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
4859         my $volid = $drive->{file};
4860         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
4861             my $sid = PVE::Storage::parse_volume_id($volid);
4862             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
4863
4864             # check if the disk is really unused
4865             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
4866                 if is_volume_in_use($storecfg, $conf, $key, $volid);
4867             PVE::Storage::vdisk_free($storecfg, $volid);
4868             return 1;
4869         } else {
4870             # If vm is not owner of this disk remove from config
4871             return 1;
4872         }
4873     }
4874
4875     return undef;
4876 }
4877
4878 sub vmconfig_delete_or_detach_drive {
4879     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
4880
4881     my $drive = parse_drive($opt, $conf->{$opt});
4882
4883     my $rpcenv = PVE::RPCEnvironment::get();
4884     my $authuser = $rpcenv->get_user();
4885
4886     if ($force) {
4887         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
4888         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
4889     } else {
4890         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
4891     }
4892 }
4893
4894 sub vmconfig_apply_pending {
4895     my ($vmid, $conf, $storecfg) = @_;
4896
4897     # cold plug
4898
4899     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4900     while (my ($opt, $force) = each %$pending_delete_hash) {
4901         die "internal error" if $opt =~ m/^unused/;
4902         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4903         if (!defined($conf->{$opt})) {
4904             vmconfig_undelete_pending_option($conf, $opt);
4905             PVE::QemuConfig->write_config($vmid, $conf);
4906         } elsif (is_valid_drivename($opt)) {
4907             vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4908             vmconfig_undelete_pending_option($conf, $opt);
4909             delete $conf->{$opt};
4910             PVE::QemuConfig->write_config($vmid, $conf);
4911         } else {
4912             vmconfig_undelete_pending_option($conf, $opt);
4913             delete $conf->{$opt};
4914             PVE::QemuConfig->write_config($vmid, $conf);
4915         }
4916     }
4917
4918     $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4919
4920     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4921         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4922
4923         if (defined($conf->{$opt}) && ($conf->{$opt} eq $conf->{pending}->{$opt})) {
4924             # skip if nothing changed
4925         } elsif (is_valid_drivename($opt)) {
4926             vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
4927                 if defined($conf->{$opt});
4928             $conf->{$opt} = $conf->{pending}->{$opt};
4929         } else {
4930             $conf->{$opt} = $conf->{pending}->{$opt};
4931         }
4932
4933         delete $conf->{pending}->{$opt};
4934         PVE::QemuConfig->write_config($vmid, $conf);
4935     }
4936 }
4937
4938 my $safe_num_ne = sub {
4939     my ($a, $b) = @_;
4940
4941     return 0 if !defined($a) && !defined($b);
4942     return 1 if !defined($a);
4943     return 1 if !defined($b);
4944
4945     return $a != $b;
4946 };
4947
4948 my $safe_string_ne = sub {
4949     my ($a, $b) = @_;
4950
4951     return 0 if !defined($a) && !defined($b);
4952     return 1 if !defined($a);
4953     return 1 if !defined($b);
4954
4955     return $a ne $b;
4956 };
4957
4958 sub vmconfig_update_net {
4959     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
4960
4961     my $newnet = parse_net($value);
4962
4963     if ($conf->{$opt}) {
4964         my $oldnet = parse_net($conf->{$opt});
4965
4966         if (&$safe_string_ne($oldnet->{model}, $newnet->{model}) ||
4967             &$safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
4968             &$safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
4969             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
4970
4971             # for non online change, we try to hot-unplug
4972             die "skip\n" if !$hotplug;
4973             vm_deviceunplug($vmid, $conf, $opt);
4974         } else {
4975
4976             die "internal error" if $opt !~ m/net(\d+)/;
4977             my $iface = "tap${vmid}i$1";
4978
4979             if (&$safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
4980                 &$safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
4981                 &$safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
4982                 &$safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
4983                 PVE::Network::tap_unplug($iface);
4984                 PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
4985             } elsif (&$safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
4986                 # Rate can be applied on its own but any change above needs to
4987                 # include the rate in tap_plug since OVS resets everything.
4988                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
4989             }
4990
4991             if (&$safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
4992                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
4993             }
4994
4995             return 1;
4996         }
4997     }
4998
4999     if ($hotplug) {
5000         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet, $arch, $machine_type);
5001     } else {
5002         die "skip\n";
5003     }
5004 }
5005
5006 sub vmconfig_update_disk {
5007     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $force, $arch, $machine_type) = @_;
5008
5009     # fixme: do we need force?
5010
5011     my $drive = parse_drive($opt, $value);
5012
5013     if ($conf->{$opt}) {
5014
5015         if (my $old_drive = parse_drive($opt, $conf->{$opt}))  {
5016
5017             my $media = $drive->{media} || 'disk';
5018             my $oldmedia = $old_drive->{media} || 'disk';
5019             die "unable to change media type\n" if $media ne $oldmedia;
5020
5021             if (!drive_is_cdrom($old_drive)) {
5022
5023                 if ($drive->{file} ne $old_drive->{file}) {
5024
5025                     die "skip\n" if !$hotplug;
5026
5027                     # unplug and register as unused
5028                     vm_deviceunplug($vmid, $conf, $opt);
5029                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
5030
5031                 } else {
5032                     # update existing disk
5033
5034                     # skip non hotpluggable value
5035                     if (&$safe_string_ne($drive->{discard}, $old_drive->{discard}) ||
5036                         &$safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
5037                         &$safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
5038                         &$safe_string_ne($drive->{cache}, $old_drive->{cache})) {
5039                         die "skip\n";
5040                     }
5041
5042                     # apply throttle
5043                     if (&$safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
5044                         &$safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
5045                         &$safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
5046                         &$safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
5047                         &$safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
5048                         &$safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
5049                         &$safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
5050                         &$safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
5051                         &$safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
5052                         &$safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
5053                         &$safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
5054                         &$safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max}) ||
5055                         &$safe_num_ne($drive->{bps_max_length}, $old_drive->{bps_max_length}) ||
5056                         &$safe_num_ne($drive->{bps_rd_max_length}, $old_drive->{bps_rd_max_length}) ||
5057                         &$safe_num_ne($drive->{bps_wr_max_length}, $old_drive->{bps_wr_max_length}) ||
5058                         &$safe_num_ne($drive->{iops_max_length}, $old_drive->{iops_max_length}) ||
5059                         &$safe_num_ne($drive->{iops_rd_max_length}, $old_drive->{iops_rd_max_length}) ||
5060                         &$safe_num_ne($drive->{iops_wr_max_length}, $old_drive->{iops_wr_max_length})) {
5061
5062                         qemu_block_set_io_throttle($vmid,"drive-$opt",
5063                                                    ($drive->{mbps} || 0)*1024*1024,
5064                                                    ($drive->{mbps_rd} || 0)*1024*1024,
5065                                                    ($drive->{mbps_wr} || 0)*1024*1024,
5066                                                    $drive->{iops} || 0,
5067                                                    $drive->{iops_rd} || 0,
5068                                                    $drive->{iops_wr} || 0,
5069                                                    ($drive->{mbps_max} || 0)*1024*1024,
5070                                                    ($drive->{mbps_rd_max} || 0)*1024*1024,
5071                                                    ($drive->{mbps_wr_max} || 0)*1024*1024,
5072                                                    $drive->{iops_max} || 0,
5073                                                    $drive->{iops_rd_max} || 0,
5074                                                    $drive->{iops_wr_max} || 0,
5075                                                    $drive->{bps_max_length} || 1,
5076                                                    $drive->{bps_rd_max_length} || 1,
5077                                                    $drive->{bps_wr_max_length} || 1,
5078                                                    $drive->{iops_max_length} || 1,
5079                                                    $drive->{iops_rd_max_length} || 1,
5080                                                    $drive->{iops_wr_max_length} || 1);
5081
5082                     }
5083
5084                     return 1;
5085                 }
5086
5087             } else { # cdrom
5088
5089                 if ($drive->{file} eq 'none') {
5090                     vm_mon_cmd($vmid, "eject",force => JSON::true,device => "drive-$opt");
5091                     if (drive_is_cloudinit($old_drive)) {
5092                         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive);
5093                     }
5094                 } else {
5095                     my $path = get_iso_path($storecfg, $vmid, $drive->{file});
5096                     vm_mon_cmd($vmid, "eject", force => JSON::true,device => "drive-$opt"); # force eject if locked
5097                     vm_mon_cmd($vmid, "change", device => "drive-$opt",target => "$path") if $path;
5098                 }
5099
5100                 return 1;
5101             }
5102         }
5103     }
5104
5105     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
5106     # hotplug new disks
5107     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
5108     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive, $arch, $machine_type);
5109 }
5110
5111 sub vm_start {
5112     my ($storecfg, $vmid, $statefile, $skiplock, $migratedfrom, $paused,
5113         $forcemachine, $spice_ticket, $migration_network, $migration_type, $targetstorage) = @_;
5114
5115     PVE::QemuConfig->lock_config($vmid, sub {
5116         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
5117
5118         die "you can't start a vm if it's a template\n" if PVE::QemuConfig->is_template($conf);
5119
5120         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5121
5122         die "VM $vmid already running\n" if check_running($vmid, undef, $migratedfrom);
5123
5124         if (!$statefile && scalar(keys %{$conf->{pending}})) {
5125             vmconfig_apply_pending($vmid, $conf, $storecfg);
5126             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5127         }
5128
5129         PVE::QemuServer::Cloudinit::generate_cloudinitconfig($conf, $vmid);
5130
5131         my $defaults = load_defaults();
5132
5133         # set environment variable useful inside network script
5134         $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
5135
5136         my $local_volumes = {};
5137
5138         if ($targetstorage) {
5139             foreach_drive($conf, sub {
5140                 my ($ds, $drive) = @_;
5141
5142                 return if drive_is_cdrom($drive);
5143
5144                 my $volid = $drive->{file};
5145
5146                 return if !$volid;
5147
5148                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
5149
5150                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5151                 return if $scfg->{shared};
5152                 $local_volumes->{$ds} = [$volid, $storeid, $volname];
5153             });
5154
5155             my $format = undef;
5156
5157             foreach my $opt (sort keys %$local_volumes) {
5158
5159                 my ($volid, $storeid, $volname) = @{$local_volumes->{$opt}};
5160                 my $drive = parse_drive($opt, $conf->{$opt});
5161
5162                 #if remote storage is specified, use default format
5163                 if ($targetstorage && $targetstorage ne "1") {
5164                     $storeid = $targetstorage;
5165                     my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
5166                     $format = $defFormat;
5167                 } else {
5168                     #else we use same format than original
5169                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5170                     $format = qemu_img_format($scfg, $volid);
5171                 }
5172
5173                 my $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $format, undef, ($drive->{size}/1024));
5174                 my $newdrive = $drive;
5175                 $newdrive->{format} = $format;
5176                 $newdrive->{file} = $newvolid;
5177                 my $drivestr = PVE::QemuServer::print_drive($vmid, $newdrive);
5178                 $local_volumes->{$opt} = $drivestr;
5179                 #pass drive to conf for command line
5180                 $conf->{$opt} = $drivestr;
5181             }
5182         }
5183
5184         PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-start', 1);
5185
5186         my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine);
5187
5188         my $migrate_port = 0;
5189         my $migrate_uri;
5190         if ($statefile) {
5191             if ($statefile eq 'tcp') {
5192                 my $localip = "localhost";
5193                 my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
5194                 my $nodename = PVE::INotify::nodename();
5195
5196                 if (!defined($migration_type)) {
5197                     if (defined($datacenterconf->{migration}->{type})) {
5198                         $migration_type = $datacenterconf->{migration}->{type};
5199                     } else {
5200                         $migration_type = 'secure';
5201                     }
5202                 }
5203
5204                 if ($migration_type eq 'insecure') {
5205                     my $migrate_network_addr = PVE::Cluster::get_local_migration_ip($migration_network);
5206                     if ($migrate_network_addr) {
5207                         $localip = $migrate_network_addr;
5208                     } else {
5209                         $localip = PVE::Cluster::remote_node_ip($nodename, 1);
5210                     }
5211
5212                     $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5213                 }
5214
5215                 my $pfamily = PVE::Tools::get_host_address_family($nodename);
5216                 $migrate_port = PVE::Tools::next_migrate_port($pfamily);
5217                 $migrate_uri = "tcp:${localip}:${migrate_port}";
5218                 push @$cmd, '-incoming', $migrate_uri;
5219                 push @$cmd, '-S';
5220
5221             } elsif ($statefile eq 'unix') {
5222                 # should be default for secure migrations as a ssh TCP forward
5223                 # tunnel is not deterministic reliable ready and fails regurarly
5224                 # to set up in time, so use UNIX socket forwards
5225                 my $socket_addr = "/run/qemu-server/$vmid.migrate";
5226                 unlink $socket_addr;
5227
5228                 $migrate_uri = "unix:$socket_addr";
5229
5230                 push @$cmd, '-incoming', $migrate_uri;
5231                 push @$cmd, '-S';
5232
5233             } else {
5234                 push @$cmd, '-loadstate', $statefile;
5235             }
5236         } elsif ($paused) {
5237             push @$cmd, '-S';
5238         }
5239
5240         # host pci devices
5241         for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
5242           my $d = parse_hostpci($conf->{"hostpci$i"});
5243           next if !$d;
5244           my $pcidevices = $d->{pciid};
5245           foreach my $pcidevice (@$pcidevices) {
5246                 my $pciid = $pcidevice->{id};
5247
5248                 my $info = PVE::SysFSTools::pci_device_info("0000:$pciid");
5249                 die "IOMMU not present\n" if !PVE::SysFSTools::check_iommu_support();
5250                 die "no pci device info for device '$pciid'\n" if !$info;
5251
5252                 if ($d->{mdev}) {
5253                     my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $i);
5254                     PVE::SysFSTools::pci_create_mdev_device($pciid, $uuid, $d->{mdev});
5255                 } else {
5256                     die "can't unbind/bind pci group to vfio '$pciid'\n"
5257                         if !PVE::SysFSTools::pci_dev_group_bind_to_vfio($pciid);
5258                     die "can't reset pci device '$pciid'\n"
5259                         if $info->{has_fl_reset} and !PVE::SysFSTools::pci_dev_reset($info);
5260                 }
5261           }
5262         }
5263
5264         PVE::Storage::activate_volumes($storecfg, $vollist);
5265
5266         if (-d "/sys/fs/cgroup/systemd/qemu.slice/$vmid.scope") {
5267             eval {
5268                 run_command(['/bin/systemctl', 'stop', "$vmid.scope"],
5269                     outfunc => sub {}, errfunc => sub {});
5270             };
5271         }
5272
5273         my $cpuunits = defined($conf->{cpuunits}) ? $conf->{cpuunits}
5274                                                   : $defaults->{cpuunits};
5275
5276         my $start_timeout = $conf->{hugepages} ? 300 : 30;
5277         my %run_params = (timeout => $statefile ? undef : $start_timeout, umask => 0077);
5278
5279         my %properties = (
5280             Slice => 'qemu.slice',
5281             KillMode => 'none',
5282             CPUShares => $cpuunits
5283         );
5284
5285         if (my $cpulimit = $conf->{cpulimit}) {
5286             $properties{CPUQuota} = int($cpulimit * 100);
5287         }
5288         $properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
5289
5290         my $run_qemu = sub {
5291             PVE::Tools::run_fork sub {
5292                 PVE::Systemd::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
5293                 run_command($cmd, %run_params);
5294             };
5295         };
5296
5297         if ($conf->{hugepages}) {
5298
5299             my $code = sub {
5300                 my $hugepages_topology = PVE::QemuServer::Memory::hugepages_topology($conf);
5301                 my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
5302
5303                 PVE::QemuServer::Memory::hugepages_mount();
5304                 PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
5305
5306                 eval { $run_qemu->() };
5307                 if (my $err = $@) {
5308                     PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology);
5309                     die $err;
5310                 }
5311
5312                 PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology);
5313             };
5314             eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
5315
5316         } else {
5317             eval { $run_qemu->() };
5318         }
5319
5320         if (my $err = $@) {
5321             # deactivate volumes if start fails
5322             eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
5323             die "start failed: $err";
5324         }
5325
5326         print "migration listens on $migrate_uri\n" if $migrate_uri;
5327
5328         if ($statefile && $statefile ne 'tcp' && $statefile ne 'unix')  {
5329             eval { vm_mon_cmd_nocheck($vmid, "cont"); };
5330             warn $@ if $@;
5331         }
5332
5333         #start nbd server for storage migration
5334         if ($targetstorage) {
5335             my $nodename = PVE::INotify::nodename();
5336             my $migrate_network_addr = PVE::Cluster::get_local_migration_ip($migration_network);
5337             my $localip = $migrate_network_addr ? $migrate_network_addr : PVE::Cluster::remote_node_ip($nodename, 1);
5338             my $pfamily = PVE::Tools::get_host_address_family($nodename);
5339             $migrate_port = PVE::Tools::next_migrate_port($pfamily);
5340
5341             vm_mon_cmd_nocheck($vmid, "nbd-server-start", addr => { type => 'inet', data => { host => "${localip}", port => "${migrate_port}" } } );
5342
5343             $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5344
5345             foreach my $opt (sort keys %$local_volumes) {
5346                 my $volid = $local_volumes->{$opt};
5347                 vm_mon_cmd_nocheck($vmid, "nbd-server-add", device => "drive-$opt", writable => JSON::true );
5348                 my $migrate_storage_uri = "nbd:${localip}:${migrate_port}:exportname=drive-$opt";
5349                 print "storage migration listens on $migrate_storage_uri volume:$volid\n";
5350             }
5351         }
5352
5353         if ($migratedfrom) {
5354             eval {
5355                 set_migration_caps($vmid);
5356             };
5357             warn $@ if $@;
5358
5359             if ($spice_port) {
5360                 print "spice listens on port $spice_port\n";
5361                 if ($spice_ticket) {
5362                     vm_mon_cmd_nocheck($vmid, "set_password", protocol => 'spice', password => $spice_ticket);
5363                     vm_mon_cmd_nocheck($vmid, "expire_password", protocol => 'spice', time => "+30");
5364                 }
5365             }
5366
5367         } else {
5368             vm_mon_cmd_nocheck($vmid, "balloon", value => $conf->{balloon}*1024*1024)
5369                 if !$statefile && $conf->{balloon};
5370
5371             foreach my $opt (keys %$conf) {
5372                 next if $opt !~  m/^net\d+$/;
5373                 my $nicconf = parse_net($conf->{$opt});
5374                 qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
5375             }
5376         }
5377
5378         vm_mon_cmd_nocheck($vmid, 'qom-set',
5379                     path => "machine/peripheral/balloon0",
5380                     property => "guest-stats-polling-interval",
5381                     value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
5382
5383         PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'post-start');
5384     });
5385 }
5386
5387 sub vm_mon_cmd {
5388     my ($vmid, $execute, %params) = @_;
5389
5390     my $cmd = { execute => $execute, arguments => \%params };
5391     vm_qmp_command($vmid, $cmd);
5392 }
5393
5394 sub vm_mon_cmd_nocheck {
5395     my ($vmid, $execute, %params) = @_;
5396
5397     my $cmd = { execute => $execute, arguments => \%params };
5398     vm_qmp_command($vmid, $cmd, 1);
5399 }
5400
5401 sub vm_qmp_command {
5402     my ($vmid, $cmd, $nocheck) = @_;
5403
5404     my $res;
5405
5406     my $timeout;
5407     if ($cmd->{arguments} && $cmd->{arguments}->{timeout}) {
5408         $timeout = $cmd->{arguments}->{timeout};
5409         delete $cmd->{arguments}->{timeout};
5410     }
5411
5412     eval {
5413         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
5414         my $sname = qmp_socket($vmid);
5415         if (-e $sname) { # test if VM is reasonambe new and supports qmp/qga
5416             my $qmpclient = PVE::QMPClient->new();
5417
5418             $res = $qmpclient->cmd($vmid, $cmd, $timeout);
5419         } else {
5420             die "unable to open monitor socket\n";
5421         }
5422     };
5423     if (my $err = $@) {
5424         syslog("err", "VM $vmid qmp command failed - $err");
5425         die $err;
5426     }
5427
5428     return $res;
5429 }
5430
5431 sub vm_human_monitor_command {
5432     my ($vmid, $cmdline) = @_;
5433
5434     my $res;
5435
5436     my $cmd = {
5437         execute => 'human-monitor-command',
5438         arguments => { 'command-line' => $cmdline},
5439     };
5440
5441     return vm_qmp_command($vmid, $cmd);
5442 }
5443
5444 sub vm_commandline {
5445     my ($storecfg, $vmid, $snapname) = @_;
5446
5447     my $conf = PVE::QemuConfig->load_config($vmid);
5448
5449     if ($snapname) {
5450         my $snapshot = $conf->{snapshots}->{$snapname};
5451         die "snapshot '$snapname' does not exist\n" if !defined($snapshot);
5452
5453         $snapshot->{digest} = $conf->{digest}; # keep file digest for API
5454
5455         $conf = $snapshot;
5456     }
5457
5458     my $defaults = load_defaults();
5459
5460     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults);
5461
5462     return PVE::Tools::cmd2string($cmd);
5463 }
5464
5465 sub vm_reset {
5466     my ($vmid, $skiplock) = @_;
5467
5468     PVE::QemuConfig->lock_config($vmid, sub {
5469
5470         my $conf = PVE::QemuConfig->load_config($vmid);
5471
5472         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5473
5474         vm_mon_cmd($vmid, "system_reset");
5475     });
5476 }
5477
5478 sub get_vm_volumes {
5479     my ($conf) = @_;
5480
5481     my $vollist = [];
5482     foreach_volid($conf, sub {
5483         my ($volid, $attr) = @_;
5484
5485         return if $volid =~ m|^/|;
5486
5487         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5488         return if !$sid;
5489
5490         push @$vollist, $volid;
5491     });
5492
5493     return $vollist;
5494 }
5495
5496 sub vm_stop_cleanup {
5497     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
5498
5499     eval {
5500
5501         if (!$keepActive) {
5502             my $vollist = get_vm_volumes($conf);
5503             PVE::Storage::deactivate_volumes($storecfg, $vollist);
5504         }
5505
5506         foreach my $ext (qw(mon qmp pid vnc qga)) {
5507             unlink "/var/run/qemu-server/${vmid}.$ext";
5508         }
5509
5510         if ($conf->{ivshmem}) {
5511             my $ivshmem = PVE::JSONSchema::parse_property_string($ivshmem_fmt, $conf->{ivshmem});
5512             # just delete it for now, VMs which have this already open do not
5513             # are affected, but new VMs will get a separated one. If this
5514             # becomes an issue we either add some sort of ref-counting or just
5515             # add a "don't delete on stop" flag to the ivshmem format.
5516             unlink '/dev/shm/pve-shm-' . ($ivshmem->{name} // $vmid);
5517         }
5518
5519         foreach my $key (keys %$conf) {
5520             next if $key !~ m/^hostpci(\d+)$/;
5521             my $hostpciindex = $1;
5522             my $d = parse_hostpci($conf->{$key});
5523             my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $hostpciindex);
5524
5525             foreach my $pci (@{$d->{pciid}}) {
5526                 my $pciid = $pci->{id};
5527                 PVE::SysFSTools::pci_cleanup_mdev_device($pciid, $uuid);
5528             }
5529         }
5530
5531         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
5532     };
5533     warn $@ if $@; # avoid errors - just warn
5534 }
5535
5536 # Note: use $nockeck to skip tests if VM configuration file exists.
5537 # We need that when migration VMs to other nodes (files already moved)
5538 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
5539 sub vm_stop {
5540     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
5541
5542     $force = 1 if !defined($force) && !$shutdown;
5543
5544     if ($migratedfrom){
5545         my $pid = check_running($vmid, $nocheck, $migratedfrom);
5546         kill 15, $pid if $pid;
5547         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
5548         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
5549         return;
5550     }
5551
5552     PVE::QemuConfig->lock_config($vmid, sub {
5553
5554         my $pid = check_running($vmid, $nocheck);
5555         return if !$pid;
5556
5557         my $conf;
5558         if (!$nocheck) {
5559             $conf = PVE::QemuConfig->load_config($vmid);
5560             PVE::QemuConfig->check_lock($conf) if !$skiplock;
5561             if (!defined($timeout) && $shutdown && $conf->{startup}) {
5562                 my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
5563                 $timeout = $opts->{down} if $opts->{down};
5564             }
5565             PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-stop');
5566         }
5567
5568         $timeout = 60 if !defined($timeout);
5569
5570         eval {
5571             if ($shutdown) {
5572                 if (defined($conf) && parse_guest_agent($conf)->{enabled}) {
5573                     vm_qmp_command($vmid, { execute => "guest-shutdown" }, $nocheck);
5574                 } else {
5575                     vm_qmp_command($vmid, { execute => "system_powerdown" }, $nocheck);
5576                 }
5577             } else {
5578                 vm_qmp_command($vmid, { execute => "quit" }, $nocheck);
5579             }
5580         };
5581         my $err = $@;
5582
5583         if (!$err) {
5584             my $count = 0;
5585             while (($count < $timeout) && check_running($vmid, $nocheck)) {
5586                 $count++;
5587                 sleep 1;
5588             }
5589
5590             if ($count >= $timeout) {
5591                 if ($force) {
5592                     warn "VM still running - terminating now with SIGTERM\n";
5593                     kill 15, $pid;
5594                 } else {
5595                     die "VM quit/powerdown failed - got timeout\n";
5596                 }
5597             } else {
5598                 vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5599                 return;
5600             }
5601         } else {
5602             if ($force) {
5603                 warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
5604                 kill 15, $pid;
5605             } else {
5606                 die "VM quit/powerdown failed\n";
5607             }
5608         }
5609
5610         # wait again
5611         $timeout = 10;
5612
5613         my $count = 0;
5614         while (($count < $timeout) && check_running($vmid, $nocheck)) {
5615             $count++;
5616             sleep 1;
5617         }
5618
5619         if ($count >= $timeout) {
5620             warn "VM still running - terminating now with SIGKILL\n";
5621             kill 9, $pid;
5622             sleep 1;
5623         }
5624
5625         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5626    });
5627 }
5628
5629 sub vm_suspend {
5630     my ($vmid, $skiplock) = @_;
5631
5632     PVE::QemuConfig->lock_config($vmid, sub {
5633
5634         my $conf = PVE::QemuConfig->load_config($vmid);
5635
5636         PVE::QemuConfig->check_lock($conf)
5637             if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
5638
5639         vm_mon_cmd($vmid, "stop");
5640     });
5641 }
5642
5643 sub vm_resume {
5644     my ($vmid, $skiplock, $nocheck) = @_;
5645
5646     PVE::QemuConfig->lock_config($vmid, sub {
5647
5648         my $res = vm_mon_cmd($vmid, 'query-status');
5649         my $resume_cmd = 'cont';
5650
5651         if ($res->{status} && $res->{status} eq 'suspended') {
5652             $resume_cmd = 'system_wakeup';
5653         }
5654
5655         if (!$nocheck) {
5656
5657             my $conf = PVE::QemuConfig->load_config($vmid);
5658
5659             PVE::QemuConfig->check_lock($conf)
5660                 if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
5661
5662             vm_mon_cmd($vmid, $resume_cmd);
5663
5664         } else {
5665             vm_mon_cmd_nocheck($vmid, $resume_cmd);
5666         }
5667     });
5668 }
5669
5670 sub vm_sendkey {
5671     my ($vmid, $skiplock, $key) = @_;
5672
5673     PVE::QemuConfig->lock_config($vmid, sub {
5674
5675         my $conf = PVE::QemuConfig->load_config($vmid);
5676
5677         # there is no qmp command, so we use the human monitor command
5678         vm_human_monitor_command($vmid, "sendkey $key");
5679     });
5680 }
5681
5682 sub vm_destroy {
5683     my ($storecfg, $vmid, $skiplock) = @_;
5684
5685     PVE::QemuConfig->lock_config($vmid, sub {
5686
5687         my $conf = PVE::QemuConfig->load_config($vmid);
5688
5689         if (!check_running($vmid)) {
5690             destroy_vm($storecfg, $vmid, undef, $skiplock);
5691         } else {
5692             die "VM $vmid is running - destroy failed\n";
5693         }
5694     });
5695 }
5696
5697 # vzdump restore implementaion
5698
5699 sub tar_archive_read_firstfile {
5700     my $archive = shift;
5701
5702     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
5703
5704     # try to detect archive type first
5705     my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
5706         die "unable to open file '$archive'\n";
5707     my $firstfile = <$fh>;
5708     kill 15, $pid;
5709     close $fh;
5710
5711     die "ERROR: archive contaions no data\n" if !$firstfile;
5712     chomp $firstfile;
5713
5714     return $firstfile;
5715 }
5716
5717 sub tar_restore_cleanup {
5718     my ($storecfg, $statfile) = @_;
5719
5720     print STDERR "starting cleanup\n";
5721
5722     if (my $fd = IO::File->new($statfile, "r")) {
5723         while (defined(my $line = <$fd>)) {
5724             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5725                 my $volid = $2;
5726                 eval {
5727                     if ($volid =~ m|^/|) {
5728                         unlink $volid || die 'unlink failed\n';
5729                     } else {
5730                         PVE::Storage::vdisk_free($storecfg, $volid);
5731                     }
5732                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
5733                 };
5734                 print STDERR "unable to cleanup '$volid' - $@" if $@;
5735             } else {
5736                 print STDERR "unable to parse line in statfile - $line";
5737             }
5738         }
5739         $fd->close();
5740     }
5741 }
5742
5743 sub restore_archive {
5744     my ($archive, $vmid, $user, $opts) = @_;
5745
5746     my $format = $opts->{format};
5747     my $comp;
5748
5749     if ($archive =~ m/\.tgz$/ || $archive =~ m/\.tar\.gz$/) {
5750         $format = 'tar' if !$format;
5751         $comp = 'gzip';
5752     } elsif ($archive =~ m/\.tar$/) {
5753         $format = 'tar' if !$format;
5754     } elsif ($archive =~ m/.tar.lzo$/) {
5755         $format = 'tar' if !$format;
5756         $comp = 'lzop';
5757     } elsif ($archive =~ m/\.vma$/) {
5758         $format = 'vma' if !$format;
5759     } elsif ($archive =~ m/\.vma\.gz$/) {
5760         $format = 'vma' if !$format;
5761         $comp = 'gzip';
5762     } elsif ($archive =~ m/\.vma\.lzo$/) {
5763         $format = 'vma' if !$format;
5764         $comp = 'lzop';
5765     } else {
5766         $format = 'vma' if !$format; # default
5767     }
5768
5769     # try to detect archive format
5770     if ($format eq 'tar') {
5771         return restore_tar_archive($archive, $vmid, $user, $opts);
5772     } else {
5773         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
5774     }
5775 }
5776
5777 sub restore_update_config_line {
5778     my ($outfd, $cookie, $vmid, $map, $line, $unique) = @_;
5779
5780     return if $line =~ m/^\#qmdump\#/;
5781     return if $line =~ m/^\#vzdump\#/;
5782     return if $line =~ m/^lock:/;
5783     return if $line =~ m/^unused\d+:/;
5784     return if $line =~ m/^parent:/;
5785     return if $line =~ m/^template:/; # restored VM is never a template
5786
5787     my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
5788     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
5789         # try to convert old 1.X settings
5790         my ($id, $ind, $ethcfg) = ($1, $2, $3);
5791         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
5792             my ($model, $macaddr) = split(/\=/, $devconfig);
5793             $macaddr = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if !$macaddr || $unique;
5794             my $net = {
5795                 model => $model,
5796                 bridge => "vmbr$ind",
5797                 macaddr => $macaddr,
5798             };
5799             my $netstr = print_net($net);
5800
5801             print $outfd "net$cookie->{netcount}: $netstr\n";
5802             $cookie->{netcount}++;
5803         }
5804     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
5805         my ($id, $netstr) = ($1, $2);
5806         my $net = parse_net($netstr);
5807         $net->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if $net->{macaddr};
5808         $netstr = print_net($net);
5809         print $outfd "$id: $netstr\n";
5810     } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk)\d+):\s*(\S+)\s*$/) {
5811         my $virtdev = $1;
5812         my $value = $3;
5813         my $di = parse_drive($virtdev, $value);
5814         if (defined($di->{backup}) && !$di->{backup}) {
5815             print $outfd "#$line";
5816         } elsif ($map->{$virtdev}) {
5817             delete $di->{format}; # format can change on restore
5818             $di->{file} = $map->{$virtdev};
5819             $value = print_drive($vmid, $di);
5820             print $outfd "$virtdev: $value\n";
5821         } else {
5822             print $outfd $line;
5823         }
5824     } elsif (($line =~ m/^vmgenid: (.*)/)) {
5825         my $vmgenid = $1;
5826         if ($vmgenid ne '0') {
5827             # always generate a new vmgenid if there was a valid one setup
5828             $vmgenid = generate_uuid();
5829         }
5830         print $outfd "vmgenid: $vmgenid\n";
5831     } elsif (($line =~ m/^(smbios1: )(.*)/) && $unique) {
5832         my ($uuid, $uuid_str);
5833         UUID::generate($uuid);
5834         UUID::unparse($uuid, $uuid_str);
5835         my $smbios1 = parse_smbios1($2);
5836         $smbios1->{uuid} = $uuid_str;
5837         print $outfd $1.print_smbios1($smbios1)."\n";
5838     } else {
5839         print $outfd $line;
5840     }
5841 }
5842
5843 sub scan_volids {
5844     my ($cfg, $vmid) = @_;
5845
5846     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid);
5847
5848     my $volid_hash = {};
5849     foreach my $storeid (keys %$info) {
5850         foreach my $item (@{$info->{$storeid}}) {
5851             next if !($item->{volid} && $item->{size});
5852             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
5853             $volid_hash->{$item->{volid}} = $item;
5854         }
5855     }
5856
5857     return $volid_hash;
5858 }
5859
5860 sub is_volume_in_use {
5861     my ($storecfg, $conf, $skip_drive, $volid) = @_;
5862
5863     my $path = PVE::Storage::path($storecfg, $volid);
5864
5865     my $scan_config = sub {
5866         my ($cref, $snapname) = @_;
5867
5868         foreach my $key (keys %$cref) {
5869             my $value = $cref->{$key};
5870             if (is_valid_drivename($key)) {
5871                 next if $skip_drive && $key eq $skip_drive;
5872                 my $drive = parse_drive($key, $value);
5873                 next if !$drive || !$drive->{file} || drive_is_cdrom($drive);
5874                 return 1 if $volid eq $drive->{file};
5875                 if ($drive->{file} =~ m!^/!) {
5876                     return 1 if $drive->{file} eq $path;
5877                 } else {
5878                     my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file}, 1);
5879                     next if !$storeid;
5880                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid, 1);
5881                     next if !$scfg;
5882                     return 1 if $path eq PVE::Storage::path($storecfg, $drive->{file}, $snapname);
5883                 }
5884             }
5885         }
5886
5887         return 0;
5888     };
5889
5890     return 1 if &$scan_config($conf);
5891
5892     undef $skip_drive;
5893
5894     foreach my $snapname (keys %{$conf->{snapshots}}) {
5895         return 1 if &$scan_config($conf->{snapshots}->{$snapname}, $snapname);
5896     }
5897
5898     return 0;
5899 }
5900
5901 sub update_disksize {
5902     my ($vmid, $conf, $volid_hash) = @_;
5903
5904     my $changes;
5905     my $prefix = "VM $vmid:";
5906
5907     # used and unused disks
5908     my $referenced = {};
5909
5910     # Note: it is allowed to define multiple storages with same path (alias), so
5911     # we need to check both 'volid' and real 'path' (two different volid can point
5912     # to the same path).
5913
5914     my $referencedpath = {};
5915
5916     # update size info
5917     foreach my $opt (keys %$conf) {
5918         if (is_valid_drivename($opt)) {
5919             my $drive = parse_drive($opt, $conf->{$opt});
5920             my $volid = $drive->{file};
5921             next if !$volid;
5922
5923             $referenced->{$volid} = 1;
5924             if ($volid_hash->{$volid} &&
5925                 (my $path = $volid_hash->{$volid}->{path})) {
5926                 $referencedpath->{$path} = 1;
5927             }
5928
5929             next if drive_is_cdrom($drive);
5930             next if !$volid_hash->{$volid};
5931
5932             $drive->{size} = $volid_hash->{$volid}->{size};
5933             my $new = print_drive($vmid, $drive);
5934             if ($new ne $conf->{$opt}) {
5935                 $changes = 1;
5936                 $conf->{$opt} = $new;
5937                 print "$prefix update disk '$opt' information.\n";
5938             }
5939         }
5940     }
5941
5942     # remove 'unusedX' entry if volume is used
5943     foreach my $opt (keys %$conf) {
5944         next if $opt !~ m/^unused\d+$/;
5945         my $volid = $conf->{$opt};
5946         my $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
5947         if ($referenced->{$volid} || ($path && $referencedpath->{$path})) {
5948             print "$prefix remove entry '$opt', its volume '$volid' is in use.\n";
5949             $changes = 1;
5950             delete $conf->{$opt};
5951         }
5952
5953         $referenced->{$volid} = 1;
5954         $referencedpath->{$path} = 1 if $path;
5955     }
5956
5957     foreach my $volid (sort keys %$volid_hash) {
5958         next if $volid =~ m/vm-$vmid-state-/;
5959         next if $referenced->{$volid};
5960         my $path = $volid_hash->{$volid}->{path};
5961         next if !$path; # just to be sure
5962         next if $referencedpath->{$path};
5963         $changes = 1;
5964         my $key = PVE::QemuConfig->add_unused_volume($conf, $volid);
5965         print "$prefix add unreferenced volume '$volid' as '$key' to config.\n";
5966         $referencedpath->{$path} = 1; # avoid to add more than once (aliases)
5967     }
5968
5969     return $changes;
5970 }
5971
5972 sub rescan {
5973     my ($vmid, $nolock, $dryrun) = @_;
5974
5975     my $cfg = PVE::Storage::config();
5976
5977     # FIXME: Remove once our RBD plugin can handle CT and VM on a single storage
5978     # see: https://pve.proxmox.com/pipermail/pve-devel/2018-July/032900.html
5979     foreach my $stor (keys %{$cfg->{ids}}) {
5980         delete($cfg->{ids}->{$stor}) if ! $cfg->{ids}->{$stor}->{content}->{images};
5981     }
5982
5983     print "rescan volumes...\n";
5984     my $volid_hash = scan_volids($cfg, $vmid);
5985
5986     my $updatefn =  sub {
5987         my ($vmid) = @_;
5988
5989         my $conf = PVE::QemuConfig->load_config($vmid);
5990
5991         PVE::QemuConfig->check_lock($conf);
5992
5993         my $vm_volids = {};
5994         foreach my $volid (keys %$volid_hash) {
5995             my $info = $volid_hash->{$volid};
5996             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
5997         }
5998
5999         my $changes = update_disksize($vmid, $conf, $vm_volids);
6000
6001         PVE::QemuConfig->write_config($vmid, $conf) if $changes && !$dryrun;
6002     };
6003
6004     if (defined($vmid)) {
6005         if ($nolock) {
6006             &$updatefn($vmid);
6007         } else {
6008             PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6009         }
6010     } else {
6011         my $vmlist = config_list();
6012         foreach my $vmid (keys %$vmlist) {
6013             if ($nolock) {
6014                 &$updatefn($vmid);
6015             } else {
6016                 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6017             }
6018         }
6019     }
6020 }
6021
6022 sub restore_vma_archive {
6023     my ($archive, $vmid, $user, $opts, $comp) = @_;
6024
6025     my $readfrom = $archive;
6026
6027     my $cfg = PVE::Storage::config();
6028     my $commands = [];
6029     my $bwlimit = $opts->{bwlimit};
6030
6031     my $dbg_cmdstring = '';
6032     my $add_pipe = sub {
6033         my ($cmd) = @_;
6034         push @$commands, $cmd;
6035         $dbg_cmdstring .= ' | ' if length($dbg_cmdstring);
6036         $dbg_cmdstring .= PVE::Tools::cmd2string($cmd);
6037         $readfrom = '-';
6038     };
6039
6040     my $input = undef;
6041     if ($archive eq '-') {
6042         $input = '<&STDIN';
6043     } else {
6044         # If we use a backup from a PVE defined storage we also consider that
6045         # storage's rate limit:
6046         my (undef, $volid) = PVE::Storage::path_to_volume_id($cfg, $archive);
6047         if (defined($volid)) {
6048             my ($sid, undef) = PVE::Storage::parse_volume_id($volid);
6049             my $readlimit = PVE::Storage::get_bandwidth_limit('restore', [$sid], $bwlimit);
6050             if ($readlimit) {
6051                 print STDERR "applying read rate limit: $readlimit\n";
6052                 my $cstream = ['cstream', '-t', $readlimit*1024, '--', $readfrom];
6053                 $add_pipe->($cstream);
6054             }
6055         }
6056     }
6057
6058     if ($comp) {
6059         my $cmd;
6060         if ($comp eq 'gzip') {
6061             $cmd = ['zcat', $readfrom];
6062         } elsif ($comp eq 'lzop') {
6063             $cmd = ['lzop', '-d', '-c', $readfrom];
6064         } else {
6065             die "unknown compression method '$comp'\n";
6066         }
6067         $add_pipe->($cmd);
6068     }
6069
6070     my $tmpdir = "/var/tmp/vzdumptmp$$";
6071     rmtree $tmpdir;
6072
6073     # disable interrupts (always do cleanups)
6074     local $SIG{INT} =
6075         local $SIG{TERM} =
6076         local $SIG{QUIT} =
6077         local $SIG{HUP} = sub { warn "got interrupt - ignored\n"; };
6078
6079     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
6080     POSIX::mkfifo($mapfifo, 0600);
6081     my $fifofh;
6082
6083     my $openfifo = sub {
6084         open($fifofh, '>', $mapfifo) || die $!;
6085     };
6086
6087     $add_pipe->(['vma', 'extract', '-v', '-r', $mapfifo, $readfrom, $tmpdir]);
6088
6089     my $oldtimeout;
6090     my $timeout = 5;
6091
6092     my $devinfo = {};
6093
6094     my $rpcenv = PVE::RPCEnvironment::get();
6095
6096     my $conffile = PVE::QemuConfig->config_file($vmid);
6097     my $tmpfn = "$conffile.$$.tmp";
6098
6099     # Note: $oldconf is undef if VM does not exists
6100     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
6101     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
6102
6103     my %storage_limits;
6104
6105     my $print_devmap = sub {
6106         my $virtdev_hash = {};
6107
6108         my $cfgfn = "$tmpdir/qemu-server.conf";
6109
6110         # we can read the config - that is already extracted
6111         my $fh = IO::File->new($cfgfn, "r") ||
6112             "unable to read qemu-server.conf - $!\n";
6113
6114         my $fwcfgfn = "$tmpdir/qemu-server.fw";
6115         if (-f $fwcfgfn) {
6116             my $pve_firewall_dir = '/etc/pve/firewall';
6117             mkdir $pve_firewall_dir; # make sure the dir exists
6118             PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
6119         }
6120
6121         while (defined(my $line = <$fh>)) {
6122             if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
6123                 my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
6124                 die "archive does not contain data for drive '$virtdev'\n"
6125                     if !$devinfo->{$devname};
6126                 if (defined($opts->{storage})) {
6127                     $storeid = $opts->{storage} || 'local';
6128                 } elsif (!$storeid) {
6129                     $storeid = 'local';
6130                 }
6131                 $format = 'raw' if !$format;
6132                 $devinfo->{$devname}->{devname} = $devname;
6133                 $devinfo->{$devname}->{virtdev} = $virtdev;
6134                 $devinfo->{$devname}->{format} = $format;
6135                 $devinfo->{$devname}->{storeid} = $storeid;
6136
6137                 # check permission on storage
6138                 my $pool = $opts->{pool}; # todo: do we need that?
6139                 if ($user ne 'root@pam') {
6140                     $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
6141                 }
6142
6143                 $storage_limits{$storeid} = $bwlimit;
6144
6145                 $virtdev_hash->{$virtdev} = $devinfo->{$devname};
6146             }
6147         }
6148
6149         foreach my $key (keys %storage_limits) {
6150             my $limit = PVE::Storage::get_bandwidth_limit('restore', [$key], $bwlimit);
6151             next if !$limit;
6152             print STDERR "rate limit for storage $key: $limit KiB/s\n";
6153             $storage_limits{$key} = $limit * 1024;
6154         }
6155
6156         foreach my $devname (keys %$devinfo) {
6157             die "found no device mapping information for device '$devname'\n"
6158                 if !$devinfo->{$devname}->{virtdev};
6159         }
6160
6161         # create empty/temp config
6162         if ($oldconf) {
6163             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
6164             foreach_drive($oldconf, sub {
6165                 my ($ds, $drive) = @_;
6166
6167                 return if drive_is_cdrom($drive);
6168
6169                 my $volid = $drive->{file};
6170
6171                 return if !$volid || $volid =~ m|^/|;
6172
6173                 my ($path, $owner) = PVE::Storage::path($cfg, $volid);
6174                 return if !$path || !$owner || ($owner != $vmid);
6175
6176                 # Note: only delete disk we want to restore
6177                 # other volumes will become unused
6178                 if ($virtdev_hash->{$ds}) {
6179                     eval { PVE::Storage::vdisk_free($cfg, $volid); };
6180                     if (my $err = $@) {
6181                         warn $err;
6182                     }
6183                 }
6184             });
6185
6186             # delete vmstate files
6187             # since after the restore we have no snapshots anymore
6188             foreach my $snapname (keys %{$oldconf->{snapshots}}) {
6189                 my $snap = $oldconf->{snapshots}->{$snapname};
6190                 if ($snap->{vmstate}) {
6191                     eval { PVE::Storage::vdisk_free($cfg, $snap->{vmstate}); };
6192                     if (my $err = $@) {
6193                         warn $err;
6194                     }
6195                 }
6196             }
6197         }
6198
6199         my $map = {};
6200         foreach my $virtdev (sort keys %$virtdev_hash) {
6201             my $d = $virtdev_hash->{$virtdev};
6202             my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
6203             my $storeid = $d->{storeid};
6204             my $scfg = PVE::Storage::storage_config($cfg, $storeid);
6205
6206             my $map_opts = '';
6207             if (my $limit = $storage_limits{$storeid}) {
6208                 $map_opts .= "throttling.bps=$limit:throttling.group=$storeid:";
6209             }
6210
6211             # test if requested format is supported
6212             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($cfg, $storeid);
6213             my $supported = grep { $_ eq $d->{format} } @$validFormats;
6214             $d->{format} = $defFormat if !$supported;
6215
6216             my $volid = PVE::Storage::vdisk_alloc($cfg, $storeid, $vmid,
6217                                                   $d->{format}, undef, $alloc_size);
6218             print STDERR "new volume ID is '$volid'\n";
6219             $d->{volid} = $volid;
6220             my $path = PVE::Storage::path($cfg, $volid);
6221
6222             PVE::Storage::activate_volumes($cfg,[$volid]);
6223
6224             my $write_zeros = 1;
6225             if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
6226                 $write_zeros = 0;
6227             }
6228
6229             print $fifofh "${map_opts}format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
6230
6231             print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
6232             $map->{$virtdev} = $volid;
6233         }
6234
6235         $fh->seek(0, 0) || die "seek failed - $!\n";
6236
6237         my $outfd = new IO::File ($tmpfn, "w") ||
6238             die "unable to write config for VM $vmid\n";
6239
6240         my $cookie = { netcount => 0 };
6241         while (defined(my $line = <$fh>)) {
6242             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
6243         }
6244
6245         $fh->close();
6246         $outfd->close();
6247     };
6248
6249     eval {
6250         # enable interrupts
6251         local $SIG{INT} =
6252             local $SIG{TERM} =
6253             local $SIG{QUIT} =
6254             local $SIG{HUP} =
6255             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6256         local $SIG{ALRM} = sub { die "got timeout\n"; };
6257
6258         $oldtimeout = alarm($timeout);
6259
6260         my $parser = sub {
6261             my $line = shift;
6262
6263             print "$line\n";
6264
6265             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
6266                 my ($dev_id, $size, $devname) = ($1, $2, $3);
6267                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
6268             } elsif ($line =~ m/^CTIME: /) {
6269                 # we correctly received the vma config, so we can disable
6270                 # the timeout now for disk allocation (set to 10 minutes, so
6271                 # that we always timeout if something goes wrong)
6272                 alarm(600);
6273                 &$print_devmap();
6274                 print $fifofh "done\n";
6275                 my $tmp = $oldtimeout || 0;
6276                 $oldtimeout = undef;
6277                 alarm($tmp);
6278                 close($fifofh);
6279             }
6280         };
6281
6282         print "restore vma archive: $dbg_cmdstring\n";
6283         run_command($commands, input => $input, outfunc => $parser, afterfork => $openfifo);
6284     };
6285     my $err = $@;
6286
6287     alarm($oldtimeout) if $oldtimeout;
6288
6289     my $vollist = [];
6290     foreach my $devname (keys %$devinfo) {
6291         my $volid = $devinfo->{$devname}->{volid};
6292         push @$vollist, $volid if $volid;
6293     }
6294
6295     PVE::Storage::deactivate_volumes($cfg, $vollist);
6296
6297     unlink $mapfifo;
6298
6299     if ($err) {
6300         rmtree $tmpdir;
6301         unlink $tmpfn;
6302
6303         foreach my $devname (keys %$devinfo) {
6304             my $volid = $devinfo->{$devname}->{volid};
6305             next if !$volid;
6306             eval {
6307                 if ($volid =~ m|^/|) {
6308                     unlink $volid || die 'unlink failed\n';
6309                 } else {
6310                     PVE::Storage::vdisk_free($cfg, $volid);
6311                 }
6312                 print STDERR "temporary volume '$volid' sucessfuly removed\n";
6313             };
6314             print STDERR "unable to cleanup '$volid' - $@" if $@;
6315         }
6316         die $err;
6317     }
6318
6319     rmtree $tmpdir;
6320
6321     rename($tmpfn, $conffile) ||
6322         die "unable to commit configuration file '$conffile'\n";
6323
6324     PVE::Cluster::cfs_update(); # make sure we read new file
6325
6326     eval { rescan($vmid, 1); };
6327     warn $@ if $@;
6328 }
6329
6330 sub restore_tar_archive {
6331     my ($archive, $vmid, $user, $opts) = @_;
6332
6333     if ($archive ne '-') {
6334         my $firstfile = tar_archive_read_firstfile($archive);
6335         die "ERROR: file '$archive' dos not lock like a QemuServer vzdump backup\n"
6336             if $firstfile ne 'qemu-server.conf';
6337     }
6338
6339     my $storecfg = PVE::Storage::config();
6340
6341     # destroy existing data - keep empty config
6342     my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
6343     destroy_vm($storecfg, $vmid, 1) if -f $vmcfgfn;
6344
6345     my $tocmd = "/usr/lib/qemu-server/qmextract";
6346
6347     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
6348     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
6349     $tocmd .= ' --prealloc' if $opts->{prealloc};
6350     $tocmd .= ' --info' if $opts->{info};
6351
6352     # tar option "xf" does not autodetect compression when read from STDIN,
6353     # so we pipe to zcat
6354     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
6355         PVE::Tools::shellquote("--to-command=$tocmd");
6356
6357     my $tmpdir = "/var/tmp/vzdumptmp$$";
6358     mkpath $tmpdir;
6359
6360     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
6361     local $ENV{VZDUMP_VMID} = $vmid;
6362     local $ENV{VZDUMP_USER} = $user;
6363
6364     my $conffile = PVE::QemuConfig->config_file($vmid);
6365     my $tmpfn = "$conffile.$$.tmp";
6366
6367     # disable interrupts (always do cleanups)
6368     local $SIG{INT} =
6369         local $SIG{TERM} =
6370         local $SIG{QUIT} =
6371         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
6372
6373     eval {
6374         # enable interrupts
6375         local $SIG{INT} =
6376             local $SIG{TERM} =
6377             local $SIG{QUIT} =
6378             local $SIG{HUP} =
6379             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6380
6381         if ($archive eq '-') {
6382             print "extracting archive from STDIN\n";
6383             run_command($cmd, input => "<&STDIN");
6384         } else {
6385             print "extracting archive '$archive'\n";
6386             run_command($cmd);
6387         }
6388
6389         return if $opts->{info};
6390
6391         # read new mapping
6392         my $map = {};
6393         my $statfile = "$tmpdir/qmrestore.stat";
6394         if (my $fd = IO::File->new($statfile, "r")) {
6395             while (defined (my $line = <$fd>)) {
6396                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
6397                     $map->{$1} = $2 if $1;
6398                 } else {
6399                     print STDERR "unable to parse line in statfile - $line\n";
6400                 }
6401             }
6402             $fd->close();
6403         }
6404
6405         my $confsrc = "$tmpdir/qemu-server.conf";
6406
6407         my $srcfd = new IO::File($confsrc, "r") ||
6408             die "unable to open file '$confsrc'\n";
6409
6410         my $outfd = new IO::File ($tmpfn, "w") ||
6411             die "unable to write config for VM $vmid\n";
6412
6413         my $cookie = { netcount => 0 };
6414         while (defined (my $line = <$srcfd>)) {
6415             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
6416         }
6417
6418         $srcfd->close();
6419         $outfd->close();
6420     };
6421     my $err = $@;
6422
6423     if ($err) {
6424
6425         unlink $tmpfn;
6426
6427         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
6428
6429         die $err;
6430     }
6431
6432     rmtree $tmpdir;
6433
6434     rename $tmpfn, $conffile ||
6435         die "unable to commit configuration file '$conffile'\n";
6436
6437     PVE::Cluster::cfs_update(); # make sure we read new file
6438
6439     eval { rescan($vmid, 1); };
6440     warn $@ if $@;
6441 };
6442
6443 sub foreach_storage_used_by_vm {
6444     my ($conf, $func) = @_;
6445
6446     my $sidhash = {};
6447
6448     foreach_drive($conf, sub {
6449         my ($ds, $drive) = @_;
6450         return if drive_is_cdrom($drive);
6451
6452         my $volid = $drive->{file};
6453
6454         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
6455         $sidhash->{$sid} = $sid if $sid;
6456     });
6457
6458     foreach my $sid (sort keys %$sidhash) {
6459         &$func($sid);
6460     }
6461 }
6462
6463 sub do_snapshots_with_qemu {
6464     my ($storecfg, $volid) = @_;
6465
6466     my $storage_name = PVE::Storage::parse_volume_id($volid);
6467
6468     if ($qemu_snap_storage->{$storecfg->{ids}->{$storage_name}->{type}}
6469         && !$storecfg->{ids}->{$storage_name}->{krbd}){
6470         return 1;
6471     }
6472
6473     if ($volid =~ m/\.(qcow2|qed)$/){
6474         return 1;
6475     }
6476
6477     return undef;
6478 }
6479
6480 sub qga_check_running {
6481     my ($vmid, $nowarn) = @_;
6482
6483     eval { vm_mon_cmd($vmid, "guest-ping", timeout => 3); };
6484     if ($@) {
6485         warn "Qemu Guest Agent is not running - $@" if !$nowarn;
6486         return 0;
6487     }
6488     return 1;
6489 }
6490
6491 sub template_create {
6492     my ($vmid, $conf, $disk) = @_;
6493
6494     my $storecfg = PVE::Storage::config();
6495
6496     foreach_drive($conf, sub {
6497         my ($ds, $drive) = @_;
6498
6499         return if drive_is_cdrom($drive);
6500         return if $disk && $ds ne $disk;
6501
6502         my $volid = $drive->{file};
6503         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
6504
6505         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
6506         $drive->{file} = $voliddst;
6507         $conf->{$ds} = print_drive($vmid, $drive);
6508         PVE::QemuConfig->write_config($vmid, $conf);
6509     });
6510 }
6511
6512 sub qemu_img_convert {
6513     my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized) = @_;
6514
6515     my $storecfg = PVE::Storage::config();
6516     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
6517     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
6518
6519     if ($src_storeid && $dst_storeid) {
6520
6521         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
6522
6523         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
6524         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6525
6526         my $src_format = qemu_img_format($src_scfg, $src_volname);
6527         my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
6528
6529         my $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
6530         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6531
6532         my $cmd = [];
6533         push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
6534         push @$cmd, '-l', "snapshot.name=$snapname" if($snapname && $src_format eq "qcow2");
6535         push @$cmd, '-t', 'none' if $dst_scfg->{type} eq 'zfspool';
6536         push @$cmd, '-T', 'none' if $src_scfg->{type} eq 'zfspool';
6537         push @$cmd, '-f', $src_format, '-O', $dst_format, $src_path;
6538         if ($is_zero_initialized) {
6539             push @$cmd, "zeroinit:$dst_path";
6540         } else {
6541             push @$cmd, $dst_path;
6542         }
6543
6544         my $parser = sub {
6545             my $line = shift;
6546             if($line =~ m/\((\S+)\/100\%\)/){
6547                 my $percent = $1;
6548                 my $transferred = int($size * $percent / 100);
6549                 my $remaining = $size - $transferred;
6550
6551                 print "transferred: $transferred bytes remaining: $remaining bytes total: $size bytes progression: $percent %\n";
6552             }
6553
6554         };
6555
6556         eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
6557         my $err = $@;
6558         die "copy failed: $err" if $err;
6559     }
6560 }
6561
6562 sub qemu_img_format {
6563     my ($scfg, $volname) = @_;
6564
6565     if ($scfg->{path} && $volname =~ m/\.($QEMU_FORMAT_RE)$/) {
6566         return $1;
6567     } else {
6568         return "raw";
6569     }
6570 }
6571
6572 sub qemu_drive_mirror {
6573     my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $skipcomplete, $qga) = @_;
6574
6575     $jobs = {} if !$jobs;
6576
6577     my $qemu_target;
6578     my $format;
6579     $jobs->{"drive-$drive"} = {};
6580
6581     if ($dst_volid =~ /^nbd:/) {
6582         $qemu_target = $dst_volid;
6583         $format = "nbd";
6584     } else {
6585         my $storecfg = PVE::Storage::config();
6586         my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
6587
6588         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6589
6590         $format = qemu_img_format($dst_scfg, $dst_volname);
6591
6592         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6593
6594         $qemu_target = $is_zero_initialized ? "zeroinit:$dst_path" : $dst_path;
6595     }
6596
6597     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $qemu_target };
6598     $opts->{format} = $format if $format;
6599
6600     print "drive mirror is starting for drive-$drive\n";
6601
6602     eval { vm_mon_cmd($vmid, "drive-mirror", %$opts); }; #if a job already run for this device,it's throw an error
6603
6604     if (my $err = $@) {
6605         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6606         die "mirroring error: $err";
6607     }
6608
6609     qemu_drive_mirror_monitor ($vmid, $vmiddst, $jobs, $skipcomplete, $qga);
6610 }
6611
6612 sub qemu_drive_mirror_monitor {
6613     my ($vmid, $vmiddst, $jobs, $skipcomplete, $qga) = @_;
6614
6615     eval {
6616         my $err_complete = 0;
6617
6618         while (1) {
6619             die "storage migration timed out\n" if $err_complete > 300;
6620
6621             my $stats = vm_mon_cmd($vmid, "query-block-jobs");
6622
6623             my $running_mirror_jobs = {};
6624             foreach my $stat (@$stats) {
6625                 next if $stat->{type} ne 'mirror';
6626                 $running_mirror_jobs->{$stat->{device}} = $stat;
6627             }
6628
6629             my $readycounter = 0;
6630
6631             foreach my $job (keys %$jobs) {
6632
6633                 if(defined($jobs->{$job}->{complete}) && !defined($running_mirror_jobs->{$job})) {
6634                     print "$job : finished\n";
6635                     delete $jobs->{$job};
6636                     next;
6637                 }
6638
6639                 die "$job: mirroring has been cancelled\n" if !defined($running_mirror_jobs->{$job});
6640
6641                 my $busy = $running_mirror_jobs->{$job}->{busy};
6642                 my $ready = $running_mirror_jobs->{$job}->{ready};
6643                 if (my $total = $running_mirror_jobs->{$job}->{len}) {
6644                     my $transferred = $running_mirror_jobs->{$job}->{offset} || 0;
6645                     my $remaining = $total - $transferred;
6646                     my $percent = sprintf "%.2f", ($transferred * 100 / $total);
6647
6648                     print "$job: transferred: $transferred bytes remaining: $remaining bytes total: $total bytes progression: $percent % busy: $busy ready: $ready \n";
6649                 }
6650
6651                 $readycounter++ if $running_mirror_jobs->{$job}->{ready};
6652             }
6653
6654             last if scalar(keys %$jobs) == 0;
6655
6656             if ($readycounter == scalar(keys %$jobs)) {
6657                 print "all mirroring jobs are ready \n";
6658                 last if $skipcomplete; #do the complete later
6659
6660                 if ($vmiddst && $vmiddst != $vmid) {
6661                     my $agent_running = $qga && qga_check_running($vmid);
6662                     if ($agent_running) {
6663                         print "freeze filesystem\n";
6664                         eval { PVE::QemuServer::vm_mon_cmd($vmid, "guest-fsfreeze-freeze"); };
6665                     } else {
6666                         print "suspend vm\n";
6667                         eval { PVE::QemuServer::vm_suspend($vmid, 1); };
6668                     }
6669
6670                     # if we clone a disk for a new target vm, we don't switch the disk
6671                     PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs);
6672
6673                     if ($agent_running) {
6674                         print "unfreeze filesystem\n";
6675                         eval { PVE::QemuServer::vm_mon_cmd($vmid, "guest-fsfreeze-thaw"); };
6676                     } else {
6677                         print "resume vm\n";
6678                         eval {  PVE::QemuServer::vm_resume($vmid, 1, 1); };
6679                     }
6680
6681                     last;
6682                 } else {
6683
6684                     foreach my $job (keys %$jobs) {
6685                         # try to switch the disk if source and destination are on the same guest
6686                         print "$job: Completing block job...\n";
6687
6688                         eval { vm_mon_cmd($vmid, "block-job-complete", device => $job) };
6689                         if ($@ =~ m/cannot be completed/) {
6690                             print "$job: Block job cannot be completed, try again.\n";
6691                             $err_complete++;
6692                         }else {
6693                             print "$job: Completed successfully.\n";
6694                             $jobs->{$job}->{complete} = 1;
6695                         }
6696                     }
6697                 }
6698             }
6699             sleep 1;
6700         }
6701     };
6702     my $err = $@;
6703
6704     if ($err) {
6705         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6706         die "mirroring error: $err";
6707     }
6708
6709 }
6710
6711 sub qemu_blockjobs_cancel {
6712     my ($vmid, $jobs) = @_;
6713
6714     foreach my $job (keys %$jobs) {
6715         print "$job: Cancelling block job\n";
6716         eval { vm_mon_cmd($vmid, "block-job-cancel", device => $job); };
6717         $jobs->{$job}->{cancel} = 1;
6718     }
6719
6720     while (1) {
6721         my $stats = vm_mon_cmd($vmid, "query-block-jobs");
6722
6723         my $running_jobs = {};
6724         foreach my $stat (@$stats) {
6725             $running_jobs->{$stat->{device}} = $stat;
6726         }
6727
6728         foreach my $job (keys %$jobs) {
6729
6730             if (defined($jobs->{$job}->{cancel}) && !defined($running_jobs->{$job})) {
6731                 print "$job: Done.\n";
6732                 delete $jobs->{$job};
6733             }
6734         }
6735
6736         last if scalar(keys %$jobs) == 0;
6737
6738         sleep 1;
6739     }
6740 }
6741
6742 sub clone_disk {
6743     my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
6744         $newvmid, $storage, $format, $full, $newvollist, $jobs, $skipcomplete, $qga) = @_;
6745
6746     my $newvolid;
6747
6748     if (!$full) {
6749         print "create linked clone of drive $drivename ($drive->{file})\n";
6750         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
6751         push @$newvollist, $newvolid;
6752     } else {
6753
6754         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6755         $storeid = $storage if $storage;
6756
6757         my $dst_format = resolve_dst_disk_format($storecfg, $storeid, $volname, $format);
6758         my ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 3);
6759
6760         print "create full clone of drive $drivename ($drive->{file})\n";
6761         my $name = undef;
6762         if (drive_is_cloudinit($drive)) {
6763             $name = "vm-$newvmid-cloudinit";
6764             $snapname = undef;
6765             # cloudinit only supports raw and qcow2 atm:
6766             if ($dst_format eq 'qcow2') {
6767                 $name .= '.qcow2';
6768             } elsif ($dst_format ne 'raw') {
6769                 die "clone: unhandled format for cloudinit image\n";
6770             }
6771         }
6772         $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $newvmid, $dst_format, $name, ($size/1024));
6773         push @$newvollist, $newvolid;
6774
6775         PVE::Storage::activate_volumes($storecfg, [$newvolid]);
6776
6777         my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
6778         if (!$running || $snapname) {
6779             qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit);
6780         } else {
6781
6782             my $kvmver = get_running_qemu_version ($vmid);
6783             if (!qemu_machine_feature_enabled (undef, $kvmver, 2, 7)) {
6784                 die "drive-mirror with iothread requires qemu version 2.7 or higher\n"
6785                     if $drive->{iothread};
6786             }
6787
6788             qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid, $sparseinit, $jobs, $skipcomplete, $qga);
6789         }
6790     }
6791
6792     my ($size) = PVE::Storage::volume_size_info($storecfg, $newvolid, 3);
6793
6794     my $disk = $drive;
6795     $disk->{format} = undef;
6796     $disk->{file} = $newvolid;
6797     $disk->{size} = $size;
6798
6799     return $disk;
6800 }
6801
6802 # this only works if VM is running
6803 sub get_current_qemu_machine {
6804     my ($vmid) = @_;
6805
6806     my $cmd = { execute => 'query-machines', arguments => {} };
6807     my $res = vm_qmp_command($vmid, $cmd);
6808
6809     my ($current, $default);
6810     foreach my $e (@$res) {
6811         $default = $e->{name} if $e->{'is-default'};
6812         $current = $e->{name} if $e->{'is-current'};
6813     }
6814
6815     # fallback to the default machine if current is not supported by qemu
6816     return $current || $default || 'pc';
6817 }
6818
6819 sub get_running_qemu_version {
6820     my ($vmid) = @_;
6821     my $cmd = { execute => 'query-version', arguments => {} };
6822     my $res = vm_qmp_command($vmid, $cmd);
6823     return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
6824 }
6825
6826 sub qemu_machine_feature_enabled {
6827     my ($machine, $kvmver, $version_major, $version_minor) = @_;
6828
6829     my $current_major;
6830     my $current_minor;
6831
6832     if ($machine && $machine =~ m/^((?:pc(-i440fx|-q35)?|virt)-(\d+)\.(\d+))/) {
6833
6834         $current_major = $3;
6835         $current_minor = $4;
6836
6837     } elsif ($kvmver =~ m/^(\d+)\.(\d+)/) {
6838
6839         $current_major = $1;
6840         $current_minor = $2;
6841     }
6842
6843     return 1 if $current_major > $version_major ||
6844                 ($current_major == $version_major &&
6845                  $current_minor >= $version_minor);
6846 }
6847
6848 sub qemu_machine_pxe {
6849     my ($vmid, $conf, $machine) = @_;
6850
6851     $machine =  PVE::QemuServer::get_current_qemu_machine($vmid) if !$machine;
6852
6853     if ($conf->{machine} && $conf->{machine} =~ m/\.pxe$/) {
6854         $machine .= '.pxe';
6855     }
6856
6857     return $machine;
6858 }
6859
6860 sub qemu_use_old_bios_files {
6861     my ($machine_type) = @_;
6862
6863     return if !$machine_type;
6864
6865     my $use_old_bios_files = undef;
6866
6867     if ($machine_type =~ m/^(\S+)\.pxe$/) {
6868         $machine_type = $1;
6869         $use_old_bios_files = 1;
6870     } else {
6871         my $kvmver = kvm_user_version();
6872         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
6873         # load new efi bios files on migration. So this hack is required to allow
6874         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
6875         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
6876         $use_old_bios_files = !qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 4);
6877     }
6878
6879     return ($use_old_bios_files, $machine_type);
6880 }
6881
6882 sub create_efidisk($$$$$) {
6883     my ($storecfg, $storeid, $vmid, $fmt, $arch) = @_;
6884
6885     my (undef, $ovmf_vars) = get_ovmf_files($arch);
6886     die "EFI vars default image not found\n" if ! -f $ovmf_vars;
6887
6888     my $vars_size = PVE::Tools::convert_size(-s $ovmf_vars, 'b' => 'kb');
6889     my $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $fmt, undef, $vars_size);
6890     PVE::Storage::activate_volumes($storecfg, [$volid]);
6891
6892     my $path = PVE::Storage::path($storecfg, $volid);
6893     eval {
6894         run_command(['/usr/bin/qemu-img', 'convert', '-n', '-f', 'raw', '-O', $fmt, $ovmf_vars, $path]);
6895     };
6896     die "Copying EFI vars image failed: $@" if $@;
6897
6898     return ($volid, $vars_size);
6899 }
6900
6901 sub vm_iothreads_list {
6902     my ($vmid) = @_;
6903
6904     my $res = vm_mon_cmd($vmid, 'query-iothreads');
6905
6906     my $iothreads = {};
6907     foreach my $iothread (@$res) {
6908         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
6909     }
6910
6911     return $iothreads;
6912 }
6913
6914 sub scsihw_infos {
6915     my ($conf, $drive) = @_;
6916
6917     my $maxdev = 0;
6918
6919     if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)) {
6920         $maxdev = 7;
6921     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
6922         $maxdev = 1;
6923     } else {
6924         $maxdev = 256;
6925     }
6926
6927     my $controller = int($drive->{index} / $maxdev);
6928     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single') ? "virtioscsi" : "scsihw";
6929
6930     return ($maxdev, $controller, $controller_prefix);
6931 }
6932
6933 sub add_hyperv_enlightenments {
6934     my ($cpuFlags, $winversion, $machine_type, $kvmver, $bios, $gpu_passthrough, $hv_vendor_id) = @_;
6935
6936     return if $winversion < 6;
6937     return if $bios && $bios eq 'ovmf' && $winversion < 8;
6938
6939     if ($gpu_passthrough || defined($hv_vendor_id)) {
6940         $hv_vendor_id //= 'proxmox';
6941         push @$cpuFlags , "hv_vendor_id=$hv_vendor_id";
6942     }
6943
6944     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
6945         push @$cpuFlags , 'hv_spinlocks=0x1fff';
6946         push @$cpuFlags , 'hv_vapic';
6947         push @$cpuFlags , 'hv_time';
6948     } else {
6949         push @$cpuFlags , 'hv_spinlocks=0xffff';
6950     }
6951
6952     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 6)) {
6953         push @$cpuFlags , 'hv_reset';
6954         push @$cpuFlags , 'hv_vpindex';
6955         push @$cpuFlags , 'hv_runtime';
6956     }
6957
6958     if ($winversion >= 7) {
6959         push @$cpuFlags , 'hv_relaxed';
6960
6961         if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 12)) {
6962             push @$cpuFlags , 'hv_synic';
6963             push @$cpuFlags , 'hv_stimer';
6964         }
6965     }
6966 }
6967
6968 sub windows_version {
6969     my ($ostype) = @_;
6970
6971     return 0 if !$ostype;
6972
6973     my $winversion = 0;
6974
6975     if($ostype eq 'wxp' || $ostype eq 'w2k3' || $ostype eq 'w2k') {
6976         $winversion = 5;
6977     } elsif($ostype eq 'w2k8' || $ostype eq 'wvista') {
6978         $winversion = 6;
6979     } elsif ($ostype =~ m/^win(\d+)$/) {
6980         $winversion = $1;
6981     }
6982
6983     return $winversion;
6984 }
6985
6986 sub resolve_dst_disk_format {
6987         my ($storecfg, $storeid, $src_volname, $format) = @_;
6988         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
6989
6990         if (!$format) {
6991             # if no target format is specified, use the source disk format as hint
6992             if ($src_volname) {
6993                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6994                 $format = qemu_img_format($scfg, $src_volname);
6995             } else {
6996                 return $defFormat;
6997             }
6998         }
6999
7000         # test if requested format is supported - else use default
7001         my $supported = grep { $_ eq $format } @$validFormats;
7002         $format = $defFormat if !$supported;
7003         return $format;
7004 }
7005
7006 sub resolve_first_disk {
7007     my $conf = shift;
7008     my @disks = PVE::QemuServer::valid_drive_names();
7009     my $firstdisk;
7010     foreach my $ds (reverse @disks) {
7011         next if !$conf->{$ds};
7012         my $disk = PVE::QemuServer::parse_drive($ds, $conf->{$ds});
7013         next if PVE::QemuServer::drive_is_cdrom($disk);
7014         $firstdisk = $ds;
7015     }
7016     return $firstdisk;
7017 }
7018
7019 sub generate_uuid {
7020     my ($uuid, $uuid_str);
7021     UUID::generate($uuid);
7022     UUID::unparse($uuid, $uuid_str);
7023     return $uuid_str;
7024 }
7025
7026 sub generate_smbios1_uuid {
7027     return "uuid=".generate_uuid();
7028 }
7029
7030 sub nbd_stop {
7031     my ($vmid) = @_;
7032
7033     vm_mon_cmd($vmid, 'nbd-server-stop');
7034 }
7035
7036 # bash completion helper
7037
7038 sub complete_backup_archives {
7039     my ($cmdname, $pname, $cvalue) = @_;
7040
7041     my $cfg = PVE::Storage::config();
7042
7043     my $storeid;
7044
7045     if ($cvalue =~ m/^([^:]+):/) {
7046         $storeid = $1;
7047     }
7048
7049     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
7050
7051     my $res = [];
7052     foreach my $id (keys %$data) {
7053         foreach my $item (@{$data->{$id}}) {
7054             next if $item->{format} !~ m/^vma\.(gz|lzo)$/;
7055             push @$res, $item->{volid} if defined($item->{volid});
7056         }
7057     }
7058
7059     return $res;
7060 }
7061
7062 my $complete_vmid_full = sub {
7063     my ($running) = @_;
7064
7065     my $idlist = vmstatus();
7066
7067     my $res = [];
7068
7069     foreach my $id (keys %$idlist) {
7070         my $d = $idlist->{$id};
7071         if (defined($running)) {
7072             next if $d->{template};
7073             next if $running && $d->{status} ne 'running';
7074             next if !$running && $d->{status} eq 'running';
7075         }
7076         push @$res, $id;
7077
7078     }
7079     return $res;
7080 };
7081
7082 sub complete_vmid {
7083     return &$complete_vmid_full();
7084 }
7085
7086 sub complete_vmid_stopped {
7087     return &$complete_vmid_full(0);
7088 }
7089
7090 sub complete_vmid_running {
7091     return &$complete_vmid_full(1);
7092 }
7093
7094 sub complete_storage {
7095
7096     my $cfg = PVE::Storage::config();
7097     my $ids = $cfg->{ids};
7098
7099     my $res = [];
7100     foreach my $sid (keys %$ids) {
7101         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
7102         next if !$ids->{$sid}->{content}->{images};
7103         push @$res, $sid;
7104     }
7105
7106     return $res;
7107 }
7108
7109 1;