PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5 use POSIX;
   6 use IO::Handle;
   7 use IO::Select;
   8 use IO::File;
   9 use IO::Dir;
  10 use IO::Socket::UNIX;
  11 use File::Basename;
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use Digest::SHA;
  16 use Fcntl ':flock';
  17 use Cwd 'abs_path';
  18 use IPC::Open3;
  19 use JSON;
  20 use Fcntl;
  21 use PVE::SafeSyslog;
  22 use Storable qw(dclone);
  23 use PVE::Exception qw(raise raise_param_exc);
  24 use PVE::Storage;
  25 use PVE::Tools qw(run_command lock_file lock_file_full file_read_firstline dir_glob_foreach $IPV6RE);
  26 use PVE::JSONSchema qw(get_standard_option);
  27 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file cfs_lock_file);
  28 use PVE::INotify;
  29 use PVE::ProcFSTools;
  30 use PVE::QemuConfig;
  31 use PVE::QMPClient;
  32 use PVE::RPCEnvironment;
  33 use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr);
  34 use PVE::QemuServer::Memory;
  35 use PVE::QemuServer::USB qw(parse_usb_device);
  36 use PVE::QemuServer::Cloudinit;
  37 use PVE::Systemd;
  38 use Time::HiRes qw(gettimeofday);
  39 use File::Copy qw(copy);
  40 use URI::Escape;
  41
  42 my $EDK2_FW_BASE = '/usr/share/pve-edk2-firmware/';
  43 my $OVMF = {
  44     x86_64 => [
  45         "$EDK2_FW_BASE/OVMF_CODE.fd",
  46         "$EDK2_FW_BASE/OVMF_VARS.fd"
  47     ],
  48     aarch64 => [
  49         "$EDK2_FW_BASE/AAVMF_CODE.fd",
  50         "$EDK2_FW_BASE/AAVMF_VARS.fd"
  51     ],
  52 };
  53
  54 my $qemu_snap_storage = {rbd => 1, sheepdog => 1};
  55
  56 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  57
  58 my $QEMU_FORMAT_RE = qr/raw|cow|qcow|qcow2|qed|vmdk|cloop/;
  59
  60 # Note about locking: we use flock on the config file protect
  61 # against concurent actions.
  62 # Aditionaly, we have a 'lock' setting in the config file. This
  63 # can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
  64 # allowed when such lock is set. But you can ignore this kind of
  65 # lock with the --skiplock flag.
  66
  67 cfs_register_file('/qemu-server/',
  68                   \&parse_vm_config,
  69                   \&write_vm_config);
  70
  71 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
  72     description => "Some command save/restore state from this location.",
  73     type => 'string',
  74     maxLength => 128,
  75     optional => 1,
  76 });
  77
  78 PVE::JSONSchema::register_standard_option('pve-snapshot-name', {
  79     description => "The name of the snapshot.",
  80     type => 'string', format => 'pve-configid',
  81     maxLength => 40,
  82 });
  83
  84 PVE::JSONSchema::register_standard_option('pve-qm-image-format', {
  85     type => 'string',
  86     enum => [qw(raw cow qcow qed qcow2 vmdk cloop)],
  87     description => "The drive's backing file's data format.",
  88     optional => 1,
  89 });
  90
  91 PVE::JSONSchema::register_standard_option('pve-qemu-machine', {
  92         description => "Specifies the Qemu machine type.",
  93         type => 'string',
  94         pattern => '(pc|pc(-i440fx)?-\d+\.\d+(\.pxe)?|q35|pc-q35-\d+\.\d+(\.pxe)?|virt(?:-\d+\.\d+)?)',
  95         maxLength => 40,
  96         optional => 1,
  97 });
  98
  99 #no warnings 'redefine';
 100
 101 sub cgroups_write {
 102    my ($controller, $vmid, $option, $value) = @_;
 103
 104    my $path = "/sys/fs/cgroup/$controller/qemu.slice/$vmid.scope/$option";
 105    PVE::ProcFSTools::write_proc_entry($path, $value);
 106
 107 }
 108
 109 my $nodename = PVE::INotify::nodename();
 110
 111 mkdir "/etc/pve/nodes/$nodename";
 112 my $confdir = "/etc/pve/nodes/$nodename/qemu-server";
 113 mkdir $confdir;
 114
 115 my $var_run_tmpdir = "/var/run/qemu-server";
 116 mkdir $var_run_tmpdir;
 117
 118 my $lock_dir = "/var/lock/qemu-server";
 119 mkdir $lock_dir;
 120
 121 my $pcisysfs = "/sys/bus/pci";
 122
 123 my $cpu_vendor_list = {
 124     # Intel CPUs
 125     486 => 'GenuineIntel',
 126     pentium => 'GenuineIntel',
 127     pentium2  => 'GenuineIntel',
 128     pentium3  => 'GenuineIntel',
 129     coreduo => 'GenuineIntel',
 130     core2duo => 'GenuineIntel',
 131     Conroe  => 'GenuineIntel',
 132     Penryn  => 'GenuineIntel',
 133     Nehalem  => 'GenuineIntel',
 134     'Nehalem-IBRS'  => 'GenuineIntel',
 135     Westmere => 'GenuineIntel',
 136     'Westmere-IBRS' => 'GenuineIntel',
 137     SandyBridge => 'GenuineIntel',
 138     'SandyBridge-IBRS' => 'GenuineIntel',
 139     IvyBridge => 'GenuineIntel',
 140     'IvyBridge-IBRS' => 'GenuineIntel',
 141     Haswell => 'GenuineIntel',
 142     'Haswell-IBRS' => 'GenuineIntel',
 143     'Haswell-noTSX' => 'GenuineIntel',
 144     'Haswell-noTSX-IBRS' => 'GenuineIntel',
 145     Broadwell => 'GenuineIntel',
 146     'Broadwell-IBRS' => 'GenuineIntel',
 147     'Broadwell-noTSX' => 'GenuineIntel',
 148     'Broadwell-noTSX-IBRS' => 'GenuineIntel',
 149     'Skylake-Client' => 'GenuineIntel',
 150     'Skylake-Client-IBRS' => 'GenuineIntel',
 151     'Skylake-Server' => 'GenuineIntel',
 152     'Skylake-Server-IBRS' => 'GenuineIntel',
 153
 154     # AMD CPUs
 155     athlon => 'AuthenticAMD',
 156     phenom  => 'AuthenticAMD',
 157     Opteron_G1  => 'AuthenticAMD',
 158     Opteron_G2  => 'AuthenticAMD',
 159     Opteron_G3  => 'AuthenticAMD',
 160     Opteron_G4  => 'AuthenticAMD',
 161     Opteron_G5  => 'AuthenticAMD',
 162     EPYC => 'AuthenticAMD',
 163     'EPYC-IBPB' => 'AuthenticAMD',
 164
 165     # generic types, use vendor from host node
 166     host => 'default',
 167     kvm32 => 'default',
 168     kvm64 => 'default',
 169     qemu32 => 'default',
 170     qemu64 => 'default',
 171     max => 'default',
 172 };
 173
 174 my $cpu_flag = qr/[+-](pcid|spec-ctrl|ibpb|ssbd|virt-ssbd|amd-ssbd|amd-no-ssb|pdpe1gb)/;
 175
 176 my $cpu_fmt = {
 177     cputype => {
 178         description => "Emulated CPU type.",
 179         type => 'string',
 180         enum => [ sort { "\L$a" cmp "\L$b" } keys %$cpu_vendor_list ],
 181         default => 'kvm64',
 182         default_key => 1,
 183     },
 184     hidden => {
 185         description => "Do not identify as a KVM virtual machine.",
 186         type => 'boolean',
 187         optional => 1,
 188         default => 0
 189     },
 190     flags => {
 191         description => "List of additional CPU flags separated by ';'."
 192                      . " Use '+FLAG' to enable, '-FLAG' to disable a flag."
 193                      . " Currently supported flags: 'pcid', 'spec-ctrl', 'ibpb', 'ssbd', 'virt-ssbd', 'amd-ssbd', 'amd-no-ssb', 'pdpe1gb'.",
 194         format_description => '+FLAG[;-FLAG...]',
 195         type => 'string',
 196         pattern => qr/$cpu_flag(;$cpu_flag)*/,
 197         optional => 1,
 198     },
 199 };
 200
 201 my $watchdog_fmt = {
 202     model => {
 203         default_key => 1,
 204         type => 'string',
 205         enum => [qw(i6300esb ib700)],
 206         description => "Watchdog type to emulate.",
 207         default => 'i6300esb',
 208         optional => 1,
 209     },
 210     action => {
 211         type => 'string',
 212         enum => [qw(reset shutdown poweroff pause debug none)],
 213         description => "The action to perform if after activation the guest fails to poll the watchdog in time.",
 214         optional => 1,
 215     },
 216 };
 217 PVE::JSONSchema::register_format('pve-qm-watchdog', $watchdog_fmt);
 218
 219 my $agent_fmt = {
 220     enabled => {
 221         description => "Enable/disable Qemu GuestAgent.",
 222         type => 'boolean',
 223         default => 0,
 224         default_key => 1,
 225     },
 226     fstrim_cloned_disks => {
 227         description => "Run fstrim after cloning/moving a disk.",
 228         type => 'boolean',
 229         optional => 1,
 230         default => 0
 231     },
 232 };
 233
 234 my $vga_fmt = {
 235     type => {
 236         description => "Select the VGA type.",
 237         type => 'string',
 238         default => 'std',
 239         optional => 1,
 240         default_key => 1,
 241         enum => [qw(cirrus qxl qxl2 qxl3 qxl4 serial0 serial1 serial2 serial3 std virtio vmware)],
 242     },
 243     memory => {
 244         description => "Sets the VGA memory (in MiB). Has no effect with serial display.",
 245         type => 'integer',
 246         optional => 1,
 247         minimum => 4,
 248         maximum => 512,
 249     },
 250 };
 251
 252 my $confdesc = {
 253     onboot => {
 254         optional => 1,
 255         type => 'boolean',
 256         description => "Specifies whether a VM will be started during system bootup.",
 257         default => 0,
 258     },
 259     autostart => {
 260         optional => 1,
 261         type => 'boolean',
 262         description => "Automatic restart after crash (currently ignored).",
 263         default => 0,
 264     },
 265     hotplug => {
 266         optional => 1,
 267         type => 'string', format => 'pve-hotplug-features',
 268         description => "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
 269         default => 'network,disk,usb',
 270     },
 271     reboot => {
 272         optional => 1,
 273         type => 'boolean',
 274         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 275         default => 1,
 276     },
 277     lock => {
 278         optional => 1,
 279         type => 'string',
 280         description => "Lock/unlock the VM.",
 281         enum => [qw(migrate backup snapshot rollback)],
 282     },
 283     cpulimit => {
 284         optional => 1,
 285         type => 'number',
 286         description => "Limit of CPU usage.",
 287         verbose_description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
 288         minimum => 0,
 289         maximum => 128,
 290         default => 0,
 291     },
 292     cpuunits => {
 293         optional => 1,
 294         type => 'integer',
 295         description => "CPU weight for a VM.",
 296         verbose_description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.",
 297         minimum => 2,
 298         maximum => 262144,
 299         default => 1024,
 300     },
 301     memory => {
 302         optional => 1,
 303         type => 'integer',
 304         description => "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
 305         minimum => 16,
 306         default => 512,
 307     },
 308     balloon => {
 309         optional => 1,
 310         type => 'integer',
 311         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 312         minimum => 0,
 313     },
 314     shares => {
 315         optional => 1,
 316         type => 'integer',
 317         description => "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning. Auto-ballooning is done by pvestatd.",
 318         minimum => 0,
 319         maximum => 50000,
 320         default => 1000,
 321     },
 322     keyboard => {
 323         optional => 1,
 324         type => 'string',
 325         description => "Keybord layout for vnc server. Default is read from the '/etc/pve/datacenter.cfg' configuration file.".
 326                        "It should not be necessary to set it.",
 327         enum => PVE::Tools::kvmkeymaplist(),
 328         default => undef,
 329     },
 330     name => {
 331         optional => 1,
 332         type => 'string', format => 'dns-name',
 333         description => "Set a name for the VM. Only used on the configuration web interface.",
 334     },
 335     scsihw => {
 336         optional => 1,
 337         type => 'string',
 338         description => "SCSI controller model",
 339         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 340         default => 'lsi',
 341     },
 342     description => {
 343         optional => 1,
 344         type => 'string',
 345         description => "Description for the VM. Only used on the configuration web interface. This is saved as comment inside the configuration file.",
 346     },
 347     ostype => {
 348         optional => 1,
 349         type => 'string',
 350         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 l24 l26 solaris)],
 351         description => "Specify guest operating system.",
 352         verbose_description => <<EODESC,
 353 Specify guest operating system. This is used to enable special
 354 optimization/features for specific operating systems:
 355
 356 [horizontal]
 357 other;; unspecified OS
 358 wxp;; Microsoft Windows XP
 359 w2k;; Microsoft Windows 2000
 360 w2k3;; Microsoft Windows 2003
 361 w2k8;; Microsoft Windows 2008
 362 wvista;; Microsoft Windows Vista
 363 win7;; Microsoft Windows 7
 364 win8;; Microsoft Windows 8/2012/2012r2
 365 win10;; Microsoft Windows 10/2016
 366 l24;; Linux 2.4 Kernel
 367 l26;; Linux 2.6/3.X Kernel
 368 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
 369 EODESC
 370     },
 371     boot => {
 372         optional => 1,
 373         type => 'string',
 374         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n).",
 375         pattern => '[acdn]{1,4}',
 376         default => 'cdn',
 377     },
 378     bootdisk => {
 379         optional => 1,
 380         type => 'string', format => 'pve-qm-bootdisk',
 381         description => "Enable booting from specified disk.",
 382         pattern => '(ide|sata|scsi|virtio)\d+',
 383     },
 384     smp => {
 385         optional => 1,
 386         type => 'integer',
 387         description => "The number of CPUs. Please use option -sockets instead.",
 388         minimum => 1,
 389         default => 1,
 390     },
 391     sockets => {
 392         optional => 1,
 393         type => 'integer',
 394         description => "The number of CPU sockets.",
 395         minimum => 1,
 396         default => 1,
 397     },
 398     cores => {
 399         optional => 1,
 400         type => 'integer',
 401         description => "The number of cores per socket.",
 402         minimum => 1,
 403         default => 1,
 404     },
 405     numa => {
 406         optional => 1,
 407         type => 'boolean',
 408         description => "Enable/disable NUMA.",
 409         default => 0,
 410     },
 411     hugepages => {
 412         optional => 1,
 413         type => 'string',
 414         description => "Enable/disable hugepages memory.",
 415         enum => [qw(any 2 1024)],
 416     },
 417     vcpus => {
 418         optional => 1,
 419         type => 'integer',
 420         description => "Number of hotplugged vcpus.",
 421         minimum => 1,
 422         default => 0,
 423     },
 424     acpi => {
 425         optional => 1,
 426         type => 'boolean',
 427         description => "Enable/disable ACPI.",
 428         default => 1,
 429     },
 430     agent => {
 431         optional => 1,
 432         description => "Enable/disable Qemu GuestAgent and its properties.",
 433         type => 'string',
 434         format => $agent_fmt,
 435     },
 436     kvm => {
 437         optional => 1,
 438         type => 'boolean',
 439         description => "Enable/disable KVM hardware virtualization.",
 440         default => 1,
 441     },
 442     tdf => {
 443         optional => 1,
 444         type => 'boolean',
 445         description => "Enable/disable time drift fix.",
 446         default => 0,
 447     },
 448     localtime => {
 449         optional => 1,
 450         type => 'boolean',
 451         description => "Set the real time clock to local time. This is enabled by default if ostype indicates a Microsoft OS.",
 452     },
 453     freeze => {
 454         optional => 1,
 455         type => 'boolean',
 456         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 457     },
 458     vga => {
 459         optional => 1,
 460         type => 'string', format => $vga_fmt,
 461         description => "Configure the VGA hardware.",
 462         verbose_description => "Configure the VGA Hardware. If you want to use ".
 463             "high resolution modes (>= 1280x1024x16) you may need to increase " .
 464             "the vga memory option. Since QEMU 2.9 the default VGA display type " .
 465             "is 'std' for all OS types besides some Windows versions (XP and " .
 466             "older) which use 'cirrus'. The 'qxl' option enables the SPICE " .
 467             "display server. For win* OS you can select how many independent " .
 468             "displays you want, Linux guests can add displays them self.\n".
 469             "You can also run without any graphic card, using a serial device as terminal.",
 470     },
 471     watchdog => {
 472         optional => 1,
 473         type => 'string', format => 'pve-qm-watchdog',
 474         description => "Create a virtual hardware watchdog device.",
 475         verbose_description => "Create a virtual hardware watchdog device. Once enabled" .
 476             " (by a guest action), the watchdog must be periodically polled " .
 477             "by an agent inside the guest or else the watchdog will reset " .
 478             "the guest (or execute the respective action specified)",
 479     },
 480     startdate => {
 481         optional => 1,
 482         type => 'string',
 483         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 484         description => "Set the initial date of the real time clock. Valid format for date are: 'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 485         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 486         default => 'now',
 487     },
 488     startup =>  get_standard_option('pve-startup-order'),
 489     template => {
 490         optional => 1,
 491         type => 'boolean',
 492         description => "Enable/disable Template.",
 493         default => 0,
 494     },
 495     args => {
 496         optional => 1,
 497         type => 'string',
 498         description => "Arbitrary arguments passed to kvm.",
 499         verbose_description => <<EODESCR,
 500 Arbitrary arguments passed to kvm, for example:
 501
 502 args: -no-reboot -no-hpet
 503
 504 NOTE: this option is for experts only.
 505 EODESCR
 506     },
 507     tablet => {
 508         optional => 1,
 509         type => 'boolean',
 510         default => 1,
 511         description => "Enable/disable the USB tablet device.",
 512         verbose_description => "Enable/disable the USB tablet device. This device is " .
 513             "usually needed to allow absolute mouse positioning with VNC. " .
 514             "Else the mouse runs out of sync with normal VNC clients. " .
 515             "If you're running lots of console-only guests on one host, " .
 516             "you may consider disabling this to save some context switches. " .
 517             "This is turned off by default if you use spice (-vga=qxl).",
 518     },
 519     migrate_speed => {
 520         optional => 1,
 521         type => 'integer',
 522         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 523         minimum => 0,
 524         default => 0,
 525     },
 526     migrate_downtime => {
 527         optional => 1,
 528         type => 'number',
 529         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 530         minimum => 0,
 531         default => 0.1,
 532     },
 533     cdrom => {
 534         optional => 1,
 535         type => 'string', format => 'pve-qm-ide',
 536         typetext => '<volume>',
 537         description => "This is an alias for option -ide2",
 538     },
 539     cpu => {
 540         optional => 1,
 541         description => "Emulated CPU type.",
 542         type => 'string',
 543         format => $cpu_fmt,
 544     },
 545     parent => get_standard_option('pve-snapshot-name', {
 546         optional => 1,
 547         description => "Parent snapshot name. This is used internally, and should not be modified.",
 548     }),
 549     snaptime => {
 550         optional => 1,
 551         description => "Timestamp for snapshots.",
 552         type => 'integer',
 553         minimum => 0,
 554     },
 555     vmstate => {
 556         optional => 1,
 557         type => 'string', format => 'pve-volume-id',
 558         description => "Reference to a volume which stores the VM state. This is used internally for snapshots.",
 559     },
 560     vmstatestorage => get_standard_option('pve-storage-id', {
 561         description => "Default storage for VM state volumes/files.",
 562         optional => 1,
 563     }),
 564     runningmachine => get_standard_option('pve-qemu-machine', {
 565         description => "Specifies the Qemu machine type of the running vm. This is used internally for snapshots.",
 566     }),
 567     machine => get_standard_option('pve-qemu-machine'),
 568     arch => {
 569         description => "Virtual processor architecture. Defaults to the host.",
 570         optional => 1,
 571         type => 'string',
 572         enum => [qw(x86_64 aarch64)],
 573     },
 574     smbios1 => {
 575         description => "Specify SMBIOS type 1 fields.",
 576         type => 'string', format => 'pve-qm-smbios1',
 577         maxLength => 256,
 578         optional => 1,
 579     },
 580     protection => {
 581         optional => 1,
 582         type => 'boolean',
 583         description => "Sets the protection flag of the VM. This will disable the remove VM and remove disk operations.",
 584         default => 0,
 585     },
 586     bios => {
 587         optional => 1,
 588         type => 'string',
 589         enum => [ qw(seabios ovmf) ],
 590         description => "Select BIOS implementation.",
 591         default => 'seabios',
 592     },
 593     vmgenid => {
 594         type => 'string',
 595         pattern => '(?:[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}|[01])',
 596         format_description => 'UUID',
 597         description => "Set VM Generation ID. Use '1' to autogenerate on create or update, pass '0' to disable explicitly.",
 598         verbose_description => "The VM generation ID (vmgenid) device exposes a".
 599             " 128-bit integer value identifier to the guest OS. This allows to".
 600             " notify the guest operating system when the virtual machine is".
 601             " executed with a different configuration (e.g. snapshot execution".
 602             " or creation from a template). The guest operating system notices".
 603             " the change, and is then able to react as appropriate by marking".
 604             " its copies of distributed databases as dirty, re-initializing its".
 605             " random number generator, etc.\n".
 606             "Note that auto-creation only works when done throug API/CLI create".
 607             " or update methods, but not when manually editing the config file.",
 608         default => "1 (autogenerated)",
 609         optional => 1,
 610     },
 611 };
 612
 613 my $confdesc_cloudinit = {
 614     citype => {
 615         optional => 1,
 616         type => 'string',
 617         description => 'Specifies the cloud-init configuration format. The default depends on the configured operating system type (`ostype`. We use the `nocloud` format for Linux, and `configdrive2` for windows.',
 618         enum => ['configdrive2', 'nocloud'],
 619     },
 620     ciuser => {
 621         optional => 1,
 622         type => 'string',
 623         description => "cloud-init: User name to change ssh keys and password for instead of the image's configured default user.",
 624     },
 625     cipassword => {
 626         optional => 1,
 627         type => 'string',
 628         description => 'cloud-init: Password to assign the user. Using this is generally not recommended. Use ssh keys instead. Also note that older cloud-init versions do not support hashed passwords.',
 629     },
 630     searchdomain => {
 631         optional => 1,
 632         type => 'string',
 633         description => "cloud-init: Sets DNS search domains for a container. Create will automatically use the setting from the host if neither searchdomain nor nameserver are set.",
 634     },
 635     nameserver => {
 636         optional => 1,
 637         type => 'string', format => 'address-list',
 638         description => "cloud-init: Sets DNS server IP address for a container. Create will automatically use the setting from the host if neither searchdomain nor nameserver are set.",
 639     },
 640     sshkeys => {
 641         optional => 1,
 642         type => 'string',
 643         format => 'urlencoded',
 644         description => "cloud-init: Setup public SSH keys (one key per line, OpenSSH format).",
 645     },
 646 };
 647
 648 # what about other qemu settings ?
 649 #cpu => 'string',
 650 #machine => 'string',
 651 #fda => 'file',
 652 #fdb => 'file',
 653 #mtdblock => 'file',
 654 #sd => 'file',
 655 #pflash => 'file',
 656 #snapshot => 'bool',
 657 #bootp => 'file',
 658 ##tftp => 'dir',
 659 ##smb => 'dir',
 660 #kernel => 'file',
 661 #append => 'string',
 662 #initrd => 'file',
 663 ##soundhw => 'string',
 664
 665 while (my ($k, $v) = each %$confdesc) {
 666     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 667 }
 668
 669 my $MAX_IDE_DISKS = 4;
 670 my $MAX_SCSI_DISKS = 14;
 671 my $MAX_VIRTIO_DISKS = 16;
 672 my $MAX_SATA_DISKS = 6;
 673 my $MAX_USB_DEVICES = 5;
 674 my $MAX_NETS = 32;
 675 my $MAX_UNUSED_DISKS = 256;
 676 my $MAX_HOSTPCI_DEVICES = 4;
 677 my $MAX_SERIAL_PORTS = 4;
 678 my $MAX_PARALLEL_PORTS = 3;
 679 my $MAX_NUMA = 8;
 680
 681 my $numa_fmt = {
 682     cpus => {
 683         type => "string",
 684         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 685         description => "CPUs accessing this NUMA node.",
 686         format_description => "id[-id];...",
 687     },
 688     memory => {
 689         type => "number",
 690         description => "Amount of memory this NUMA node provides.",
 691         optional => 1,
 692     },
 693     hostnodes => {
 694         type => "string",
 695         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 696         description => "Host NUMA nodes to use.",
 697         format_description => "id[-id];...",
 698         optional => 1,
 699     },
 700     policy => {
 701         type => 'string',
 702         enum => [qw(preferred bind interleave)],
 703         description => "NUMA allocation policy.",
 704         optional => 1,
 705     },
 706 };
 707 PVE::JSONSchema::register_format('pve-qm-numanode', $numa_fmt);
 708 my $numadesc = {
 709     optional => 1,
 710     type => 'string', format => $numa_fmt,
 711     description => "NUMA topology.",
 712 };
 713 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 714
 715 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 716     $confdesc->{"numa$i"} = $numadesc;
 717 }
 718
 719 my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000',  'pcnet',  'virtio',
 720                       'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3',
 721                       'e1000-82540em', 'e1000-82544gc', 'e1000-82545em'];
 722 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 723
 724 my $net_fmt_bridge_descr = <<__EOD__;
 725 Bridge to attach the network device to. The Proxmox VE standard bridge
 726 is called 'vmbr0'.
 727
 728 If you do not specify a bridge, we create a kvm user (NATed) network
 729 device, which provides DHCP and DNS services. The following addresses
 730 are used:
 731
 732  10.0.2.2   Gateway
 733  10.0.2.3   DNS Server
 734  10.0.2.4   SMB Server
 735
 736 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 737 __EOD__
 738
 739 my $net_fmt = {
 740     macaddr => {
 741         type => 'string',
 742         pattern => qr/[0-9a-f]{2}(?::[0-9a-f]{2}){5}/i,
 743         description => "MAC address. That address must be unique withing your network. This is automatically generated if not specified.",
 744         format_description => "XX:XX:XX:XX:XX:XX",
 745         optional => 1,
 746     },
 747     model => {
 748         type => 'string',
 749         description => "Network Card Model. The 'virtio' model provides the best performance with very low CPU overhead. If your guest does not support this driver, it is usually best to use 'e1000'.",
 750         enum => $nic_model_list,
 751         default_key => 1,
 752     },
 753     (map { $_ => { keyAlias => 'model', alias => 'macaddr' }} @$nic_model_list),
 754     bridge => {
 755         type => 'string',
 756         description => $net_fmt_bridge_descr,
 757         format_description => 'bridge',
 758         optional => 1,
 759     },
 760     queues => {
 761         type => 'integer',
 762         minimum => 0, maximum => 16,
 763         description => 'Number of packet queues to be used on the device.',
 764         optional => 1,
 765     },
 766     rate => {
 767         type => 'number',
 768         minimum => 0,
 769         description => "Rate limit in mbps (megabytes per second) as floating point number.",
 770         optional => 1,
 771     },
 772     tag => {
 773         type => 'integer',
 774         minimum => 1, maximum => 4094,
 775         description => 'VLAN tag to apply to packets on this interface.',
 776         optional => 1,
 777     },
 778     trunks => {
 779         type => 'string',
 780         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 781         description => 'VLAN trunks to pass through this interface.',
 782         format_description => 'vlanid[;vlanid...]',
 783         optional => 1,
 784     },
 785     firewall => {
 786         type => 'boolean',
 787         description => 'Whether this interface should be protected by the firewall.',
 788         optional => 1,
 789     },
 790     link_down => {
 791         type => 'boolean',
 792         description => 'Whether this interface should be disconnected (like pulling the plug).',
 793         optional => 1,
 794     },
 795 };
 796
 797 my $netdesc = {
 798     optional => 1,
 799     type => 'string', format => $net_fmt,
 800     description => "Specify network devices.",
 801 };
 802
 803 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 804
 805 my $ipconfig_fmt = {
 806     ip => {
 807         type => 'string',
 808         format => 'pve-ipv4-config',
 809         format_description => 'IPv4Format/CIDR',
 810         description => 'IPv4 address in CIDR format.',
 811         optional => 1,
 812         default => 'dhcp',
 813     },
 814     gw => {
 815         type => 'string',
 816         format => 'ipv4',
 817         format_description => 'GatewayIPv4',
 818         description => 'Default gateway for IPv4 traffic.',
 819         optional => 1,
 820         requires => 'ip',
 821     },
 822     ip6 => {
 823         type => 'string',
 824         format => 'pve-ipv6-config',
 825         format_description => 'IPv6Format/CIDR',
 826         description => 'IPv6 address in CIDR format.',
 827         optional => 1,
 828         default => 'dhcp',
 829     },
 830     gw6 => {
 831         type => 'string',
 832         format => 'ipv6',
 833         format_description => 'GatewayIPv6',
 834         description => 'Default gateway for IPv6 traffic.',
 835         optional => 1,
 836         requires => 'ip6',
 837     },
 838 };
 839 PVE::JSONSchema::register_format('pve-qm-ipconfig', $ipconfig_fmt);
 840 my $ipconfigdesc = {
 841     optional => 1,
 842     type => 'string', format => 'pve-qm-ipconfig',
 843     description => <<'EODESCR',
 844 cloud-init: Specify IP addresses and gateways for the corresponding interface.
 845
 846 IP addresses use CIDR notation, gateways are optional but need an IP of the same type specified.
 847
 848 The special string 'dhcp' can be used for IP addresses to use DHCP, in which case no explicit gateway should be provided.
 849 For IPv6 the special string 'auto' can be used to use stateless autoconfiguration.
 850
 851 If cloud-init is enabled and neither an IPv4 nor an IPv6 address is specified, it defaults to using dhcp on IPv4.
 852 EODESCR
 853 };
 854 PVE::JSONSchema::register_standard_option("pve-qm-ipconfig", $netdesc);
 855
 856 for (my $i = 0; $i < $MAX_NETS; $i++)  {
 857     $confdesc->{"net$i"} = $netdesc;
 858     $confdesc_cloudinit->{"ipconfig$i"} = $ipconfigdesc;
 859 }
 860
 861 foreach my $key (keys %$confdesc_cloudinit) {
 862     $confdesc->{$key} = $confdesc_cloudinit->{$key};
 863 }
 864
 865 PVE::JSONSchema::register_format('pve-volume-id-or-qm-path', \&verify_volume_id_or_qm_path);
 866 sub verify_volume_id_or_qm_path {
 867     my ($volid, $noerr) = @_;
 868
 869     if ($volid eq 'none' || $volid eq 'cdrom' || $volid =~ m|^/|) {
 870         return $volid;
 871     }
 872
 873     # if its neither 'none' nor 'cdrom' nor a path, check if its a volume-id
 874     $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') };
 875     if ($@) {
 876         return undef if $noerr;
 877         die $@;
 878     }
 879     return $volid;
 880 }
 881
 882 my $drivename_hash;
 883
 884 my %drivedesc_base = (
 885     volume => { alias => 'file' },
 886     file => {
 887         type => 'string',
 888         format => 'pve-volume-id-or-qm-path',
 889         default_key => 1,
 890         format_description => 'volume',
 891         description => "The drive's backing volume.",
 892     },
 893     media => {
 894         type => 'string',
 895         enum => [qw(cdrom disk)],
 896         description => "The drive's media type.",
 897         default => 'disk',
 898         optional => 1
 899     },
 900     cyls => {
 901         type => 'integer',
 902         description => "Force the drive's physical geometry to have a specific cylinder count.",
 903         optional => 1
 904     },
 905     heads => {
 906         type => 'integer',
 907         description => "Force the drive's physical geometry to have a specific head count.",
 908         optional => 1
 909     },
 910     secs => {
 911         type => 'integer',
 912         description => "Force the drive's physical geometry to have a specific sector count.",
 913         optional => 1
 914     },
 915     trans => {
 916         type => 'string',
 917         enum => [qw(none lba auto)],
 918         description => "Force disk geometry bios translation mode.",
 919         optional => 1,
 920     },
 921     snapshot => {
 922         type => 'boolean',
 923         description => "Controls qemu's snapshot mode feature."
 924             . " If activated, changes made to the disk are temporary and will"
 925             . " be discarded when the VM is shutdown.",
 926         optional => 1,
 927     },
 928     cache => {
 929         type => 'string',
 930         enum => [qw(none writethrough writeback unsafe directsync)],
 931         description => "The drive's cache mode",
 932         optional => 1,
 933     },
 934     format => get_standard_option('pve-qm-image-format'),
 935     size => {
 936         type => 'string',
 937         format => 'disk-size',
 938         format_description => 'DiskSize',
 939         description => "Disk size. This is purely informational and has no effect.",
 940         optional => 1,
 941     },
 942     backup => {
 943         type => 'boolean',
 944         description => "Whether the drive should be included when making backups.",
 945         optional => 1,
 946     },
 947     replicate => {
 948         type => 'boolean',
 949         description => 'Whether the drive should considered for replication jobs.',
 950         optional => 1,
 951         default => 1,
 952     },
 953     rerror => {
 954         type => 'string',
 955         enum => [qw(ignore report stop)],
 956         description => 'Read error action.',
 957         optional => 1,
 958     },
 959     werror => {
 960         type => 'string',
 961         enum => [qw(enospc ignore report stop)],
 962         description => 'Write error action.',
 963         optional => 1,
 964     },
 965     aio => {
 966         type => 'string',
 967         enum => [qw(native threads)],
 968         description => 'AIO type to use.',
 969         optional => 1,
 970     },
 971     discard => {
 972         type => 'string',
 973         enum => [qw(ignore on)],
 974         description => 'Controls whether to pass discard/trim requests to the underlying storage.',
 975         optional => 1,
 976     },
 977     detect_zeroes => {
 978         type => 'boolean',
 979         description => 'Controls whether to detect and try to optimize writes of zeroes.',
 980         optional => 1,
 981     },
 982     serial => {
 983         type => 'string',
 984         format => 'urlencoded',
 985         format_description => 'serial',
 986         maxLength => 20*3, # *3 since it's %xx url enoded
 987         description => "The drive's reported serial number, url-encoded, up to 20 bytes long.",
 988         optional => 1,
 989     },
 990     shared => {
 991         type => 'boolean',
 992         description => 'Mark this locally-managed volume as available on all nodes',
 993         verbose_description => "Mark this locally-managed volume as available on all nodes.\n\nWARNING: This option does not share the volume automatically, it assumes it is shared already!",
 994         optional => 1,
 995         default => 0,
 996     }
 997 );
 998
 999 my %iothread_fmt = ( iothread => {
1000         type => 'boolean',
1001         description => "Whether to use iothreads for this drive",
1002         optional => 1,
1003 });
1004
1005 my %model_fmt = (
1006     model => {
1007         type => 'string',
1008         format => 'urlencoded',
1009         format_description => 'model',
1010         maxLength => 40*3, # *3 since it's %xx url enoded
1011         description => "The drive's reported model name, url-encoded, up to 40 bytes long.",
1012         optional => 1,
1013     },
1014 );
1015
1016 my %queues_fmt = (
1017     queues => {
1018         type => 'integer',
1019         description => "Number of queues.",
1020         minimum => 2,
1021         optional => 1
1022     }
1023 );
1024
1025 my %scsiblock_fmt = (
1026     scsiblock => {
1027         type => 'boolean',
1028         description => "whether to use scsi-block for full passthrough of host block device\n\nWARNING: can lead to I/O errors in combination with low memory or high memory fragmentation on host",
1029         optional => 1,
1030         default => 0,
1031     },
1032 );
1033
1034 my %ssd_fmt = (
1035     ssd => {
1036         type => 'boolean',
1037         description => "Whether to expose this drive as an SSD, rather than a rotational hard disk.",
1038         optional => 1,
1039     },
1040 );
1041
1042 my $add_throttle_desc = sub {
1043     my ($key, $type, $what, $unit, $longunit, $minimum) = @_;
1044     my $d = {
1045         type => $type,
1046         format_description => $unit,
1047         description => "Maximum $what in $longunit.",
1048         optional => 1,
1049     };
1050     $d->{minimum} = $minimum if defined($minimum);
1051     $drivedesc_base{$key} = $d;
1052 };
1053 # throughput: (leaky bucket)
1054 $add_throttle_desc->('bps',     'integer', 'r/w speed',   'bps',  'bytes per second');
1055 $add_throttle_desc->('bps_rd',  'integer', 'read speed',  'bps',  'bytes per second');
1056 $add_throttle_desc->('bps_wr',  'integer', 'write speed', 'bps',  'bytes per second');
1057 $add_throttle_desc->('mbps',    'number',  'r/w speed',   'mbps', 'megabytes per second');
1058 $add_throttle_desc->('mbps_rd', 'number',  'read speed',  'mbps', 'megabytes per second');
1059 $add_throttle_desc->('mbps_wr', 'number',  'write speed', 'mbps', 'megabytes per second');
1060 $add_throttle_desc->('iops',    'integer', 'r/w I/O',     'iops', 'operations per second');
1061 $add_throttle_desc->('iops_rd', 'integer', 'read I/O',    'iops', 'operations per second');
1062 $add_throttle_desc->('iops_wr', 'integer', 'write I/O',   'iops', 'operations per second');
1063
1064 # pools: (pool of IO before throttling starts taking effect)
1065 $add_throttle_desc->('mbps_max',    'number',  'unthrottled r/w pool',       'mbps', 'megabytes per second');
1066 $add_throttle_desc->('mbps_rd_max', 'number',  'unthrottled read pool',      'mbps', 'megabytes per second');
1067 $add_throttle_desc->('mbps_wr_max', 'number',  'unthrottled write pool',     'mbps', 'megabytes per second');
1068 $add_throttle_desc->('iops_max',    'integer', 'unthrottled r/w I/O pool',   'iops', 'operations per second');
1069 $add_throttle_desc->('iops_rd_max', 'integer', 'unthrottled read I/O pool',  'iops', 'operations per second');
1070 $add_throttle_desc->('iops_wr_max', 'integer', 'unthrottled write I/O pool', 'iops', 'operations per second');
1071
1072 # burst lengths
1073 $add_throttle_desc->('bps_max_length',     'integer', 'length of I/O bursts',       'seconds', 'seconds', 1);
1074 $add_throttle_desc->('bps_rd_max_length',  'integer', 'length of read I/O bursts',  'seconds', 'seconds', 1);
1075 $add_throttle_desc->('bps_wr_max_length',  'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
1076 $add_throttle_desc->('iops_max_length',    'integer', 'length of I/O bursts',       'seconds', 'seconds', 1);
1077 $add_throttle_desc->('iops_rd_max_length', 'integer', 'length of read I/O bursts',  'seconds', 'seconds', 1);
1078 $add_throttle_desc->('iops_wr_max_length', 'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
1079
1080 # legacy support
1081 $drivedesc_base{'bps_rd_length'} = { alias => 'bps_rd_max_length' };
1082 $drivedesc_base{'bps_wr_length'} = { alias => 'bps_wr_max_length' };
1083 $drivedesc_base{'iops_rd_length'} = { alias => 'iops_rd_max_length' };
1084 $drivedesc_base{'iops_wr_length'} = { alias => 'iops_wr_max_length' };
1085
1086 my $ide_fmt = {
1087     %drivedesc_base,
1088     %model_fmt,
1089     %ssd_fmt,
1090 };
1091 PVE::JSONSchema::register_format("pve-qm-ide", $ide_fmt);
1092
1093 my $idedesc = {
1094     optional => 1,
1095     type => 'string', format => $ide_fmt,
1096     description => "Use volume as IDE hard disk or CD-ROM (n is 0 to " .($MAX_IDE_DISKS -1) . ").",
1097 };
1098 PVE::JSONSchema::register_standard_option("pve-qm-ide", $idedesc);
1099
1100 my $scsi_fmt = {
1101     %drivedesc_base,
1102     %iothread_fmt,
1103     %queues_fmt,
1104     %scsiblock_fmt,
1105     %ssd_fmt,
1106 };
1107 my $scsidesc = {
1108     optional => 1,
1109     type => 'string', format => $scsi_fmt,
1110     description => "Use volume as SCSI hard disk or CD-ROM (n is 0 to " . ($MAX_SCSI_DISKS - 1) . ").",
1111 };
1112 PVE::JSONSchema::register_standard_option("pve-qm-scsi", $scsidesc);
1113
1114 my $sata_fmt = {
1115     %drivedesc_base,
1116     %ssd_fmt,
1117 };
1118 my $satadesc = {
1119     optional => 1,
1120     type => 'string', format => $sata_fmt,
1121     description => "Use volume as SATA hard disk or CD-ROM (n is 0 to " . ($MAX_SATA_DISKS - 1). ").",
1122 };
1123 PVE::JSONSchema::register_standard_option("pve-qm-sata", $satadesc);
1124
1125 my $virtio_fmt = {
1126     %drivedesc_base,
1127     %iothread_fmt,
1128 };
1129 my $virtiodesc = {
1130     optional => 1,
1131     type => 'string', format => $virtio_fmt,
1132     description => "Use volume as VIRTIO hard disk (n is 0 to " . ($MAX_VIRTIO_DISKS - 1) . ").",
1133 };
1134 PVE::JSONSchema::register_standard_option("pve-qm-virtio", $virtiodesc);
1135
1136 my $alldrive_fmt = {
1137     %drivedesc_base,
1138     %iothread_fmt,
1139     %model_fmt,
1140     %queues_fmt,
1141     %scsiblock_fmt,
1142     %ssd_fmt,
1143 };
1144
1145 my $efidisk_fmt = {
1146     volume => { alias => 'file' },
1147     file => {
1148         type => 'string',
1149         format => 'pve-volume-id-or-qm-path',
1150         default_key => 1,
1151         format_description => 'volume',
1152         description => "The drive's backing volume.",
1153     },
1154     format => get_standard_option('pve-qm-image-format'),
1155     size => {
1156         type => 'string',
1157         format => 'disk-size',
1158         format_description => 'DiskSize',
1159         description => "Disk size. This is purely informational and has no effect.",
1160         optional => 1,
1161     },
1162 };
1163
1164 my $efidisk_desc = {
1165     optional => 1,
1166     type => 'string', format => $efidisk_fmt,
1167     description => "Configure a Disk for storing EFI vars",
1168 };
1169
1170 PVE::JSONSchema::register_standard_option("pve-qm-efidisk", $efidisk_desc);
1171
1172 my $usb_fmt = {
1173     host => {
1174         default_key => 1,
1175         type => 'string', format => 'pve-qm-usb-device',
1176         format_description => 'HOSTUSBDEVICE|spice',
1177         description => <<EODESCR,
1178 The Host USB device or port or the value 'spice'. HOSTUSBDEVICE syntax is:
1179
1180  'bus-port(.port)*' (decimal numbers) or
1181  'vendor_id:product_id' (hexadeciaml numbers) or
1182  'spice'
1183
1184 You can use the 'lsusb -t' command to list existing usb devices.
1185
1186 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1187
1188 The value 'spice' can be used to add a usb redirection devices for spice.
1189 EODESCR
1190     },
1191     usb3 => {
1192         optional => 1,
1193         type => 'boolean',
1194         description => "Specifies whether if given host option is a USB3 device or port (this does currently not work reliably with spice redirection and is then ignored).",
1195         default => 0,
1196     },
1197 };
1198
1199 my $usbdesc = {
1200     optional => 1,
1201     type => 'string', format => $usb_fmt,
1202     description => "Configure an USB device (n is 0 to 4).",
1203 };
1204 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
1205
1206 # NOTE: the match-groups of this regex are used in parse_hostpci
1207 my $PCIRE = qr/([a-f0-9]{2}:[a-f0-9]{2})(?:\.([a-f0-9]))?/;
1208 my $hostpci_fmt = {
1209     host => {
1210         default_key => 1,
1211         type => 'string',
1212         pattern => qr/$PCIRE(;$PCIRE)*/,
1213         format_description => 'HOSTPCIID[;HOSTPCIID2...]',
1214         description => <<EODESCR,
1215 Host PCI device pass through. The PCI ID of a host's PCI device or a list
1216 of PCI virtual functions of the host. HOSTPCIID syntax is:
1217
1218 'bus:dev.func' (hexadecimal numbers)
1219
1220 You can us the 'lspci' command to list existing PCI devices.
1221 EODESCR
1222     },
1223     rombar => {
1224         type => 'boolean',
1225         description =>  "Specify whether or not the device's ROM will be visible in the guest's memory map.",
1226         optional => 1,
1227         default => 1,
1228     },
1229     romfile => {
1230         type => 'string',
1231         pattern => '[^,;]+',
1232         format_description => 'string',
1233         description => "Custom pci device rom filename (must be located in /usr/share/kvm/).",
1234         optional => 1,
1235     },
1236     pcie => {
1237         type => 'boolean',
1238         description =>  "Choose the PCI-express bus (needs the 'q35' machine model).",
1239         optional => 1,
1240         default => 0,
1241     },
1242     'x-vga' => {
1243         type => 'boolean',
1244         description =>  "Enable vfio-vga device support.",
1245         optional => 1,
1246         default => 0,
1247     },
1248 };
1249 PVE::JSONSchema::register_format('pve-qm-hostpci', $hostpci_fmt);
1250
1251 my $hostpcidesc = {
1252         optional => 1,
1253         type => 'string', format => 'pve-qm-hostpci',
1254         description => "Map host PCI devices into guest.",
1255         verbose_description =>  <<EODESCR,
1256 Map host PCI devices into guest.
1257
1258 NOTE: This option allows direct access to host hardware. So it is no longer
1259 possible to migrate such machines - use with special care.
1260
1261 CAUTION: Experimental! User reported problems with this option.
1262 EODESCR
1263 };
1264 PVE::JSONSchema::register_standard_option("pve-qm-hostpci", $hostpcidesc);
1265
1266 my $serialdesc = {
1267         optional => 1,
1268         type => 'string',
1269         pattern => '(/dev/.+|socket)',
1270         description =>  "Create a serial device inside the VM (n is 0 to 3)",
1271         verbose_description =>  <<EODESCR,
1272 Create a serial device inside the VM (n is 0 to 3), and pass through a
1273 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1274 host side (use 'qm terminal' to open a terminal connection).
1275
1276 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines - use with special care.
1277
1278 CAUTION: Experimental! User reported problems with this option.
1279 EODESCR
1280 };
1281
1282 my $paralleldesc= {
1283         optional => 1,
1284         type => 'string',
1285         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
1286         description =>  "Map host parallel devices (n is 0 to 2).",
1287         verbose_description =>  <<EODESCR,
1288 Map host parallel devices (n is 0 to 2).
1289
1290 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1291
1292 CAUTION: Experimental! User reported problems with this option.
1293 EODESCR
1294 };
1295
1296 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
1297     $confdesc->{"parallel$i"} = $paralleldesc;
1298 }
1299
1300 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
1301     $confdesc->{"serial$i"} = $serialdesc;
1302 }
1303
1304 for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
1305     $confdesc->{"hostpci$i"} = $hostpcidesc;
1306 }
1307
1308 for (my $i = 0; $i < $MAX_IDE_DISKS; $i++)  {
1309     $drivename_hash->{"ide$i"} = 1;
1310     $confdesc->{"ide$i"} = $idedesc;
1311 }
1312
1313 for (my $i = 0; $i < $MAX_SATA_DISKS; $i++)  {
1314     $drivename_hash->{"sata$i"} = 1;
1315     $confdesc->{"sata$i"} = $satadesc;
1316 }
1317
1318 for (my $i = 0; $i < $MAX_SCSI_DISKS; $i++)  {
1319     $drivename_hash->{"scsi$i"} = 1;
1320     $confdesc->{"scsi$i"} = $scsidesc ;
1321 }
1322
1323 for (my $i = 0; $i < $MAX_VIRTIO_DISKS; $i++)  {
1324     $drivename_hash->{"virtio$i"} = 1;
1325     $confdesc->{"virtio$i"} = $virtiodesc;
1326 }
1327
1328 $drivename_hash->{efidisk0} = 1;
1329 $confdesc->{efidisk0} = $efidisk_desc;
1330
1331 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
1332     $confdesc->{"usb$i"} = $usbdesc;
1333 }
1334
1335 my $unuseddesc = {
1336     optional => 1,
1337     type => 'string', format => 'pve-volume-id',
1338     description => "Reference to unused volumes. This is used internally, and should not be modified manually.",
1339 };
1340
1341 for (my $i = 0; $i < $MAX_UNUSED_DISKS; $i++)  {
1342     $confdesc->{"unused$i"} = $unuseddesc;
1343 }
1344
1345 my $kvm_api_version = 0;
1346
1347 sub kvm_version {
1348     return $kvm_api_version if $kvm_api_version;
1349
1350     open my $fh, '<', '/dev/kvm'
1351         or return undef;
1352
1353     # 0xae00 => KVM_GET_API_VERSION
1354     $kvm_api_version = ioctl($fh, 0xae00, 0);
1355
1356     return $kvm_api_version;
1357 }
1358
1359 my $kvm_user_version;
1360
1361 sub kvm_user_version {
1362
1363     return $kvm_user_version if $kvm_user_version;
1364
1365     $kvm_user_version = 'unknown';
1366
1367     my $code = sub {
1368         my $line = shift;
1369         if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1370             $kvm_user_version = $2;
1371         }
1372     };
1373
1374     eval { run_command("kvm -version", outfunc => $code); };
1375     warn $@ if $@;
1376
1377     return $kvm_user_version;
1378
1379 }
1380
1381 my $kernel_has_vhost_net = -c '/dev/vhost-net';
1382
1383 sub valid_drive_names {
1384     # order is important - used to autoselect boot disk
1385     return ((map { "ide$_" } (0 .. ($MAX_IDE_DISKS - 1))),
1386             (map { "scsi$_" } (0 .. ($MAX_SCSI_DISKS - 1))),
1387             (map { "virtio$_" } (0 .. ($MAX_VIRTIO_DISKS - 1))),
1388             (map { "sata$_" } (0 .. ($MAX_SATA_DISKS - 1))),
1389             'efidisk0');
1390 }
1391
1392 sub is_valid_drivename {
1393     my $dev = shift;
1394
1395     return defined($drivename_hash->{$dev});
1396 }
1397
1398 sub option_exists {
1399     my $key = shift;
1400     return defined($confdesc->{$key});
1401 }
1402
1403 sub nic_models {
1404     return $nic_model_list;
1405 }
1406
1407 sub os_list_description {
1408
1409     return {
1410         other => 'Other',
1411         wxp => 'Windows XP',
1412         w2k => 'Windows 2000',
1413         w2k3 =>, 'Windows 2003',
1414         w2k8 => 'Windows 2008',
1415         wvista => 'Windows Vista',
1416         win7 => 'Windows 7',
1417         win8 => 'Windows 8/2012',
1418         win10 => 'Windows 10/2016',
1419         l24 => 'Linux 2.4',
1420         l26 => 'Linux 2.6',
1421     };
1422 }
1423
1424 my $cdrom_path;
1425
1426 sub get_cdrom_path {
1427
1428     return  $cdrom_path if $cdrom_path;
1429
1430     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
1431     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
1432     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
1433 }
1434
1435 sub get_iso_path {
1436     my ($storecfg, $vmid, $cdrom) = @_;
1437
1438     if ($cdrom eq 'cdrom') {
1439         return get_cdrom_path();
1440     } elsif ($cdrom eq 'none') {
1441         return '';
1442     } elsif ($cdrom =~ m|^/|) {
1443         return $cdrom;
1444     } else {
1445         return PVE::Storage::path($storecfg, $cdrom);
1446     }
1447 }
1448
1449 # try to convert old style file names to volume IDs
1450 sub filename_to_volume_id {
1451     my ($vmid, $file, $media) = @_;
1452
1453      if (!($file eq 'none' || $file eq 'cdrom' ||
1454           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1455
1456         return undef if $file =~ m|/|;
1457
1458         if ($media && $media eq 'cdrom') {
1459             $file = "local:iso/$file";
1460         } else {
1461             $file = "local:$vmid/$file";
1462         }
1463     }
1464
1465     return $file;
1466 }
1467
1468 sub verify_media_type {
1469     my ($opt, $vtype, $media) = @_;
1470
1471     return if !$media;
1472
1473     my $etype;
1474     if ($media eq 'disk') {
1475         $etype = 'images';
1476     } elsif ($media eq 'cdrom') {
1477         $etype = 'iso';
1478     } else {
1479         die "internal error";
1480     }
1481
1482     return if ($vtype eq $etype);
1483
1484     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1485 }
1486
1487 sub cleanup_drive_path {
1488     my ($opt, $storecfg, $drive) = @_;
1489
1490     # try to convert filesystem paths to volume IDs
1491
1492     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1493         ($drive->{file} !~ m|^/dev/.+|) &&
1494         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1495         ($drive->{file} !~ m/^\d+$/)) {
1496         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1497         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"}) if !$vtype;
1498         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1499         verify_media_type($opt, $vtype, $drive->{media});
1500         $drive->{file} = $volid;
1501     }
1502
1503     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1504 }
1505
1506 sub parse_hotplug_features {
1507     my ($data) = @_;
1508
1509     my $res = {};
1510
1511     return $res if $data eq '0';
1512
1513     $data = $confdesc->{hotplug}->{default} if $data eq '1';
1514
1515     foreach my $feature (PVE::Tools::split_list($data)) {
1516         if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
1517             $res->{$1} = 1;
1518         } else {
1519             die "invalid hotplug feature '$feature'\n";
1520         }
1521     }
1522     return $res;
1523 }
1524
1525 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1526 sub pve_verify_hotplug_features {
1527     my ($value, $noerr) = @_;
1528
1529     return $value if parse_hotplug_features($value);
1530
1531     return undef if $noerr;
1532
1533     die "unable to parse hotplug option\n";
1534 }
1535
1536 # ideX = [volume=]volume-id[,media=d][,cyls=c,heads=h,secs=s[,trans=t]]
1537 #        [,snapshot=on|off][,cache=on|off][,format=f][,backup=yes|no]
1538 #        [,rerror=ignore|report|stop][,werror=enospc|ignore|report|stop]
1539 #        [,aio=native|threads][,discard=ignore|on][,detect_zeroes=on|off]
1540 #        [,iothread=on][,serial=serial][,model=model]
1541
1542 sub parse_drive {
1543     my ($key, $data) = @_;
1544
1545     my ($interface, $index);
1546
1547     if ($key =~ m/^([^\d]+)(\d+)$/) {
1548         $interface = $1;
1549         $index = $2;
1550     } else {
1551         return undef;
1552     }
1553
1554     my $desc = $key =~ /^unused\d+$/ ? $alldrive_fmt
1555                                      : $confdesc->{$key}->{format};
1556     if (!$desc) {
1557         warn "invalid drive key: $key\n";
1558         return undef;
1559     }
1560     my $res = eval { PVE::JSONSchema::parse_property_string($desc, $data) };
1561     return undef if !$res;
1562     $res->{interface} = $interface;
1563     $res->{index} = $index;
1564
1565     my $error = 0;
1566     foreach my $opt (qw(bps bps_rd bps_wr)) {
1567         if (my $bps = defined(delete $res->{$opt})) {
1568             if (defined($res->{"m$opt"})) {
1569                 warn "both $opt and m$opt specified\n";
1570                 ++$error;
1571                 next;
1572             }
1573             $res->{"m$opt"} = sprintf("%.3f", $bps / (1024*1024.0));
1574         }
1575     }
1576
1577     # can't use the schema's 'requires' because of the mbps* => bps* "transforming aliases"
1578     for my $requirement (
1579         [mbps_max => 'mbps'],
1580         [mbps_rd_max => 'mbps_rd'],
1581         [mbps_wr_max => 'mbps_wr'],
1582         [miops_max => 'miops'],
1583         [miops_rd_max => 'miops_rd'],
1584         [miops_wr_max => 'miops_wr'],
1585         [bps_max_length => 'mbps_max'],
1586         [bps_rd_max_length => 'mbps_rd_max'],
1587         [bps_wr_max_length => 'mbps_wr_max'],
1588         [iops_max_length => 'iops_max'],
1589         [iops_rd_max_length => 'iops_rd_max'],
1590         [iops_wr_max_length => 'iops_wr_max']) {
1591         my ($option, $requires) = @$requirement;
1592         if ($res->{$option} && !$res->{$requires}) {
1593             warn "$option requires $requires\n";
1594             ++$error;
1595         }
1596     }
1597
1598     return undef if $error;
1599
1600     return undef if $res->{mbps_rd} && $res->{mbps};
1601     return undef if $res->{mbps_wr} && $res->{mbps};
1602     return undef if $res->{iops_rd} && $res->{iops};
1603     return undef if $res->{iops_wr} && $res->{iops};
1604
1605     if ($res->{media} && ($res->{media} eq 'cdrom')) {
1606         return undef if $res->{snapshot} || $res->{trans} || $res->{format};
1607         return undef if $res->{heads} || $res->{secs} || $res->{cyls};
1608         return undef if $res->{interface} eq 'virtio';
1609     }
1610
1611     if (my $size = $res->{size}) {
1612         return undef if !defined($res->{size} = PVE::JSONSchema::parse_size($size));
1613     }
1614
1615     return $res;
1616 }
1617
1618 sub print_drive {
1619     my ($vmid, $drive) = @_;
1620     my $data = { %$drive };
1621     delete $data->{$_} for qw(index interface);
1622     return PVE::JSONSchema::print_property_string($data, $alldrive_fmt);
1623 }
1624
1625 sub scsi_inquiry {
1626     my($fh, $noerr) = @_;
1627
1628     my $SG_IO = 0x2285;
1629     my $SG_GET_VERSION_NUM = 0x2282;
1630
1631     my $versionbuf = "\x00" x 8;
1632     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1633     if (!$ret) {
1634         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1635         return undef;
1636     }
1637     my $version = unpack("I", $versionbuf);
1638     if ($version < 30000) {
1639         die "scsi generic interface too old\n"  if !$noerr;
1640         return undef;
1641     }
1642
1643     my $buf = "\x00" x 36;
1644     my $sensebuf = "\x00" x 8;
1645     my $cmd = pack("C x3 C x1", 0x12, 36);
1646
1647     # see /usr/include/scsi/sg.h
1648     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1649
1650     my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1651                       length($sensebuf), 0, length($buf), $buf,
1652                       $cmd, $sensebuf, 6000);
1653
1654     $ret = ioctl($fh, $SG_IO, $packet);
1655     if (!$ret) {
1656         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1657         return undef;
1658     }
1659
1660     my @res = unpack($sg_io_hdr_t, $packet);
1661     if ($res[17] || $res[18]) {
1662         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1663         return undef;
1664     }
1665
1666     my $res = {};
1667     (my $byte0, my $byte1, $res->{vendor},
1668      $res->{product}, $res->{revision}) = unpack("C C x6 A8 A16 A4", $buf);
1669
1670     $res->{removable} = $byte1 & 128 ? 1 : 0;
1671     $res->{type} = $byte0 & 31;
1672
1673     return $res;
1674 }
1675
1676 sub path_is_scsi {
1677     my ($path) = @_;
1678
1679     my $fh = IO::File->new("+<$path") || return undef;
1680     my $res = scsi_inquiry($fh, 1);
1681     close($fh);
1682
1683     return $res;
1684 }
1685
1686 sub machine_type_is_q35 {
1687     my ($conf) = @_;
1688
1689     return $conf->{machine} && ($conf->{machine} =~ m/q35/) ? 1 : 0;
1690 }
1691
1692 sub print_tabletdevice_full {
1693     my ($conf) = @_;
1694
1695     my $q35 = machine_type_is_q35($conf);
1696
1697     # we use uhci for old VMs because tablet driver was buggy in older qemu
1698     my $usbbus = $q35 ? "ehci" : "uhci";
1699
1700     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1701 }
1702
1703 sub print_drivedevice_full {
1704     my ($storecfg, $conf, $vmid, $drive, $bridges) = @_;
1705
1706     my $device = '';
1707     my $maxdev = 0;
1708
1709     if ($drive->{interface} eq 'virtio') {
1710         my $pciaddr = print_pci_addr("$drive->{interface}$drive->{index}", $bridges);
1711         $device = "virtio-blk-pci,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}$pciaddr";
1712         $device .= ",iothread=iothread-$drive->{interface}$drive->{index}" if $drive->{iothread};
1713     } elsif ($drive->{interface} eq 'scsi') {
1714
1715         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1716         my $unit = $drive->{index} % $maxdev;
1717         my $devicetype = 'hd';
1718         my $path = '';
1719         if (drive_is_cdrom($drive)) {
1720             $devicetype = 'cd';
1721         } else {
1722             if ($drive->{file} =~ m|^/|) {
1723                 $path = $drive->{file};
1724                 if (my $info = path_is_scsi($path)) {
1725                     if ($info->{type} == 0 && $drive->{scsiblock}) {
1726                         $devicetype = 'block';
1727                     } elsif ($info->{type} == 1) { # tape
1728                         $devicetype = 'generic';
1729                     }
1730                 }
1731             } else {
1732                  $path = PVE::Storage::path($storecfg, $drive->{file});
1733             }
1734
1735             if($path =~ m/^iscsi\:\/\//){
1736                 $devicetype = 'generic';
1737             }
1738         }
1739
1740         if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)){
1741             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1742         } else {
1743             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0,lun=$drive->{index},drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1744         }
1745
1746         if ($drive->{ssd} && ($devicetype eq 'block' || $devicetype eq 'hd')) {
1747             $device .= ",rotation_rate=1";
1748         }
1749
1750     } elsif ($drive->{interface} eq 'ide' || $drive->{interface} eq 'sata') {
1751         my $maxdev = ($drive->{interface} eq 'sata') ? $MAX_SATA_DISKS : 2;
1752         my $controller = int($drive->{index} / $maxdev);
1753         my $unit = $drive->{index} % $maxdev;
1754         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1755
1756         $device = "ide-$devicetype";
1757         if ($drive->{interface} eq 'ide') {
1758             $device .= ",bus=ide.$controller,unit=$unit";
1759         } else {
1760             $device .= ",bus=ahci$controller.$unit";
1761         }
1762         $device .= ",drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1763
1764         if ($devicetype eq 'hd') {
1765             if (my $model = $drive->{model}) {
1766                 $model = URI::Escape::uri_unescape($model);
1767                 $device .= ",model=$model";
1768             }
1769             if ($drive->{ssd}) {
1770                 $device .= ",rotation_rate=1";
1771             }
1772         }
1773     } elsif ($drive->{interface} eq 'usb') {
1774         die "implement me";
1775         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1776     } else {
1777         die "unsupported interface type";
1778     }
1779
1780     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1781
1782     if (my $serial = $drive->{serial}) {
1783         $serial = URI::Escape::uri_unescape($serial);
1784         $device .= ",serial=$serial";
1785     }
1786
1787
1788     return $device;
1789 }
1790
1791 sub get_initiator_name {
1792     my $initiator;
1793
1794     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return undef;
1795     while (defined(my $line = <$fh>)) {
1796         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1797         $initiator = $1;
1798         last;
1799     }
1800     $fh->close();
1801
1802     return $initiator;
1803 }
1804
1805 sub print_drive_full {
1806     my ($storecfg, $vmid, $drive) = @_;
1807
1808     my $path;
1809     my $volid = $drive->{file};
1810     my $format;
1811
1812     if (drive_is_cdrom($drive)) {
1813         $path = get_iso_path($storecfg, $vmid, $volid);
1814     } else {
1815         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1816         if ($storeid) {
1817             $path = PVE::Storage::path($storecfg, $volid);
1818             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
1819             $format = qemu_img_format($scfg, $volname);
1820         } else {
1821             $path = $volid;
1822             $format = "raw";
1823         }
1824    }
1825
1826     my $opts = '';
1827     my @qemu_drive_options = qw(heads secs cyls trans media format cache rerror werror aio discard);
1828     foreach my $o (@qemu_drive_options) {
1829         $opts .= ",$o=$drive->{$o}" if defined($drive->{$o});
1830     }
1831
1832     # snapshot only accepts on|off
1833     if (defined($drive->{snapshot})) {
1834         my $v = $drive->{snapshot} ? 'on' : 'off';
1835         $opts .= ",snapshot=$v";
1836     }
1837
1838     foreach my $type (['', '-total'], [_rd => '-read'], [_wr => '-write']) {
1839         my ($dir, $qmpname) = @$type;
1840         if (my $v = $drive->{"mbps$dir"}) {
1841             $opts .= ",throttling.bps$qmpname=".int($v*1024*1024);
1842         }
1843         if (my $v = $drive->{"mbps${dir}_max"}) {
1844             $opts .= ",throttling.bps$qmpname-max=".int($v*1024*1024);
1845         }
1846         if (my $v = $drive->{"bps${dir}_max_length"}) {
1847             $opts .= ",throttling.bps$qmpname-max-length=$v";
1848         }
1849         if (my $v = $drive->{"iops${dir}"}) {
1850             $opts .= ",throttling.iops$qmpname=$v";
1851         }
1852         if (my $v = $drive->{"iops${dir}_max"}) {
1853             $opts .= ",throttling.iops$qmpname-max=$v";
1854         }
1855         if (my $v = $drive->{"iops${dir}_max_length"}) {
1856             $opts .= ",throttling.iops$qmpname-max-length=$v";
1857         }
1858     }
1859
1860     $opts .= ",format=$format" if $format && !$drive->{format};
1861
1862     my $cache_direct = 0;
1863
1864     if (my $cache = $drive->{cache}) {
1865         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1866     } elsif (!drive_is_cdrom($drive)) {
1867         $opts .= ",cache=none";
1868         $cache_direct = 1;
1869     }
1870
1871     # aio native works only with O_DIRECT
1872     if (!$drive->{aio}) {
1873         if($cache_direct) {
1874             $opts .= ",aio=native";
1875         } else {
1876             $opts .= ",aio=threads";
1877         }
1878     }
1879
1880     if (!drive_is_cdrom($drive)) {
1881         my $detectzeroes;
1882         if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
1883             $detectzeroes = 'off';
1884         } elsif ($drive->{discard}) {
1885             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
1886         } else {
1887             # This used to be our default with discard not being specified:
1888             $detectzeroes = 'on';
1889         }
1890         $opts .= ",detect-zeroes=$detectzeroes" if $detectzeroes;
1891     }
1892
1893     my $pathinfo = $path ? "file=$path," : '';
1894
1895     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1896 }
1897
1898 sub print_netdevice_full {
1899     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files) = @_;
1900
1901     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
1902
1903     my $device = $net->{model};
1904     if ($net->{model} eq 'virtio') {
1905          $device = 'virtio-net-pci';
1906      };
1907
1908     my $pciaddr = print_pci_addr("$netid", $bridges);
1909     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1910     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
1911         #Consider we have N queues, the number of vectors needed is 2*N + 2 (plus one config interrupt and control vq)
1912         my $vectors = $net->{queues} * 2 + 2;
1913         $tmpstr .= ",vectors=$vectors,mq=on";
1914     }
1915     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
1916
1917     if ($use_old_bios_files) {
1918         my $romfile;
1919         if ($device eq 'virtio-net-pci') {
1920             $romfile = 'pxe-virtio.rom';
1921         } elsif ($device eq 'e1000') {
1922             $romfile = 'pxe-e1000.rom';
1923         } elsif ($device eq 'ne2k') {
1924             $romfile = 'pxe-ne2k_pci.rom';
1925         } elsif ($device eq 'pcnet') {
1926             $romfile = 'pxe-pcnet.rom';
1927         } elsif ($device eq 'rtl8139') {
1928             $romfile = 'pxe-rtl8139.rom';
1929         }
1930         $tmpstr .= ",romfile=$romfile" if $romfile;
1931     }
1932
1933     return $tmpstr;
1934 }
1935
1936 sub print_netdev_full {
1937     my ($vmid, $conf, $net, $netid, $hotplug) = @_;
1938
1939     my $i = '';
1940     if ($netid =~ m/^net(\d+)$/) {
1941         $i = int($1);
1942     }
1943
1944     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
1945
1946     my $ifname = "tap${vmid}i$i";
1947
1948     # kvm uses TUNSETIFF ioctl, and that limits ifname length
1949     die "interface name '$ifname' is too long (max 15 character)\n"
1950         if length($ifname) >= 16;
1951
1952     my $vhostparam = '';
1953     $vhostparam = ',vhost=on' if $kernel_has_vhost_net && $net->{model} eq 'virtio';
1954
1955     my $vmname = $conf->{name} || "vm$vmid";
1956
1957     my $netdev = "";
1958     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
1959
1960     if ($net->{bridge}) {
1961         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script,downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1962     } else {
1963         $netdev = "type=user,id=$netid,hostname=$vmname";
1964     }
1965
1966     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
1967
1968     return $netdev;
1969 }
1970
1971
1972 sub print_cpu_device {
1973     my ($conf, $id) = @_;
1974
1975     my $kvm = $conf->{kvm} // 1;
1976     my $cpu = $kvm ? "kvm64" : "qemu64";
1977     if (my $cputype = $conf->{cpu}) {
1978         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
1979             or die "Cannot parse cpu description: $cputype\n";
1980         $cpu = $cpuconf->{cputype};
1981     }
1982
1983     my $cores = $conf->{cores} || 1;
1984
1985     my $current_core = ($id - 1) % $cores;
1986     my $current_socket = int(($id - 1 - $current_core)/$cores);
1987
1988     return "$cpu-x86_64-cpu,id=cpu$id,socket-id=$current_socket,core-id=$current_core,thread-id=0";
1989 }
1990
1991 my $vga_map = {
1992     'cirrus' => 'cirrus-vga',
1993     'std' => 'VGA',
1994     'vmware' => 'vmware-svga',
1995     'virtio' => 'virtio-vga',
1996 };
1997
1998 sub print_vga_device {
1999     my ($conf, $vga, $id, $qxlnum, $bridges) = @_;
2000
2001     my $type = $vga_map->{$vga->{type}};
2002     my $vgamem_mb = $vga->{memory};
2003     if ($qxlnum) {
2004         $type = $id ? 'qxl' : 'qxl-vga';
2005     }
2006     die "no devicetype for $vga->{type}\n" if !$type;
2007
2008     my $memory = "";
2009     if ($vgamem_mb) {
2010         if ($vga->{type} eq 'virtio') {
2011             my $bytes = PVE::Tools::convert_size($vgamem_mb, "mb" => "b");
2012             $memory = ",max_hostmem=$bytes";
2013         } elsif ($qxlnum) {
2014             # from https://www.spice-space.org/multiple-monitors.html
2015             $memory = ",vgamem_mb=$vga->{memory}";
2016             my $ram = $vgamem_mb * 4;
2017             my $vram = $vgamem_mb * 2;
2018             $memory .= ",ram_size_mb=$ram,vram_size_mb=$vram";
2019         } else {
2020             $memory = ",vgamem_mb=$vga->{memory}";
2021         }
2022     } elsif ($qxlnum && $id) {
2023         $memory = ",ram_size=67108864,vram_size=33554432";
2024     }
2025
2026     my $q35 = machine_type_is_q35($conf);
2027     my $vgaid = "vga" . ($id // '');
2028     my $pciaddr;
2029
2030     if ($q35 && $vgaid eq 'vga') {
2031         # the first display uses pcie.0 bus on q35 machines
2032         $pciaddr = print_pcie_addr($vgaid, $bridges);
2033     } else {
2034         $pciaddr = print_pci_addr($vgaid, $bridges);
2035     }
2036
2037     return "$type,id=${vgaid}${memory}${pciaddr}";
2038 }
2039
2040 sub drive_is_cloudinit {
2041     my ($drive) = @_;
2042     return $drive->{file} =~ m@[:/]vm-\d+-cloudinit(?:\.$QEMU_FORMAT_RE)?$@;
2043 }
2044
2045 sub drive_is_cdrom {
2046     my ($drive, $exclude_cloudinit) = @_;
2047
2048     return 0 if $exclude_cloudinit && drive_is_cloudinit($drive);
2049
2050     return $drive && $drive->{media} && ($drive->{media} eq 'cdrom');
2051
2052 }
2053
2054 sub parse_number_sets {
2055     my ($set) = @_;
2056     my $res = [];
2057     foreach my $part (split(/;/, $set)) {
2058         if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
2059             die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
2060             push @$res, [ $1, $2 ];
2061         } else {
2062             die "invalid range: $part\n";
2063         }
2064     }
2065     return $res;
2066 }
2067
2068 sub parse_numa {
2069     my ($data) = @_;
2070
2071     my $res = PVE::JSONSchema::parse_property_string($numa_fmt, $data);
2072     $res->{cpus} = parse_number_sets($res->{cpus}) if defined($res->{cpus});
2073     $res->{hostnodes} = parse_number_sets($res->{hostnodes}) if defined($res->{hostnodes});
2074     return $res;
2075 }
2076
2077 sub parse_hostpci {
2078     my ($value) = @_;
2079
2080     return undef if !$value;
2081
2082     my $res = PVE::JSONSchema::parse_property_string($hostpci_fmt, $value);
2083
2084     my @idlist = split(/;/, $res->{host});
2085     delete $res->{host};
2086     foreach my $id (@idlist) {
2087         if ($id =~ /^$PCIRE$/) {
2088             if (defined($2)) {
2089                 push @{$res->{pciid}}, { id => $1, function => $2 };
2090             } else {
2091                 my $pcidevices = lspci($1);
2092                 $res->{pciid} = $pcidevices->{$1};
2093             }
2094         } else {
2095             # should have been caught by parse_property_string already
2096             die "failed to parse PCI id: $id\n";
2097         }
2098     }
2099     return $res;
2100 }
2101
2102 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
2103 sub parse_net {
2104     my ($data) = @_;
2105
2106     my $res = eval { PVE::JSONSchema::parse_property_string($net_fmt, $data) };
2107     if ($@) {
2108         warn $@;
2109         return undef;
2110     }
2111     if (!defined($res->{macaddr})) {
2112         my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
2113         $res->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix});
2114     }
2115     return $res;
2116 }
2117
2118 # ipconfigX ip=cidr,gw=ip,ip6=cidr,gw6=ip
2119 sub parse_ipconfig {
2120     my ($data) = @_;
2121
2122     my $res = eval { PVE::JSONSchema::parse_property_string($ipconfig_fmt, $data) };
2123     if ($@) {
2124         warn $@;
2125         return undef;
2126     }
2127
2128     if ($res->{gw} && !$res->{ip}) {
2129         warn 'gateway specified without specifying an IP address';
2130         return undef;
2131     }
2132     if ($res->{gw6} && !$res->{ip6}) {
2133         warn 'IPv6 gateway specified without specifying an IPv6 address';
2134         return undef;
2135     }
2136     if ($res->{gw} && $res->{ip} eq 'dhcp') {
2137         warn 'gateway specified together with DHCP';
2138         return undef;
2139     }
2140     if ($res->{gw6} && $res->{ip6} !~ /^$IPV6RE/) {
2141         # gw6 + auto/dhcp
2142         warn "IPv6 gateway specified together with $res->{ip6} address";
2143         return undef;
2144     }
2145
2146     if (!$res->{ip} && !$res->{ip6}) {
2147         return { ip => 'dhcp', ip6 => 'dhcp' };
2148     }
2149
2150     return $res;
2151 }
2152
2153 sub print_net {
2154     my $net = shift;
2155
2156     return PVE::JSONSchema::print_property_string($net, $net_fmt);
2157 }
2158
2159 sub add_random_macs {
2160     my ($settings) = @_;
2161
2162     foreach my $opt (keys %$settings) {
2163         next if $opt !~ m/^net(\d+)$/;
2164         my $net = parse_net($settings->{$opt});
2165         next if !$net;
2166         $settings->{$opt} = print_net($net);
2167     }
2168 }
2169
2170 sub vm_is_volid_owner {
2171     my ($storecfg, $vmid, $volid) = @_;
2172
2173     if ($volid !~  m|^/|) {
2174         my ($path, $owner);
2175         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
2176         if ($owner && ($owner == $vmid)) {
2177             return 1;
2178         }
2179     }
2180
2181     return undef;
2182 }
2183
2184 sub split_flagged_list {
2185     my $text = shift || '';
2186     $text =~ s/[,;]/ /g;
2187     $text =~ s/^\s+//;
2188     return { map { /^(!?)(.*)$/ && ($2, $1) } ($text =~ /\S+/g) };
2189 }
2190
2191 sub join_flagged_list {
2192     my ($how, $lst) = @_;
2193     join $how, map { $lst->{$_} . $_ } keys %$lst;
2194 }
2195
2196 sub vmconfig_delete_pending_option {
2197     my ($conf, $key, $force) = @_;
2198
2199     delete $conf->{pending}->{$key};
2200     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
2201     $pending_delete_hash->{$key} = $force ? '!' : '';
2202     $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
2203 }
2204
2205 sub vmconfig_undelete_pending_option {
2206     my ($conf, $key) = @_;
2207
2208     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
2209     delete $pending_delete_hash->{$key};
2210
2211     if (%$pending_delete_hash) {
2212         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
2213     } else {
2214         delete $conf->{pending}->{delete};
2215     }
2216 }
2217
2218 sub vmconfig_register_unused_drive {
2219     my ($storecfg, $vmid, $conf, $drive) = @_;
2220
2221     if (drive_is_cloudinit($drive)) {
2222         eval { PVE::Storage::vdisk_free($storecfg, $drive->{file}) };
2223         warn $@ if $@;
2224     } elsif (!drive_is_cdrom($drive)) {
2225         my $volid = $drive->{file};
2226         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
2227             PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
2228         }
2229     }
2230 }
2231
2232 sub vmconfig_cleanup_pending {
2233     my ($conf) = @_;
2234
2235     # remove pending changes when nothing changed
2236     my $changes;
2237     foreach my $opt (keys %{$conf->{pending}}) {
2238         if (defined($conf->{$opt}) && ($conf->{pending}->{$opt} eq  $conf->{$opt})) {
2239             $changes = 1;
2240             delete $conf->{pending}->{$opt};
2241         }
2242     }
2243
2244     my $current_delete_hash = split_flagged_list($conf->{pending}->{delete});
2245     my $pending_delete_hash = {};
2246     while (my ($opt, $force) = each %$current_delete_hash) {
2247         if (defined($conf->{$opt})) {
2248             $pending_delete_hash->{$opt} = $force;
2249         } else {
2250             $changes = 1;
2251         }
2252     }
2253
2254     if (%$pending_delete_hash) {
2255         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
2256     } else {
2257         delete $conf->{pending}->{delete};
2258     }
2259
2260     return $changes;
2261 }
2262
2263 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str]
2264 my $smbios1_fmt = {
2265     uuid => {
2266         type => 'string',
2267         pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
2268         format_description => 'UUID',
2269         description => "Set SMBIOS1 UUID.",
2270         optional => 1,
2271     },
2272     version => {
2273         type => 'string',
2274         pattern => '\S+',
2275         format_description => 'string',
2276         description => "Set SMBIOS1 version.",
2277         optional => 1,
2278     },
2279     serial => {
2280         type => 'string',
2281         pattern => '\S+',
2282         format_description => 'string',
2283         description => "Set SMBIOS1 serial number.",
2284         optional => 1,
2285     },
2286     manufacturer => {
2287         type => 'string',
2288         pattern => '\S+',
2289         format_description => 'string',
2290         description => "Set SMBIOS1 manufacturer.",
2291         optional => 1,
2292     },
2293     product => {
2294         type => 'string',
2295         pattern => '\S+',
2296         format_description => 'string',
2297         description => "Set SMBIOS1 product ID.",
2298         optional => 1,
2299     },
2300     sku => {
2301         type => 'string',
2302         pattern => '\S+',
2303         format_description => 'string',
2304         description => "Set SMBIOS1 SKU string.",
2305         optional => 1,
2306     },
2307     family => {
2308         type => 'string',
2309         pattern => '\S+',
2310         format_description => 'string',
2311         description => "Set SMBIOS1 family string.",
2312         optional => 1,
2313     },
2314 };
2315
2316 sub parse_smbios1 {
2317     my ($data) = @_;
2318
2319     my $res = eval { PVE::JSONSchema::parse_property_string($smbios1_fmt, $data) };
2320     warn $@ if $@;
2321     return $res;
2322 }
2323
2324 sub print_smbios1 {
2325     my ($smbios1) = @_;
2326     return PVE::JSONSchema::print_property_string($smbios1, $smbios1_fmt);
2327 }
2328
2329 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_fmt);
2330
2331 PVE::JSONSchema::register_format('pve-qm-bootdisk', \&verify_bootdisk);
2332 sub verify_bootdisk {
2333     my ($value, $noerr) = @_;
2334
2335     return $value if is_valid_drivename($value);
2336
2337     return undef if $noerr;
2338
2339     die "invalid boot disk '$value'\n";
2340 }
2341
2342 sub parse_watchdog {
2343     my ($value) = @_;
2344
2345     return undef if !$value;
2346
2347     my $res = eval { PVE::JSONSchema::parse_property_string($watchdog_fmt, $value) };
2348     warn $@ if $@;
2349     return $res;
2350 }
2351
2352 sub parse_guest_agent {
2353     my ($value) = @_;
2354
2355     return {} if !defined($value->{agent});
2356
2357     my $res = eval { PVE::JSONSchema::parse_property_string($agent_fmt, $value->{agent}) };
2358     warn $@ if $@;
2359
2360     # if the agent is disabled ignore the other potentially set properties
2361     return {} if !$res->{enabled};
2362     return $res;
2363 }
2364
2365 sub parse_vga {
2366     my ($value) = @_;
2367
2368     return {} if !$value;
2369     my $res = eval { PVE::JSONSchema::parse_property_string($vga_fmt, $value) };
2370     warn $@ if $@;
2371     return $res;
2372 }
2373
2374 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
2375 sub verify_usb_device {
2376     my ($value, $noerr) = @_;
2377
2378     return $value if parse_usb_device($value);
2379
2380     return undef if $noerr;
2381
2382     die "unable to parse usb device\n";
2383 }
2384
2385 # add JSON properties for create and set function
2386 sub json_config_properties {
2387     my $prop = shift;
2388
2389     foreach my $opt (keys %$confdesc) {
2390         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate' || $opt eq 'runningmachine';
2391         $prop->{$opt} = $confdesc->{$opt};
2392     }
2393
2394     return $prop;
2395 }
2396
2397 # return copy of $confdesc_cloudinit to generate documentation
2398 sub cloudinit_config_properties {
2399
2400     return dclone($confdesc_cloudinit);
2401 }
2402
2403 sub check_type {
2404     my ($key, $value) = @_;
2405
2406     die "unknown setting '$key'\n" if !$confdesc->{$key};
2407
2408     my $type = $confdesc->{$key}->{type};
2409
2410     if (!defined($value)) {
2411         die "got undefined value\n";
2412     }
2413
2414     if ($value =~ m/[\n\r]/) {
2415         die "property contains a line feed\n";
2416     }
2417
2418     if ($type eq 'boolean') {
2419         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
2420         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
2421         die "type check ('boolean') failed - got '$value'\n";
2422     } elsif ($type eq 'integer') {
2423         return int($1) if $value =~ m/^(\d+)$/;
2424         die "type check ('integer') failed - got '$value'\n";
2425     } elsif ($type eq 'number') {
2426         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
2427         die "type check ('number') failed - got '$value'\n";
2428     } elsif ($type eq 'string') {
2429         if (my $fmt = $confdesc->{$key}->{format}) {
2430             PVE::JSONSchema::check_format($fmt, $value);
2431             return $value;
2432         }
2433         $value =~ s/^\"(.*)\"$/$1/;
2434         return $value;
2435     } else {
2436         die "internal error"
2437     }
2438 }
2439
2440 sub check_iommu_support{
2441     #fixme : need to check IOMMU support
2442     #http://www.linux-kvm.org/page/How_to_assign_devices_with_VT-d_in_KVM
2443
2444     my $iommu=1;
2445     return $iommu;
2446
2447 }
2448
2449 sub touch_config {
2450     my ($vmid) = @_;
2451
2452     my $conf = PVE::QemuConfig->config_file($vmid);
2453     utime undef, undef, $conf;
2454 }
2455
2456 sub destroy_vm {
2457     my ($storecfg, $vmid, $keep_empty_config, $skiplock) = @_;
2458
2459     my $conffile = PVE::QemuConfig->config_file($vmid);
2460
2461     my $conf = PVE::QemuConfig->load_config($vmid);
2462
2463     PVE::QemuConfig->check_lock($conf) if !$skiplock;
2464
2465     if ($conf->{template}) {
2466         # check if any base image is still used by a linked clone
2467         foreach_drive($conf, sub {
2468                 my ($ds, $drive) = @_;
2469
2470                 return if drive_is_cdrom($drive);
2471
2472                 my $volid = $drive->{file};
2473
2474                 return if !$volid || $volid =~ m|^/|;
2475
2476                 die "base volume '$volid' is still in use by linked cloned\n"
2477                     if PVE::Storage::volume_is_base_and_used($storecfg, $volid);
2478
2479         });
2480     }
2481
2482     # only remove disks owned by this VM
2483     foreach_drive($conf, sub {
2484         my ($ds, $drive) = @_;
2485
2486         return if drive_is_cdrom($drive, 1);
2487
2488         my $volid = $drive->{file};
2489
2490         return if !$volid || $volid =~ m|^/|;
2491
2492         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
2493         return if !$path || !$owner || ($owner != $vmid);
2494
2495         eval {
2496             PVE::Storage::vdisk_free($storecfg, $volid);
2497         };
2498         warn "Could not remove disk '$volid', check manually: $@" if $@;
2499
2500     });
2501
2502     if ($keep_empty_config) {
2503         PVE::Tools::file_set_contents($conffile, "memory: 128\n");
2504     } else {
2505         unlink $conffile;
2506     }
2507
2508     # also remove unused disk
2509     eval {
2510         my $dl = PVE::Storage::vdisk_list($storecfg, undef, $vmid);
2511
2512         eval {
2513             PVE::Storage::foreach_volid($dl, sub {
2514                 my ($volid, $sid, $volname, $d) = @_;
2515                 PVE::Storage::vdisk_free($storecfg, $volid);
2516             });
2517         };
2518         warn $@ if $@;
2519
2520     };
2521     warn $@ if $@;
2522 }
2523
2524 sub parse_vm_config {
2525     my ($filename, $raw) = @_;
2526
2527     return undef if !defined($raw);
2528
2529     my $res = {
2530         digest => Digest::SHA::sha1_hex($raw),
2531         snapshots => {},
2532         pending => {},
2533     };
2534
2535     $filename =~ m|/qemu-server/(\d+)\.conf$|
2536         || die "got strange filename '$filename'";
2537
2538     my $vmid = $1;
2539
2540     my $conf = $res;
2541     my $descr;
2542     my $section = '';
2543
2544     my @lines = split(/\n/, $raw);
2545     foreach my $line (@lines) {
2546         next if $line =~ m/^\s*$/;
2547
2548         if ($line =~ m/^\[PENDING\]\s*$/i) {
2549             $section = 'pending';
2550             if (defined($descr)) {
2551                 $descr =~ s/\s+$//;
2552                 $conf->{description} = $descr;
2553             }
2554             $descr = undef;
2555             $conf = $res->{$section} = {};
2556             next;
2557
2558         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2559             $section = $1;
2560             if (defined($descr)) {
2561                 $descr =~ s/\s+$//;
2562                 $conf->{description} = $descr;
2563             }
2564             $descr = undef;
2565             $conf = $res->{snapshots}->{$section} = {};
2566             next;
2567         }
2568
2569         if ($line =~ m/^\#(.*)\s*$/) {
2570             $descr = '' if !defined($descr);
2571             $descr .= PVE::Tools::decode_text($1) . "\n";
2572             next;
2573         }
2574
2575         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2576             $descr = '' if !defined($descr);
2577             $descr .= PVE::Tools::decode_text($2);
2578         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2579             $conf->{snapstate} = $1;
2580         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2581             my $key = $1;
2582             my $value = $2;
2583             $conf->{$key} = $value;
2584         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2585             my $value = $1;
2586             if ($section eq 'pending') {
2587                 $conf->{delete} = $value; # we parse this later
2588             } else {
2589                 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
2590             }
2591         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(.+?)\s*$/) {
2592             my $key = $1;
2593             my $value = $2;
2594             eval { $value = check_type($key, $value); };
2595             if ($@) {
2596                 warn "vm $vmid - unable to parse value of '$key' - $@";
2597             } else {
2598                 $key = 'ide2' if $key eq 'cdrom';
2599                 my $fmt = $confdesc->{$key}->{format};
2600                 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2601                     my $v = parse_drive($key, $value);
2602                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2603                         $v->{file} = $volid;
2604                         $value = print_drive($vmid, $v);
2605                     } else {
2606                         warn "vm $vmid - unable to parse value of '$key'\n";
2607                         next;
2608                     }
2609                 }
2610
2611                 $conf->{$key} = $value;
2612             }
2613         }
2614     }
2615
2616     if (defined($descr)) {
2617         $descr =~ s/\s+$//;
2618         $conf->{description} = $descr;
2619     }
2620     delete $res->{snapstate}; # just to be sure
2621
2622     return $res;
2623 }
2624
2625 sub write_vm_config {
2626     my ($filename, $conf) = @_;
2627
2628     delete $conf->{snapstate}; # just to be sure
2629
2630     if ($conf->{cdrom}) {
2631         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2632         $conf->{ide2} = $conf->{cdrom};
2633         delete $conf->{cdrom};
2634     }
2635
2636     # we do not use 'smp' any longer
2637     if ($conf->{sockets}) {
2638         delete $conf->{smp};
2639     } elsif ($conf->{smp}) {
2640         $conf->{sockets} = $conf->{smp};
2641         delete $conf->{cores};
2642         delete $conf->{smp};
2643     }
2644
2645     my $used_volids = {};
2646
2647     my $cleanup_config = sub {
2648         my ($cref, $pending, $snapname) = @_;
2649
2650         foreach my $key (keys %$cref) {
2651             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2652                 $key eq 'snapstate' || $key eq 'pending';
2653             my $value = $cref->{$key};
2654             if ($key eq 'delete') {
2655                 die "propertry 'delete' is only allowed in [PENDING]\n"
2656                     if !$pending;
2657                 # fixme: check syntax?
2658                 next;
2659             }
2660             eval { $value = check_type($key, $value); };
2661             die "unable to parse value of '$key' - $@" if $@;
2662
2663             $cref->{$key} = $value;
2664
2665             if (!$snapname && is_valid_drivename($key)) {
2666                 my $drive = parse_drive($key, $value);
2667                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2668             }
2669         }
2670     };
2671
2672     &$cleanup_config($conf);
2673
2674     &$cleanup_config($conf->{pending}, 1);
2675
2676     foreach my $snapname (keys %{$conf->{snapshots}}) {
2677         die "internal error" if $snapname eq 'pending';
2678         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2679     }
2680
2681     # remove 'unusedX' settings if we re-add a volume
2682     foreach my $key (keys %$conf) {
2683         my $value = $conf->{$key};
2684         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2685             delete $conf->{$key};
2686         }
2687     }
2688
2689     my $generate_raw_config = sub {
2690         my ($conf, $pending) = @_;
2691
2692         my $raw = '';
2693
2694         # add description as comment to top of file
2695         if (defined(my $descr = $conf->{description})) {
2696             if ($descr) {
2697                 foreach my $cl (split(/\n/, $descr)) {
2698                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2699                 }
2700             } else {
2701                 $raw .= "#\n" if $pending;
2702             }
2703         }
2704
2705         foreach my $key (sort keys %$conf) {
2706             next if $key eq 'digest' || $key eq 'description' || $key eq 'pending' || $key eq 'snapshots';
2707             $raw .= "$key: $conf->{$key}\n";
2708         }
2709         return $raw;
2710     };
2711
2712     my $raw = &$generate_raw_config($conf);
2713
2714     if (scalar(keys %{$conf->{pending}})){
2715         $raw .= "\n[PENDING]\n";
2716         $raw .= &$generate_raw_config($conf->{pending}, 1);
2717     }
2718
2719     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2720         $raw .= "\n[$snapname]\n";
2721         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2722     }
2723
2724     return $raw;
2725 }
2726
2727 sub load_defaults {
2728
2729     my $res = {};
2730
2731     # we use static defaults from our JSON schema configuration
2732     foreach my $key (keys %$confdesc) {
2733         if (defined(my $default = $confdesc->{$key}->{default})) {
2734             $res->{$key} = $default;
2735         }
2736     }
2737
2738     return $res;
2739 }
2740
2741 sub config_list {
2742     my $vmlist = PVE::Cluster::get_vmlist();
2743     my $res = {};
2744     return $res if !$vmlist || !$vmlist->{ids};
2745     my $ids = $vmlist->{ids};
2746
2747     foreach my $vmid (keys %$ids) {
2748         my $d = $ids->{$vmid};
2749         next if !$d->{node} || $d->{node} ne $nodename;
2750         next if !$d->{type} || $d->{type} ne 'qemu';
2751         $res->{$vmid}->{exists} = 1;
2752     }
2753     return $res;
2754 }
2755
2756 # test if VM uses local resources (to prevent migration)
2757 sub check_local_resources {
2758     my ($conf, $noerr) = @_;
2759
2760     my $loc_res = 0;
2761
2762     $loc_res = 1 if $conf->{hostusb}; # old syntax
2763     $loc_res = 1 if $conf->{hostpci}; # old syntax
2764
2765     foreach my $k (keys %$conf) {
2766         next if $k =~ m/^usb/ && ($conf->{$k} eq 'spice');
2767         # sockets are safe: they will recreated be on the target side post-migrate
2768         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2769         $loc_res = 1 if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2770     }
2771
2772     die "VM uses local resources\n" if $loc_res && !$noerr;
2773
2774     return $loc_res;
2775 }
2776
2777 # check if used storages are available on all nodes (use by migrate)
2778 sub check_storage_availability {
2779     my ($storecfg, $conf, $node) = @_;
2780
2781     foreach_drive($conf, sub {
2782         my ($ds, $drive) = @_;
2783
2784         my $volid = $drive->{file};
2785         return if !$volid;
2786
2787         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2788         return if !$sid;
2789
2790         # check if storage is available on both nodes
2791         my $scfg = PVE::Storage::storage_check_node($storecfg, $sid);
2792         PVE::Storage::storage_check_node($storecfg, $sid, $node);
2793    });
2794 }
2795
2796 # list nodes where all VM images are available (used by has_feature API)
2797 sub shared_nodes {
2798     my ($conf, $storecfg) = @_;
2799
2800     my $nodelist = PVE::Cluster::get_nodelist();
2801     my $nodehash = { map { $_ => 1 } @$nodelist };
2802     my $nodename = PVE::INotify::nodename();
2803
2804     foreach_drive($conf, sub {
2805         my ($ds, $drive) = @_;
2806
2807         my $volid = $drive->{file};
2808         return if !$volid;
2809
2810         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2811         if ($storeid) {
2812             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2813             if ($scfg->{disable}) {
2814                 $nodehash = {};
2815             } elsif (my $avail = $scfg->{nodes}) {
2816                 foreach my $node (keys %$nodehash) {
2817                     delete $nodehash->{$node} if !$avail->{$node};
2818                 }
2819             } elsif (!$scfg->{shared}) {
2820                 foreach my $node (keys %$nodehash) {
2821                     delete $nodehash->{$node} if $node ne $nodename
2822                 }
2823             }
2824         }
2825     });
2826
2827     return $nodehash
2828 }
2829
2830 sub check_cmdline {
2831     my ($pidfile, $pid) = @_;
2832
2833     my $fh = IO::File->new("/proc/$pid/cmdline", "r");
2834     if (defined($fh)) {
2835         my $line = <$fh>;
2836         $fh->close;
2837         return undef if !$line;
2838         my @param = split(/\0/, $line);
2839
2840         my $cmd = $param[0];
2841         return if !$cmd || ($cmd !~ m|kvm$| && $cmd !~ m@(?:^|/)qemu-system-[^/]+$@);
2842
2843         for (my $i = 0; $i < scalar (@param); $i++) {
2844             my $p = $param[$i];
2845             next if !$p;
2846             if (($p eq '-pidfile') || ($p eq '--pidfile')) {
2847                 my $p = $param[$i+1];
2848                 return 1 if $p && ($p eq $pidfile);
2849                 return undef;
2850             }
2851         }
2852     }
2853     return undef;
2854 }
2855
2856 sub check_running {
2857     my ($vmid, $nocheck, $node) = @_;
2858
2859     my $filename = PVE::QemuConfig->config_file($vmid, $node);
2860
2861     die "unable to find configuration file for VM $vmid - no such machine\n"
2862         if !$nocheck && ! -f $filename;
2863
2864     my $pidfile = pidfile_name($vmid);
2865
2866     if (my $fd = IO::File->new("<$pidfile")) {
2867         my $st = stat($fd);
2868         my $line = <$fd>;
2869         close($fd);
2870
2871         my $mtime = $st->mtime;
2872         if ($mtime > time()) {
2873             warn "file '$filename' modified in future\n";
2874         }
2875
2876         if ($line =~ m/^(\d+)$/) {
2877             my $pid = $1;
2878             if (check_cmdline($pidfile, $pid)) {
2879                 if (my $pinfo = PVE::ProcFSTools::check_process_running($pid)) {
2880                     return $pid;
2881                 }
2882             }
2883         }
2884     }
2885
2886     return undef;
2887 }
2888
2889 sub vzlist {
2890
2891     my $vzlist = config_list();
2892
2893     my $fd = IO::Dir->new($var_run_tmpdir) || return $vzlist;
2894
2895     while (defined(my $de = $fd->read)) {
2896         next if $de !~ m/^(\d+)\.pid$/;
2897         my $vmid = $1;
2898         next if !defined($vzlist->{$vmid});
2899         if (my $pid = check_running($vmid)) {
2900             $vzlist->{$vmid}->{pid} = $pid;
2901         }
2902     }
2903
2904     return $vzlist;
2905 }
2906
2907 sub disksize {
2908     my ($storecfg, $conf) = @_;
2909
2910     my $bootdisk = $conf->{bootdisk};
2911     return undef if !$bootdisk;
2912     return undef if !is_valid_drivename($bootdisk);
2913
2914     return undef if !$conf->{$bootdisk};
2915
2916     my $drive = parse_drive($bootdisk, $conf->{$bootdisk});
2917     return undef if !defined($drive);
2918
2919     return undef if drive_is_cdrom($drive);
2920
2921     my $volid = $drive->{file};
2922     return undef if !$volid;
2923
2924     return $drive->{size};
2925 }
2926
2927 our $vmstatus_return_properties = {
2928     vmid => get_standard_option('pve-vmid'),
2929     status => {
2930         description => "Qemu process status.",
2931         type => 'string',
2932         enum => ['stopped', 'running'],
2933     },
2934     maxmem => {
2935         description => "Maximum memory in bytes.",
2936         type => 'integer',
2937         optional => 1,
2938         renderer => 'bytes',
2939     },
2940     maxdisk => {
2941         description => "Root disk size in bytes.",
2942         type => 'integer',
2943         optional => 1,
2944         renderer => 'bytes',
2945     },
2946     name => {
2947         description => "VM name.",
2948         type => 'string',
2949         optional => 1,
2950     },
2951     qmpstatus => {
2952         description => "Qemu QMP agent status.",
2953         type => 'string',
2954         optional => 1,
2955     },
2956     pid => {
2957         description => "PID of running qemu process.",
2958         type => 'integer',
2959         optional => 1,
2960     },
2961     uptime => {
2962         description => "Uptime.",
2963         type => 'integer',
2964         optional => 1,
2965         renderer => 'duration',
2966     },
2967     cpus => {
2968         description => "Maximum usable CPUs.",
2969         type => 'number',
2970         optional => 1,
2971     },
2972 };
2973
2974 my $last_proc_pid_stat;
2975
2976 # get VM status information
2977 # This must be fast and should not block ($full == false)
2978 # We only query KVM using QMP if $full == true (this can be slow)
2979 sub vmstatus {
2980     my ($opt_vmid, $full) = @_;
2981
2982     my $res = {};
2983
2984     my $storecfg = PVE::Storage::config();
2985
2986     my $list = vzlist();
2987     my $defaults = load_defaults();
2988
2989     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
2990
2991     my $cpucount = $cpuinfo->{cpus} || 1;
2992
2993     foreach my $vmid (keys %$list) {
2994         next if $opt_vmid && ($vmid ne $opt_vmid);
2995
2996         my $cfspath = PVE::QemuConfig->cfs_config_path($vmid);
2997         my $conf = PVE::Cluster::cfs_read_file($cfspath) || {};
2998
2999         my $d = { vmid => $vmid };
3000         $d->{pid} = $list->{$vmid}->{pid};
3001
3002         # fixme: better status?
3003         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
3004
3005         my $size = disksize($storecfg, $conf);
3006         if (defined($size)) {
3007             $d->{disk} = 0; # no info available
3008             $d->{maxdisk} = $size;
3009         } else {
3010             $d->{disk} = 0;
3011             $d->{maxdisk} = 0;
3012         }
3013
3014         $d->{cpus} = ($conf->{sockets} || $defaults->{sockets})
3015             * ($conf->{cores} || $defaults->{cores});
3016         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
3017         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
3018
3019         $d->{name} = $conf->{name} || "VM $vmid";
3020         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024)
3021             : $defaults->{memory}*(1024*1024);
3022
3023         if ($conf->{balloon}) {
3024             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
3025             $d->{shares} = defined($conf->{shares}) ? $conf->{shares}
3026                 : $defaults->{shares};
3027         }
3028
3029         $d->{uptime} = 0;
3030         $d->{cpu} = 0;
3031         $d->{mem} = 0;
3032
3033         $d->{netout} = 0;
3034         $d->{netin} = 0;
3035
3036         $d->{diskread} = 0;
3037         $d->{diskwrite} = 0;
3038
3039         $d->{template} = PVE::QemuConfig->is_template($conf);
3040
3041         $d->{serial} = 1 if conf_has_serial($conf);
3042
3043         $res->{$vmid} = $d;
3044     }
3045
3046     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
3047     foreach my $dev (keys %$netdev) {
3048         next if $dev !~ m/^tap([1-9]\d*)i/;
3049         my $vmid = $1;
3050         my $d = $res->{$vmid};
3051         next if !$d;
3052
3053         $d->{netout} += $netdev->{$dev}->{receive};
3054         $d->{netin} += $netdev->{$dev}->{transmit};
3055
3056         if ($full) {
3057             $d->{nics}->{$dev}->{netout} = $netdev->{$dev}->{receive};
3058             $d->{nics}->{$dev}->{netin} = $netdev->{$dev}->{transmit};
3059         }
3060
3061     }
3062
3063     my $ctime = gettimeofday;
3064
3065     foreach my $vmid (keys %$list) {
3066
3067         my $d = $res->{$vmid};
3068         my $pid = $d->{pid};
3069         next if !$pid;
3070
3071         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
3072         next if !$pstat; # not running
3073
3074         my $used = $pstat->{utime} + $pstat->{stime};
3075
3076         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
3077
3078         if ($pstat->{vsize}) {
3079             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
3080         }
3081
3082         my $old = $last_proc_pid_stat->{$pid};
3083         if (!$old) {
3084             $last_proc_pid_stat->{$pid} = {
3085                 time => $ctime,
3086                 used => $used,
3087                 cpu => 0,
3088             };
3089             next;
3090         }
3091
3092         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
3093
3094         if ($dtime > 1000) {
3095             my $dutime = $used -  $old->{used};
3096
3097             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
3098             $last_proc_pid_stat->{$pid} = {
3099                 time => $ctime,
3100                 used => $used,
3101                 cpu => $d->{cpu},
3102             };
3103         } else {
3104             $d->{cpu} = $old->{cpu};
3105         }
3106     }
3107
3108     return $res if !$full;
3109
3110     my $qmpclient = PVE::QMPClient->new();
3111
3112     my $ballooncb = sub {
3113         my ($vmid, $resp) = @_;
3114
3115         my $info = $resp->{'return'};
3116         return if !$info->{max_mem};
3117
3118         my $d = $res->{$vmid};
3119
3120         # use memory assigned to VM
3121         $d->{maxmem} = $info->{max_mem};
3122         $d->{balloon} = $info->{actual};
3123
3124         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
3125             $d->{mem} = $info->{total_mem} - $info->{free_mem};
3126             $d->{freemem} = $info->{free_mem};
3127         }
3128
3129         $d->{ballooninfo} = $info;
3130     };
3131
3132     my $blockstatscb = sub {
3133         my ($vmid, $resp) = @_;
3134         my $data = $resp->{'return'} || [];
3135         my $totalrdbytes = 0;
3136         my $totalwrbytes = 0;
3137
3138         for my $blockstat (@$data) {
3139             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
3140             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
3141
3142             $blockstat->{device} =~ s/drive-//;
3143             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
3144         }
3145         $res->{$vmid}->{diskread} = $totalrdbytes;
3146         $res->{$vmid}->{diskwrite} = $totalwrbytes;
3147     };
3148
3149     my $statuscb = sub {
3150         my ($vmid, $resp) = @_;
3151
3152         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
3153         # this fails if ballon driver is not loaded, so this must be
3154         # the last commnand (following command are aborted if this fails).
3155         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
3156
3157         my $status = 'unknown';
3158         if (!defined($status = $resp->{'return'}->{status})) {
3159             warn "unable to get VM status\n";
3160             return;
3161         }
3162
3163         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
3164     };
3165
3166     foreach my $vmid (keys %$list) {
3167         next if $opt_vmid && ($vmid ne $opt_vmid);
3168         next if !$res->{$vmid}->{pid}; # not running
3169         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
3170     }
3171
3172     $qmpclient->queue_execute(undef, 2);
3173
3174     foreach my $vmid (keys %$list) {
3175         next if $opt_vmid && ($vmid ne $opt_vmid);
3176         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
3177     }
3178
3179     return $res;
3180 }
3181
3182 sub foreach_drive {
3183     my ($conf, $func, @param) = @_;
3184
3185     foreach my $ds (valid_drive_names()) {
3186         next if !defined($conf->{$ds});
3187
3188         my $drive = parse_drive($ds, $conf->{$ds});
3189         next if !$drive;
3190
3191         &$func($ds, $drive, @param);
3192     }
3193 }
3194
3195 sub foreach_volid {
3196     my ($conf, $func, @param) = @_;
3197
3198     my $volhash = {};
3199
3200     my $test_volid = sub {
3201         my ($volid, $is_cdrom, $replicate, $shared, $snapname) = @_;
3202
3203         return if !$volid;
3204
3205         $volhash->{$volid}->{cdrom} //= 1;
3206         $volhash->{$volid}->{cdrom} = 0 if !$is_cdrom;
3207
3208         $volhash->{$volid}->{replicate} //= 0;
3209         $volhash->{$volid}->{replicate} = 1 if $replicate;
3210
3211         $volhash->{$volid}->{shared} //= 0;
3212         $volhash->{$volid}->{shared} = 1 if $shared;
3213
3214         $volhash->{$volid}->{referenced_in_config} //= 0;
3215         $volhash->{$volid}->{referenced_in_config} = 1 if !defined($snapname);
3216
3217         $volhash->{$volid}->{referenced_in_snapshot}->{$snapname} = 1
3218             if defined($snapname);
3219     };
3220
3221     foreach_drive($conf, sub {
3222         my ($ds, $drive) = @_;
3223         $test_volid->($drive->{file}, drive_is_cdrom($drive), $drive->{replicate} // 1, $drive->{shared}, undef);
3224     });
3225
3226     foreach my $snapname (keys %{$conf->{snapshots}}) {
3227         my $snap = $conf->{snapshots}->{$snapname};
3228         $test_volid->($snap->{vmstate}, 0, 1, $snapname);
3229         foreach_drive($snap, sub {
3230             my ($ds, $drive) = @_;
3231             $test_volid->($drive->{file}, drive_is_cdrom($drive), $drive->{replicate} // 1, $drive->{shared}, $snapname);
3232         });
3233     }
3234
3235     foreach my $volid (keys %$volhash) {
3236         &$func($volid, $volhash->{$volid}, @param);
3237     }
3238 }
3239
3240 sub conf_has_serial {
3241     my ($conf) = @_;
3242
3243     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3244         if ($conf->{"serial$i"}) {
3245             return 1;
3246         }
3247     }
3248
3249     return 0;
3250 }
3251
3252 sub vga_conf_has_spice {
3253     my ($vga) = @_;
3254
3255     my $vgaconf = parse_vga($vga);
3256     my $vgatype = $vgaconf->{type};
3257     return 0 if !$vgatype || $vgatype !~ m/^qxl([234])?$/;
3258
3259     return $1 || 1;
3260 }
3261
3262 my $host_arch; # FIXME: fix PVE::Tools::get_host_arch
3263 sub get_host_arch() {
3264     $host_arch = (POSIX::uname())[4] if !$host_arch;
3265     return $host_arch;
3266 }
3267
3268 sub is_native($) {
3269     my ($arch) = @_;
3270     return get_host_arch() eq $arch;
3271 }
3272
3273 my $default_machines = {
3274     x86_64 => 'pc',
3275     aarch64 => 'virt',
3276 };
3277
3278 sub get_basic_machine_info {
3279     my ($conf, $forcemachine) = @_;
3280
3281     my $arch = $conf->{arch} // get_host_arch();
3282     my $machine = $forcemachine || $conf->{machine} || $default_machines->{$arch};
3283     return ($arch, $machine);
3284 }
3285
3286 sub get_ovmf_files($) {
3287     my ($arch) = @_;
3288
3289     my $ovmf = $OVMF->{$arch}
3290         or die "no OVMF images known for architecture '$arch'\n";
3291
3292     return @$ovmf;
3293 }
3294
3295 my $Arch2Qemu = {
3296     aarch64 => '/usr/bin/qemu-system-aarch64',
3297     x86_64 => '/usr/bin/qemu-system-x86_64',
3298 };
3299 sub get_command_for_arch($) {
3300     my ($arch) = @_;
3301     return '/usr/bin/kvm' if is_native($arch);
3302
3303     my $cmd = $Arch2Qemu->{$arch}
3304         or die "don't know how to emulate architecture '$arch'\n";
3305     return $cmd;
3306 }
3307
3308 sub config_to_command {
3309     my ($storecfg, $vmid, $conf, $defaults, $forcemachine) = @_;
3310
3311     my $cmd = [];
3312     my $globalFlags = [];
3313     my $machineFlags = [];
3314     my $rtcFlags = [];
3315     my $cpuFlags = [];
3316     my $devices = [];
3317     my $pciaddr = '';
3318     my $bridges = {};
3319     my $kvmver = kvm_user_version();
3320     my $vernum = 0; # unknown
3321     my $ostype = $conf->{ostype};
3322     my $winversion = windows_version($ostype);
3323     my $kvm = $conf->{kvm};
3324
3325     my ($arch, $machine_type) = get_basic_machine_info($conf, $forcemachine);
3326     $kvm //= 1 if is_native($arch);
3327
3328     if ($kvm) {
3329         die "KVM virtualisation configured, but not available. Either disable in VM configuration or enable in BIOS.\n"
3330             if !defined kvm_version();
3331     }
3332
3333     if ($kvmver =~ m/^(\d+)\.(\d+)$/) {
3334         $vernum = $1*1000000+$2*1000;
3335     } elsif ($kvmver =~ m/^(\d+)\.(\d+)\.(\d+)$/) {
3336         $vernum = $1*1000000+$2*1000+$3;
3337     }
3338
3339     die "detected old qemu-kvm binary ($kvmver)\n" if $vernum < 15000;
3340
3341     my $have_ovz = -f '/proc/vz/vestat';
3342
3343     my $q35 = machine_type_is_q35($conf);
3344     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
3345     my $use_old_bios_files = undef;
3346     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3347
3348     my $cpuunits = defined($conf->{cpuunits}) ?
3349             $conf->{cpuunits} : $defaults->{cpuunits};
3350
3351     push @$cmd, get_command_for_arch($arch);
3352
3353     push @$cmd, '-id', $vmid;
3354
3355     my $vmname = $conf->{name} || "vm$vmid";
3356
3357     push @$cmd, '-name', $vmname;
3358
3359     my $use_virtio = 0;
3360
3361     my $qmpsocket = qmp_socket($vmid);
3362     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server,nowait";
3363     push @$cmd, '-mon', "chardev=qmp,mode=control";
3364
3365     if (qemu_machine_feature_enabled($machine_type, $kvmver, 2, 12)) {
3366         my $eventsocket = qmp_socket($vmid, 0, 'event');
3367         push @$cmd, '-chardev', "socket,id=qmp-event,path=$eventsocket,server,nowait";
3368         push @$cmd, '-mon', "chardev=qmp-event,mode=control";
3369     }
3370
3371     push @$cmd, '-pidfile' , pidfile_name($vmid);
3372
3373     push @$cmd, '-daemonize';
3374
3375     if ($conf->{smbios1}) {
3376         push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
3377     }
3378
3379     if ($conf->{vmgenid}) {
3380         push @$devices, '-device', 'vmgenid,guid='.$conf->{vmgenid};
3381     }
3382
3383     my ($ovmf_code, $ovmf_vars) = get_ovmf_files($arch);
3384     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3385         die "uefi base image not found\n" if ! -f $ovmf_code;
3386
3387         my $path;
3388         my $format;
3389         if (my $efidisk = $conf->{efidisk0}) {
3390             my $d = PVE::JSONSchema::parse_property_string($efidisk_fmt, $efidisk);
3391             my ($storeid, $volname) = PVE::Storage::parse_volume_id($d->{file}, 1);
3392             $format = $d->{format};
3393             if ($storeid) {
3394                 $path = PVE::Storage::path($storecfg, $d->{file});
3395                 if (!defined($format)) {
3396                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
3397                     $format = qemu_img_format($scfg, $volname);
3398                 }
3399             } else {
3400                 $path = $d->{file};
3401                 die "efidisk format must be specified\n"
3402                     if !defined($format);
3403             }
3404         } else {
3405             warn "no efidisk configured! Using temporary efivars disk.\n";
3406             $path = "/tmp/$vmid-ovmf.fd";
3407             PVE::Tools::file_copy($ovmf_vars, $path, -s $ovmf_vars);
3408             $format = 'raw';
3409         }
3410
3411         push @$cmd, '-drive', "if=pflash,unit=0,format=raw,readonly,file=$ovmf_code";
3412         push @$cmd, '-drive', "if=pflash,unit=1,format=$format,id=drive-efidisk0,file=$path";
3413     }
3414
3415
3416     # add usb controllers
3417     my @usbcontrollers = PVE::QemuServer::USB::get_usb_controllers($conf, $bridges, $q35, $usbdesc->{format}, $MAX_USB_DEVICES);
3418     push @$devices, @usbcontrollers if @usbcontrollers;
3419     my $vga = parse_vga($conf->{vga});
3420
3421     my $qxlnum = vga_conf_has_spice($conf->{vga});
3422     $vga->{type} = 'qxl' if $qxlnum;
3423
3424     if (!$vga->{type}) {
3425         if (qemu_machine_feature_enabled($machine_type, $kvmver, 2, 9)) {
3426             $vga->{type} = (!$winversion || $winversion >= 6) ? 'std' : 'cirrus';
3427         } else {
3428             $vga->{type} = ($winversion >= 6) ? 'std' : 'cirrus';
3429         }
3430     }
3431
3432     # enable absolute mouse coordinates (needed by vnc)
3433     my $tablet;
3434     if (defined($conf->{tablet})) {
3435         $tablet = $conf->{tablet};
3436     } else {
3437         $tablet = $defaults->{tablet};
3438         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
3439         $tablet = 0 if $vga->{type} =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
3440     }
3441
3442     push @$devices, '-device', print_tabletdevice_full($conf) if $tablet;
3443
3444     my $kvm_off = 0;
3445     my $gpu_passthrough;
3446
3447     # host pci devices
3448     for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
3449         my $d = parse_hostpci($conf->{"hostpci$i"});
3450         next if !$d;
3451
3452         my $pcie = $d->{pcie};
3453         if($pcie){
3454             die "q35 machine model is not enabled" if !$q35;
3455             $pciaddr = print_pcie_addr("hostpci$i");
3456         }else{
3457             $pciaddr = print_pci_addr("hostpci$i", $bridges);
3458         }
3459
3460         my $rombar = defined($d->{rombar}) && !$d->{rombar} ? ',rombar=0' : '';
3461         my $romfile = $d->{romfile};
3462
3463         my $xvga = '';
3464         if ($d->{'x-vga'}) {
3465             $xvga = ',x-vga=on';
3466             $kvm_off = 1;
3467             $vga->{type} = 'none';
3468             $gpu_passthrough = 1;
3469
3470             if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3471                 $xvga = "";
3472             }
3473         }
3474         my $pcidevices = $d->{pciid};
3475         my $multifunction = 1 if @$pcidevices > 1;
3476
3477         my $j=0;
3478         foreach my $pcidevice (@$pcidevices) {
3479
3480             my $id = "hostpci$i";
3481             $id .= ".$j" if $multifunction;
3482             my $addr = $pciaddr;
3483             $addr .= ".$j" if $multifunction;
3484             my $devicestr = "vfio-pci,host=$pcidevice->{id}.$pcidevice->{function},id=$id$addr";
3485
3486             if($j == 0){
3487                 $devicestr .= "$rombar$xvga";
3488                 $devicestr .= ",multifunction=on" if $multifunction;
3489                 $devicestr .= ",romfile=/usr/share/kvm/$romfile" if $romfile;
3490             }
3491
3492             push @$devices, '-device', $devicestr;
3493             $j++;
3494         }
3495     }
3496
3497     # usb devices
3498     my @usbdevices = PVE::QemuServer::USB::get_usb_devices($conf, $usbdesc->{format}, $MAX_USB_DEVICES);
3499     push @$devices, @usbdevices if @usbdevices;
3500     # serial devices
3501     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3502         if (my $path = $conf->{"serial$i"}) {
3503             if ($path eq 'socket') {
3504                 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
3505                 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server,nowait";
3506                 push @$devices, '-device', "isa-serial,chardev=serial$i";
3507             } else {
3508                 die "no such serial device\n" if ! -c $path;
3509                 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
3510                 push @$devices, '-device', "isa-serial,chardev=serial$i";
3511             }
3512         }
3513     }
3514
3515     # parallel devices
3516     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
3517         if (my $path = $conf->{"parallel$i"}) {
3518             die "no such parallel device\n" if ! -c $path;
3519             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
3520             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
3521             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
3522         }
3523     }
3524
3525
3526     my $sockets = 1;
3527     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3528     $sockets = $conf->{sockets} if  $conf->{sockets};
3529
3530     my $cores = $conf->{cores} || 1;
3531
3532     my $maxcpus = $sockets * $cores;
3533
3534     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
3535
3536     my $allowed_vcpus = $cpuinfo->{cpus};
3537
3538     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n"
3539         if ($allowed_vcpus < $maxcpus);
3540
3541     if($hotplug_features->{cpu} && qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 7)) {
3542
3543         push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3544         for (my $i = 2; $i <= $vcpus; $i++)  {
3545             my $cpustr = print_cpu_device($conf,$i);
3546             push @$cmd, '-device', $cpustr;
3547         }
3548
3549     } else {
3550
3551         push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3552     }
3553     push @$cmd, '-nodefaults';
3554
3555     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
3556
3557     my $bootindex_hash = {};
3558     my $i = 1;
3559     foreach my $o (split(//, $bootorder)) {
3560         $bootindex_hash->{$o} = $i*100;
3561         $i++;
3562     }
3563
3564     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
3565
3566     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
3567
3568     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
3569
3570     if ($vga->{type} && $vga->{type} !~ m/^serial\d+$/ && $vga->{type} ne 'none'){
3571         push @$devices, '-device', print_vga_device($conf, $vga, undef, $qxlnum, $bridges);
3572         my $socket = vnc_socket($vmid);
3573         push @$cmd,  '-vnc', "unix:$socket,x509,password";
3574     } else {
3575         push @$cmd, '-vga', 'none' if $vga->{type} eq 'none';
3576         push @$cmd, '-nographic';
3577     }
3578
3579     # time drift fix
3580     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
3581
3582     my $useLocaltime = $conf->{localtime};
3583
3584     if ($winversion >= 5) { # windows
3585         $useLocaltime = 1 if !defined($conf->{localtime});
3586
3587         # use time drift fix when acpi is enabled
3588         if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
3589             $tdf = 1 if !defined($conf->{tdf});
3590         }
3591     }
3592
3593     if ($winversion >= 6) {
3594         push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3595         push @$cmd, '-no-hpet';
3596     }
3597
3598     push @$rtcFlags, 'driftfix=slew' if $tdf;
3599
3600     if (!$kvm) {
3601         push @$machineFlags, 'accel=tcg';
3602     }
3603
3604     if ($machine_type) {
3605         push @$machineFlags, "type=${machine_type}";
3606     }
3607
3608     if ($conf->{startdate}) {
3609         push @$rtcFlags, "base=$conf->{startdate}";
3610     } elsif ($useLocaltime) {
3611         push @$rtcFlags, 'base=localtime';
3612     }
3613
3614     my $cpu = $kvm ? "kvm64" : "qemu64";
3615     if (my $cputype = $conf->{cpu}) {
3616         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
3617             or die "Cannot parse cpu description: $cputype\n";
3618         $cpu = $cpuconf->{cputype};
3619         $kvm_off = 1 if $cpuconf->{hidden};
3620
3621         if (defined(my $flags = $cpuconf->{flags})) {
3622             push @$cpuFlags, split(";", $flags);
3623         }
3624     }
3625
3626     push @$cpuFlags , '+lahf_lm' if $cpu eq 'kvm64';
3627
3628     push @$cpuFlags , '-x2apic'
3629         if $conf->{ostype} && $conf->{ostype} eq 'solaris';
3630
3631     push @$cpuFlags, '+sep' if $cpu eq 'kvm64' || $cpu eq 'kvm32';
3632
3633     push @$cpuFlags, '-rdtscp' if $cpu =~ m/^Opteron/;
3634
3635     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3636
3637         push @$cpuFlags , '+kvm_pv_unhalt' if $kvm;
3638         push @$cpuFlags , '+kvm_pv_eoi' if $kvm;
3639     }
3640
3641     add_hyperv_enlightenments($cpuFlags, $winversion, $machine_type, $kvmver, $conf->{bios}, $gpu_passthrough) if $kvm;
3642
3643     push @$cpuFlags, 'enforce' if $cpu ne 'host' && $kvm;
3644
3645     push @$cpuFlags, 'kvm=off' if $kvm_off;
3646
3647     my $cpu_vendor = $cpu_vendor_list->{$cpu} ||
3648         die "internal error"; # should not happen
3649
3650     push @$cpuFlags, "vendor=${cpu_vendor}"
3651         if $cpu_vendor ne 'default';
3652
3653     $cpu .= "," . join(',', @$cpuFlags) if scalar(@$cpuFlags);
3654
3655     push @$cmd, '-cpu', $cpu;
3656
3657     PVE::QemuServer::Memory::config($conf, $vmid, $sockets, $cores, $defaults, $hotplug_features, $cmd);
3658
3659     push @$cmd, '-S' if $conf->{freeze};
3660
3661     push @$cmd, '-k', $conf->{keyboard} if defined($conf->{keyboard});
3662
3663     # enable sound
3664     #my $soundhw = $conf->{soundhw} || $defaults->{soundhw};
3665     #push @$cmd, '-soundhw', 'es1370';
3666     #push @$cmd, '-soundhw', $soundhw if $soundhw;
3667
3668     if (parse_guest_agent($conf)->{enabled}) {
3669         my $qgasocket = qmp_socket($vmid, 1);
3670         my $pciaddr = print_pci_addr("qga0", $bridges);
3671         push @$devices, '-chardev', "socket,path=$qgasocket,server,nowait,id=qga0";
3672         push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3673         push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3674     }
3675
3676     my $spice_port;
3677
3678     if ($qxlnum) {
3679         if ($qxlnum > 1) {
3680             if ($winversion){
3681                 for(my $i = 1; $i < $qxlnum; $i++){
3682                     push @$devices, '-device', print_vga_device($conf, $vga, $i, $qxlnum, $bridges);
3683                 }
3684             } else {
3685                 # assume other OS works like Linux
3686                 my ($ram, $vram) = ("134217728", "67108864");
3687                 if ($vga->{memory}) {
3688                     $ram = PVE::Tools::convert_size($qxlnum*4*$vga->{memory}, 'mb' => 'b');
3689                     $vram = PVE::Tools::convert_size($qxlnum*2*$vga->{memory}, 'mb' => 'b');
3690                 }
3691                 push @$cmd, '-global', "qxl-vga.ram_size=$ram";
3692                 push @$cmd, '-global', "qxl-vga.vram_size=$vram";
3693             }
3694         }
3695
3696         my $pciaddr = print_pci_addr("spice", $bridges);
3697
3698         my $nodename = PVE::INotify::nodename();
3699         my $pfamily = PVE::Tools::get_host_address_family($nodename);
3700         my @nodeaddrs = PVE::Tools::getaddrinfo_all('localhost', family => $pfamily);
3701         die "failed to get an ip address of type $pfamily for 'localhost'\n" if !@nodeaddrs;
3702         my $localhost = PVE::Network::addr_to_ip($nodeaddrs[0]->{addr});
3703         $spice_port = PVE::Tools::next_spice_port($pfamily, $localhost);
3704
3705         push @$devices, '-spice', "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";
3706
3707         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3708         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3709         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3710     }
3711
3712     # enable balloon by default, unless explicitly disabled
3713     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3714         $pciaddr = print_pci_addr("balloon0", $bridges);
3715         push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
3716     }
3717
3718     if ($conf->{watchdog}) {
3719         my $wdopts = parse_watchdog($conf->{watchdog});
3720         $pciaddr = print_pci_addr("watchdog", $bridges);
3721         my $watchdog = $wdopts->{model} || 'i6300esb';
3722         push @$devices, '-device', "$watchdog$pciaddr";
3723         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3724     }
3725
3726     my $vollist = [];
3727     my $scsicontroller = {};
3728     my $ahcicontroller = {};
3729     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3730
3731     # Add iscsi initiator name if available
3732     if (my $initiator = get_initiator_name()) {
3733         push @$devices, '-iscsi', "initiator-name=$initiator";
3734     }
3735
3736     foreach_drive($conf, sub {
3737         my ($ds, $drive) = @_;
3738
3739         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3740             push @$vollist, $drive->{file};
3741         }
3742
3743         # ignore efidisk here, already added in bios/fw handling code above
3744         return if $drive->{interface} eq 'efidisk';
3745
3746         $use_virtio = 1 if $ds =~ m/^virtio/;
3747
3748         if (drive_is_cdrom ($drive)) {
3749             if ($bootindex_hash->{d}) {
3750                 $drive->{bootindex} = $bootindex_hash->{d};
3751                 $bootindex_hash->{d} += 1;
3752             }
3753         } else {
3754             if ($bootindex_hash->{c}) {
3755                 $drive->{bootindex} = $bootindex_hash->{c} if $conf->{bootdisk} && ($conf->{bootdisk} eq $ds);
3756                 $bootindex_hash->{c} += 1;
3757             }
3758         }
3759
3760         if($drive->{interface} eq 'virtio'){
3761            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3762         }
3763
3764         if ($drive->{interface} eq 'scsi') {
3765
3766             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
3767
3768             $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges);
3769             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
3770
3771             my $iothread = '';
3772             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
3773                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3774                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3775             } elsif ($drive->{iothread}) {
3776                 warn "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n";
3777             }
3778
3779             my $queues = '';
3780             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
3781                 $queues = ",num_queues=$drive->{queues}";
3782             }
3783
3784             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues" if !$scsicontroller->{$controller};
3785             $scsicontroller->{$controller}=1;
3786         }
3787
3788         if ($drive->{interface} eq 'sata') {
3789            my $controller = int($drive->{index} / $MAX_SATA_DISKS);
3790            $pciaddr = print_pci_addr("ahci$controller", $bridges);
3791            push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr" if !$ahcicontroller->{$controller};
3792            $ahcicontroller->{$controller}=1;
3793         }
3794
3795         my $drive_cmd = print_drive_full($storecfg, $vmid, $drive);
3796         push @$devices, '-drive',$drive_cmd;
3797         push @$devices, '-device', print_drivedevice_full($storecfg, $conf, $vmid, $drive, $bridges);
3798     });
3799
3800     for (my $i = 0; $i < $MAX_NETS; $i++) {
3801          next if !$conf->{"net$i"};
3802          my $d = parse_net($conf->{"net$i"});
3803          next if !$d;
3804
3805          $use_virtio = 1 if $d->{model} eq 'virtio';
3806
3807          if ($bootindex_hash->{n}) {
3808             $d->{bootindex} = $bootindex_hash->{n};
3809             $bootindex_hash->{n} += 1;
3810          }
3811
3812          my $netdevfull = print_netdev_full($vmid,$conf,$d,"net$i");
3813          push @$devices, '-netdev', $netdevfull;
3814
3815          my $netdevicefull = print_netdevice_full($vmid, $conf, $d, "net$i", $bridges, $use_old_bios_files);
3816          push @$devices, '-device', $netdevicefull;
3817     }
3818
3819     if (!$q35) {
3820         # add pci bridges
3821         if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3822            $bridges->{1} = 1;
3823            $bridges->{2} = 1;
3824         }
3825
3826         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
3827
3828         while (my ($k, $v) = each %$bridges) {
3829             $pciaddr = print_pci_addr("pci.$k");
3830             unshift @$devices, '-device', "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr" if $k > 0;
3831         }
3832     }
3833
3834     # add custom args
3835     if ($conf->{args}) {
3836         my $aa = PVE::Tools::split_args($conf->{args});
3837         push @$cmd, @$aa;
3838     }
3839
3840     push @$cmd, @$devices;
3841     push @$cmd, '-rtc', join(',', @$rtcFlags)
3842         if scalar(@$rtcFlags);
3843     push @$cmd, '-machine', join(',', @$machineFlags)
3844         if scalar(@$machineFlags);
3845     push @$cmd, '-global', join(',', @$globalFlags)
3846         if scalar(@$globalFlags);
3847
3848     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
3849 }
3850
3851 sub vnc_socket {
3852     my ($vmid) = @_;
3853     return "${var_run_tmpdir}/$vmid.vnc";
3854 }
3855
3856 sub spice_port {
3857     my ($vmid) = @_;
3858
3859     my $res = vm_mon_cmd($vmid, 'query-spice');
3860
3861     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
3862 }
3863
3864 sub qmp_socket {
3865     my ($vmid, $qga, $name) = @_;
3866     my $sockettype = $qga ? 'qga' : 'qmp';
3867     my $ext = $name ? '-'.$name : '';
3868     return "${var_run_tmpdir}/$vmid$ext.$sockettype";
3869 }
3870
3871 sub pidfile_name {
3872     my ($vmid) = @_;
3873     return "${var_run_tmpdir}/$vmid.pid";
3874 }
3875
3876 sub vm_devices_list {
3877     my ($vmid) = @_;
3878
3879     my $res = vm_mon_cmd($vmid, 'query-pci');
3880     my $devices_to_check = [];
3881     my $devices = {};
3882     foreach my $pcibus (@$res) {
3883         push @$devices_to_check, @{$pcibus->{devices}},
3884     }
3885
3886     while (@$devices_to_check) {
3887         my $to_check = [];
3888         for my $d (@$devices_to_check) {
3889             $devices->{$d->{'qdev_id'}} = 1 if $d->{'qdev_id'};
3890             next if !$d->{'pci_bridge'};
3891
3892             $devices->{$d->{'qdev_id'}} += scalar(@{$d->{'pci_bridge'}->{devices}});
3893             push @$to_check, @{$d->{'pci_bridge'}->{devices}};
3894         }
3895         $devices_to_check = $to_check;
3896     }
3897
3898     my $resblock = vm_mon_cmd($vmid, 'query-block');
3899     foreach my $block (@$resblock) {
3900         if($block->{device} =~ m/^drive-(\S+)/){
3901                 $devices->{$1} = 1;
3902         }
3903     }
3904
3905     my $resmice = vm_mon_cmd($vmid, 'query-mice');
3906     foreach my $mice (@$resmice) {
3907         if ($mice->{name} eq 'QEMU HID Tablet') {
3908             $devices->{tablet} = 1;
3909             last;
3910         }
3911     }
3912
3913     # for usb devices there is no query-usb
3914     # but we can iterate over the entries in
3915     # qom-list path=/machine/peripheral
3916     my $resperipheral = vm_mon_cmd($vmid, 'qom-list', path => '/machine/peripheral');
3917     foreach my $per (@$resperipheral) {
3918         if ($per->{name} =~ m/^usb\d+$/) {
3919             $devices->{$per->{name}} = 1;
3920         }
3921     }
3922
3923     return $devices;
3924 }
3925
3926 sub vm_deviceplug {
3927     my ($storecfg, $conf, $vmid, $deviceid, $device) = @_;
3928
3929     my $q35 = machine_type_is_q35($conf);
3930
3931     my $devices_list = vm_devices_list($vmid);
3932     return 1 if defined($devices_list->{$deviceid});
3933
3934     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid); # add PCI bridge if we need it for the device
3935
3936     if ($deviceid eq 'tablet') {
3937
3938         qemu_deviceadd($vmid, print_tabletdevice_full($conf));
3939
3940     } elsif ($deviceid =~ m/^usb(\d+)$/) {
3941
3942         die "usb hotplug currently not reliable\n";
3943         # since we can't reliably hot unplug all added usb devices
3944         # and usb passthrough disables live migration
3945         # we disable usb hotplugging for now
3946         qemu_deviceadd($vmid, PVE::QemuServer::USB::print_usbdevice_full($conf, $deviceid, $device));
3947
3948     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3949
3950         qemu_iothread_add($vmid, $deviceid, $device);
3951
3952         qemu_driveadd($storecfg, $vmid, $device);
3953         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
3954
3955         qemu_deviceadd($vmid, $devicefull);
3956         eval { qemu_deviceaddverify($vmid, $deviceid); };
3957         if (my $err = $@) {
3958             eval { qemu_drivedel($vmid, $deviceid); };
3959             warn $@ if $@;
3960             die $err;
3961         }
3962
3963     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3964
3965
3966         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
3967         my $pciaddr = print_pci_addr($deviceid);
3968         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
3969
3970         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
3971
3972         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
3973             qemu_iothread_add($vmid, $deviceid, $device);
3974             $devicefull .= ",iothread=iothread-$deviceid";
3975         }
3976
3977         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
3978             $devicefull .= ",num_queues=$device->{queues}";
3979         }
3980
3981         qemu_deviceadd($vmid, $devicefull);
3982         qemu_deviceaddverify($vmid, $deviceid);
3983
3984     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3985
3986         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device);
3987         qemu_driveadd($storecfg, $vmid, $device);
3988
3989         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
3990         eval { qemu_deviceadd($vmid, $devicefull); };
3991         if (my $err = $@) {
3992             eval { qemu_drivedel($vmid, $deviceid); };
3993             warn $@ if $@;
3994             die $err;
3995         }
3996
3997     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3998
3999         return undef if !qemu_netdevadd($vmid, $conf, $device, $deviceid);
4000
4001         my $machine_type = PVE::QemuServer::qemu_machine_pxe($vmid, $conf);
4002         my $use_old_bios_files = undef;
4003         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
4004
4005         my $netdevicefull = print_netdevice_full($vmid, $conf, $device, $deviceid, undef, $use_old_bios_files);
4006         qemu_deviceadd($vmid, $netdevicefull);
4007         eval { qemu_deviceaddverify($vmid, $deviceid); };
4008         if (my $err = $@) {
4009             eval { qemu_netdevdel($vmid, $deviceid); };
4010             warn $@ if $@;
4011             die $err;
4012         }
4013
4014     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
4015
4016         my $bridgeid = $2;
4017         my $pciaddr = print_pci_addr($deviceid);
4018         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
4019
4020         qemu_deviceadd($vmid, $devicefull);
4021         qemu_deviceaddverify($vmid, $deviceid);
4022
4023     } else {
4024         die "can't hotplug device '$deviceid'\n";
4025     }
4026
4027     return 1;
4028 }
4029
4030 # fixme: this should raise exceptions on error!
4031 sub vm_deviceunplug {
4032     my ($vmid, $conf, $deviceid) = @_;
4033
4034     my $devices_list = vm_devices_list($vmid);
4035     return 1 if !defined($devices_list->{$deviceid});
4036
4037     die "can't unplug bootdisk" if $conf->{bootdisk} && $conf->{bootdisk} eq $deviceid;
4038
4039     if ($deviceid eq 'tablet') {
4040
4041         qemu_devicedel($vmid, $deviceid);
4042
4043     } elsif ($deviceid =~ m/^usb\d+$/) {
4044
4045         die "usb hotplug currently not reliable\n";
4046         # when unplugging usb devices this way,
4047         # there may be remaining usb controllers/hubs
4048         # so we disable it for now
4049         qemu_devicedel($vmid, $deviceid);
4050         qemu_devicedelverify($vmid, $deviceid);
4051
4052     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4053
4054         qemu_devicedel($vmid, $deviceid);
4055         qemu_devicedelverify($vmid, $deviceid);
4056         qemu_drivedel($vmid, $deviceid);
4057         qemu_iothread_del($conf, $vmid, $deviceid);
4058
4059     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4060
4061         qemu_devicedel($vmid, $deviceid);
4062         qemu_devicedelverify($vmid, $deviceid);
4063         qemu_iothread_del($conf, $vmid, $deviceid);
4064
4065     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4066
4067         qemu_devicedel($vmid, $deviceid);
4068         qemu_drivedel($vmid, $deviceid);
4069         qemu_deletescsihw($conf, $vmid, $deviceid);
4070
4071     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4072
4073         qemu_devicedel($vmid, $deviceid);
4074         qemu_devicedelverify($vmid, $deviceid);
4075         qemu_netdevdel($vmid, $deviceid);
4076
4077     } else {
4078         die "can't unplug device '$deviceid'\n";
4079     }
4080
4081     return 1;
4082 }
4083
4084 sub qemu_deviceadd {
4085     my ($vmid, $devicefull) = @_;
4086
4087     $devicefull = "driver=".$devicefull;
4088     my %options =  split(/[=,]/, $devicefull);
4089
4090     vm_mon_cmd($vmid, "device_add" , %options);
4091 }
4092
4093 sub qemu_devicedel {
4094     my ($vmid, $deviceid) = @_;
4095
4096     my $ret = vm_mon_cmd($vmid, "device_del", id => $deviceid);
4097 }
4098
4099 sub qemu_iothread_add {
4100     my($vmid, $deviceid, $device) = @_;
4101
4102     if ($device->{iothread}) {
4103         my $iothreads = vm_iothreads_list($vmid);
4104         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
4105     }
4106 }
4107
4108 sub qemu_iothread_del {
4109     my($conf, $vmid, $deviceid) = @_;
4110
4111     my $device = parse_drive($deviceid, $conf->{$deviceid});
4112     if ($device->{iothread}) {
4113         my $iothreads = vm_iothreads_list($vmid);
4114         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
4115     }
4116 }
4117
4118 sub qemu_objectadd {
4119     my($vmid, $objectid, $qomtype) = @_;
4120
4121     vm_mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
4122
4123     return 1;
4124 }
4125
4126 sub qemu_objectdel {
4127     my($vmid, $objectid) = @_;
4128
4129     vm_mon_cmd($vmid, "object-del", id => $objectid);
4130
4131     return 1;
4132 }
4133
4134 sub qemu_driveadd {
4135     my ($storecfg, $vmid, $device) = @_;
4136
4137     my $drive = print_drive_full($storecfg, $vmid, $device);
4138     $drive =~ s/\\/\\\\/g;
4139     my $ret = vm_human_monitor_command($vmid, "drive_add auto \"$drive\"");
4140
4141     # If the command succeeds qemu prints: "OK"
4142     return 1 if $ret =~ m/OK/s;
4143
4144     die "adding drive failed: $ret\n";
4145 }
4146
4147 sub qemu_drivedel {
4148     my($vmid, $deviceid) = @_;
4149
4150     my $ret = vm_human_monitor_command($vmid, "drive_del drive-$deviceid");
4151     $ret =~ s/^\s+//;
4152
4153     return 1 if $ret eq "";
4154
4155     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
4156     return 1 if $ret =~ m/Device \'.*?\' not found/s;
4157
4158     die "deleting drive $deviceid failed : $ret\n";
4159 }
4160
4161 sub qemu_deviceaddverify {
4162     my ($vmid, $deviceid) = @_;
4163
4164     for (my $i = 0; $i <= 5; $i++) {
4165          my $devices_list = vm_devices_list($vmid);
4166          return 1 if defined($devices_list->{$deviceid});
4167          sleep 1;
4168     }
4169
4170     die "error on hotplug device '$deviceid'\n";
4171 }
4172
4173
4174 sub qemu_devicedelverify {
4175     my ($vmid, $deviceid) = @_;
4176
4177     # need to verify that the device is correctly removed as device_del
4178     # is async and empty return is not reliable
4179
4180     for (my $i = 0; $i <= 5; $i++) {
4181          my $devices_list = vm_devices_list($vmid);
4182          return 1 if !defined($devices_list->{$deviceid});
4183          sleep 1;
4184     }
4185
4186     die "error on hot-unplugging device '$deviceid'\n";
4187 }
4188
4189 sub qemu_findorcreatescsihw {
4190     my ($storecfg, $conf, $vmid, $device) = @_;
4191
4192     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4193
4194     my $scsihwid="$controller_prefix$controller";
4195     my $devices_list = vm_devices_list($vmid);
4196
4197     if(!defined($devices_list->{$scsihwid})) {
4198         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device);
4199     }
4200
4201     return 1;
4202 }
4203
4204 sub qemu_deletescsihw {
4205     my ($conf, $vmid, $opt) = @_;
4206
4207     my $device = parse_drive($opt, $conf->{$opt});
4208
4209     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
4210         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
4211         return 1;
4212     }
4213
4214     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4215
4216     my $devices_list = vm_devices_list($vmid);
4217     foreach my $opt (keys %{$devices_list}) {
4218         if (PVE::QemuServer::is_valid_drivename($opt)) {
4219             my $drive = PVE::QemuServer::parse_drive($opt, $conf->{$opt});
4220             if($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
4221                 return 1;
4222             }
4223         }
4224     }
4225
4226     my $scsihwid="scsihw$controller";
4227
4228     vm_deviceunplug($vmid, $conf, $scsihwid);
4229
4230     return 1;
4231 }
4232
4233 sub qemu_add_pci_bridge {
4234     my ($storecfg, $conf, $vmid, $device) = @_;
4235
4236     my $bridges = {};
4237
4238     my $bridgeid;
4239
4240     print_pci_addr($device, $bridges);
4241
4242     while (my ($k, $v) = each %$bridges) {
4243         $bridgeid = $k;
4244     }
4245     return 1 if !defined($bridgeid) || $bridgeid < 1;
4246
4247     my $bridge = "pci.$bridgeid";
4248     my $devices_list = vm_devices_list($vmid);
4249
4250     if (!defined($devices_list->{$bridge})) {
4251         vm_deviceplug($storecfg, $conf, $vmid, $bridge);
4252     }
4253
4254     return 1;
4255 }
4256
4257 sub qemu_set_link_status {
4258     my ($vmid, $device, $up) = @_;
4259
4260     vm_mon_cmd($vmid, "set_link", name => $device,
4261                up => $up ? JSON::true : JSON::false);
4262 }
4263
4264 sub qemu_netdevadd {
4265     my ($vmid, $conf, $device, $deviceid) = @_;
4266
4267     my $netdev = print_netdev_full($vmid, $conf, $device, $deviceid, 1);
4268     my %options =  split(/[=,]/, $netdev);
4269
4270     vm_mon_cmd($vmid, "netdev_add",  %options);
4271     return 1;
4272 }
4273
4274 sub qemu_netdevdel {
4275     my ($vmid, $deviceid) = @_;
4276
4277     vm_mon_cmd($vmid, "netdev_del", id => $deviceid);
4278 }
4279
4280 sub qemu_usb_hotplug {
4281     my ($storecfg, $conf, $vmid, $deviceid, $device) = @_;
4282
4283     return if !$device;
4284
4285     # remove the old one first
4286     vm_deviceunplug($vmid, $conf, $deviceid);
4287
4288     # check if xhci controller is necessary and available
4289     if ($device->{usb3}) {
4290
4291         my $devicelist = vm_devices_list($vmid);
4292
4293         if (!$devicelist->{xhci}) {
4294             my $pciaddr = print_pci_addr("xhci");
4295             qemu_deviceadd($vmid, "nec-usb-xhci,id=xhci$pciaddr");
4296         }
4297     }
4298     my $d = parse_usb_device($device->{host});
4299     $d->{usb3} = $device->{usb3};
4300
4301     # add the new one
4302     vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $d);
4303 }
4304
4305 sub qemu_cpu_hotplug {
4306     my ($vmid, $conf, $vcpus) = @_;
4307
4308     my $machine_type = PVE::QemuServer::get_current_qemu_machine($vmid);
4309
4310     my $sockets = 1;
4311     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
4312     $sockets = $conf->{sockets} if  $conf->{sockets};
4313     my $cores = $conf->{cores} || 1;
4314     my $maxcpus = $sockets * $cores;
4315
4316     $vcpus = $maxcpus if !$vcpus;
4317
4318     die "you can't add more vcpus than maxcpus\n"
4319         if $vcpus > $maxcpus;
4320
4321     my $currentvcpus = $conf->{vcpus} || $maxcpus;
4322
4323     if ($vcpus < $currentvcpus) {
4324
4325         if (qemu_machine_feature_enabled ($machine_type, undef, 2, 7)) {
4326
4327             for (my $i = $currentvcpus; $i > $vcpus; $i--) {
4328                 qemu_devicedel($vmid, "cpu$i");
4329                 my $retry = 0;
4330                 my $currentrunningvcpus = undef;
4331                 while (1) {
4332                     $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
4333                     last if scalar(@{$currentrunningvcpus}) == $i-1;
4334                     raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
4335                     $retry++;
4336                     sleep 1;
4337                 }
4338                 #update conf after each succesfull cpu unplug
4339                 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4340                 PVE::QemuConfig->write_config($vmid, $conf);
4341             }
4342         } else {
4343             die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
4344         }
4345
4346         return;
4347     }
4348
4349     my $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
4350     die "vcpus in running vm does not match its configuration\n"
4351         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
4352
4353     if (qemu_machine_feature_enabled ($machine_type, undef, 2, 7)) {
4354
4355         for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
4356             my $cpustr = print_cpu_device($conf, $i);
4357             qemu_deviceadd($vmid, $cpustr);
4358
4359             my $retry = 0;
4360             my $currentrunningvcpus = undef;
4361             while (1) {
4362                 $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
4363                 last if scalar(@{$currentrunningvcpus}) == $i;
4364                 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
4365                 sleep 1;
4366                 $retry++;
4367             }
4368             #update conf after each succesfull cpu hotplug
4369             $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4370             PVE::QemuConfig->write_config($vmid, $conf);
4371         }
4372     } else {
4373
4374         for (my $i = $currentvcpus; $i < $vcpus; $i++) {
4375             vm_mon_cmd($vmid, "cpu-add", id => int($i));
4376         }
4377     }
4378 }
4379
4380 sub qemu_block_set_io_throttle {
4381     my ($vmid, $deviceid,
4382         $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
4383         $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max,
4384         $bps_max_length, $bps_rd_max_length, $bps_wr_max_length,
4385         $iops_max_length, $iops_rd_max_length, $iops_wr_max_length) = @_;
4386
4387     return if !check_running($vmid) ;
4388
4389     vm_mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
4390         bps => int($bps),
4391         bps_rd => int($bps_rd),
4392         bps_wr => int($bps_wr),
4393         iops => int($iops),
4394         iops_rd => int($iops_rd),
4395         iops_wr => int($iops_wr),
4396         bps_max => int($bps_max),
4397         bps_rd_max => int($bps_rd_max),
4398         bps_wr_max => int($bps_wr_max),
4399         iops_max => int($iops_max),
4400         iops_rd_max => int($iops_rd_max),
4401         iops_wr_max => int($iops_wr_max),
4402         bps_max_length => int($bps_max_length),
4403         bps_rd_max_length => int($bps_rd_max_length),
4404         bps_wr_max_length => int($bps_wr_max_length),
4405         iops_max_length => int($iops_max_length),
4406         iops_rd_max_length => int($iops_rd_max_length),
4407         iops_wr_max_length => int($iops_wr_max_length),
4408     );
4409
4410 }
4411
4412 # old code, only used to shutdown old VM after update
4413 sub __read_avail {
4414     my ($fh, $timeout) = @_;
4415
4416     my $sel = new IO::Select;
4417     $sel->add($fh);
4418
4419     my $res = '';
4420     my $buf;
4421
4422     my @ready;
4423     while (scalar (@ready = $sel->can_read($timeout))) {
4424         my $count;
4425         if ($count = $fh->sysread($buf, 8192)) {
4426             if ($buf =~ /^(.*)\(qemu\) $/s) {
4427                 $res .= $1;
4428                 last;
4429             } else {
4430                 $res .= $buf;
4431             }
4432         } else {
4433             if (!defined($count)) {
4434                 die "$!\n";
4435             }
4436             last;
4437         }
4438     }
4439
4440     die "monitor read timeout\n" if !scalar(@ready);
4441
4442     return $res;
4443 }
4444
4445 sub qemu_block_resize {
4446     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4447
4448     my $running = check_running($vmid);
4449
4450     $size = 0 if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4451
4452     return if !$running;
4453
4454     vm_mon_cmd($vmid, "block_resize", device => $deviceid, size => int($size));
4455
4456 }
4457
4458 sub qemu_volume_snapshot {
4459     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4460
4461     my $running = check_running($vmid);
4462
4463     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4464         vm_mon_cmd($vmid, 'blockdev-snapshot-internal-sync', device => $deviceid, name => $snap);
4465     } else {
4466         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4467     }
4468 }
4469
4470 sub qemu_volume_snapshot_delete {
4471     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4472
4473     my $running = check_running($vmid);
4474
4475     if($running) {
4476
4477         $running = undef;
4478         my $conf = PVE::QemuConfig->load_config($vmid);
4479         foreach_drive($conf, sub {
4480             my ($ds, $drive) = @_;
4481             $running = 1 if $drive->{file} eq $volid;
4482         });
4483     }
4484
4485     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4486         vm_mon_cmd($vmid, 'blockdev-snapshot-delete-internal-sync', device => $deviceid, name => $snap);
4487     } else {
4488         PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4489     }
4490 }
4491
4492 sub set_migration_caps {
4493     my ($vmid) = @_;
4494
4495     my $cap_ref = [];
4496
4497     my $enabled_cap = {
4498         "auto-converge" => 1,
4499         "xbzrle" => 1,
4500         "x-rdma-pin-all" => 0,
4501         "zero-blocks" => 0,
4502         "compress" => 0
4503     };
4504
4505     my $supported_capabilities = vm_mon_cmd_nocheck($vmid, "query-migrate-capabilities");
4506
4507     for my $supported_capability (@$supported_capabilities) {
4508         push @$cap_ref, {
4509             capability => $supported_capability->{capability},
4510             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4511         };
4512     }
4513
4514     vm_mon_cmd_nocheck($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4515 }
4516
4517 my $fast_plug_option = {
4518     'lock' => 1,
4519     'name' => 1,
4520     'onboot' => 1,
4521     'shares' => 1,
4522     'startup' => 1,
4523     'description' => 1,
4524     'protection' => 1,
4525     'vmstatestorage' => 1,
4526 };
4527
4528 # hotplug changes in [PENDING]
4529 # $selection hash can be used to only apply specified options, for
4530 # example: { cores => 1 } (only apply changed 'cores')
4531 # $errors ref is used to return error messages
4532 sub vmconfig_hotplug_pending {
4533     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4534
4535     my $defaults = load_defaults();
4536
4537     # commit values which do not have any impact on running VM first
4538     # Note: those option cannot raise errors, we we do not care about
4539     # $selection and always apply them.
4540
4541     my $add_error = sub {
4542         my ($opt, $msg) = @_;
4543         $errors->{$opt} = "hotplug problem - $msg";
4544     };
4545
4546     my $changes = 0;
4547     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4548         if ($fast_plug_option->{$opt}) {
4549             $conf->{$opt} = $conf->{pending}->{$opt};
4550             delete $conf->{pending}->{$opt};
4551             $changes = 1;
4552         }
4553     }
4554
4555     if ($changes) {
4556         PVE::QemuConfig->write_config($vmid, $conf);
4557         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4558     }
4559
4560     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4561
4562     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4563     while (my ($opt, $force) = each %$pending_delete_hash) {
4564         next if $selection && !$selection->{$opt};
4565         eval {
4566             if ($opt eq 'hotplug') {
4567                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
4568             } elsif ($opt eq 'tablet') {
4569                 die "skip\n" if !$hotplug_features->{usb};
4570                 if ($defaults->{tablet}) {
4571                     vm_deviceplug($storecfg, $conf, $vmid, $opt);
4572                 } else {
4573                     vm_deviceunplug($vmid, $conf, $opt);
4574                 }
4575             } elsif ($opt =~ m/^usb\d+/) {
4576                 die "skip\n";
4577                 # since we cannot reliably hot unplug usb devices
4578                 # we are disabling it
4579                 die "skip\n" if !$hotplug_features->{usb} || $conf->{$opt} =~ m/spice/i;
4580                 vm_deviceunplug($vmid, $conf, $opt);
4581             } elsif ($opt eq 'vcpus') {
4582                 die "skip\n" if !$hotplug_features->{cpu};
4583                 qemu_cpu_hotplug($vmid, $conf, undef);
4584             } elsif ($opt eq 'balloon') {
4585                 # enable balloon device is not hotpluggable
4586                 die "skip\n" if defined($conf->{balloon}) && $conf->{balloon} == 0;
4587                 # here we reset the ballooning value to memory
4588                 my $balloon = $conf->{memory} || $defaults->{memory};
4589                 vm_mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4590             } elsif ($fast_plug_option->{$opt}) {
4591                 # do nothing
4592             } elsif ($opt =~ m/^net(\d+)$/) {
4593                 die "skip\n" if !$hotplug_features->{network};
4594                 vm_deviceunplug($vmid, $conf, $opt);
4595             } elsif (is_valid_drivename($opt)) {
4596                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4597                 vm_deviceunplug($vmid, $conf, $opt);
4598                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4599             } elsif ($opt =~ m/^memory$/) {
4600                 die "skip\n" if !$hotplug_features->{memory};
4601                 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
4602             } elsif ($opt eq 'cpuunits') {
4603                 cgroups_write("cpu", $vmid, "cpu.shares", $defaults->{cpuunits});
4604             } elsif ($opt eq 'cpulimit') {
4605                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", -1);
4606             } else {
4607                 die "skip\n";
4608             }
4609         };
4610         if (my $err = $@) {
4611             &$add_error($opt, $err) if $err ne "skip\n";
4612         } else {
4613             # save new config if hotplug was successful
4614             delete $conf->{$opt};
4615             vmconfig_undelete_pending_option($conf, $opt);
4616             PVE::QemuConfig->write_config($vmid, $conf);
4617             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4618         }
4619     }
4620
4621     my $apply_pending_cloudinit;
4622     $apply_pending_cloudinit = sub {
4623         my ($key, $value) = @_;
4624         $apply_pending_cloudinit = sub {}; # once is enough
4625
4626         my @cloudinit_opts = keys %$confdesc_cloudinit;
4627         foreach my $opt (keys %{$conf->{pending}}) {
4628             next if !grep { $_ eq $opt } @cloudinit_opts;
4629             $conf->{$opt} = delete $conf->{pending}->{$opt};
4630         }
4631
4632         my $new_conf = { %$conf };
4633         $new_conf->{$key} = $value;
4634         PVE::QemuServer::Cloudinit::generate_cloudinitconfig($new_conf, $vmid);
4635     };
4636
4637     foreach my $opt (keys %{$conf->{pending}}) {
4638         next if $selection && !$selection->{$opt};
4639         my $value = $conf->{pending}->{$opt};
4640         eval {
4641             if ($opt eq 'hotplug') {
4642                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
4643             } elsif ($opt eq 'tablet') {
4644                 die "skip\n" if !$hotplug_features->{usb};
4645                 if ($value == 1) {
4646                     vm_deviceplug($storecfg, $conf, $vmid, $opt);
4647                 } elsif ($value == 0) {
4648                     vm_deviceunplug($vmid, $conf, $opt);
4649                 }
4650             } elsif ($opt =~ m/^usb\d+$/) {
4651                 die "skip\n";
4652                 # since we cannot reliably hot unplug usb devices
4653                 # we are disabling it
4654                 die "skip\n" if !$hotplug_features->{usb} || $value =~ m/spice/i;
4655                 my $d = eval { PVE::JSONSchema::parse_property_string($usbdesc->{format}, $value) };
4656                 die "skip\n" if !$d;
4657                 qemu_usb_hotplug($storecfg, $conf, $vmid, $opt, $d);
4658             } elsif ($opt eq 'vcpus') {
4659                 die "skip\n" if !$hotplug_features->{cpu};
4660                 qemu_cpu_hotplug($vmid, $conf, $value);
4661             } elsif ($opt eq 'balloon') {
4662                 # enable/disable balloning device is not hotpluggable
4663                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
4664                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
4665                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
4666
4667                 # allow manual ballooning if shares is set to zero
4668                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
4669                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
4670                     vm_mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4671                 }
4672             } elsif ($opt =~ m/^net(\d+)$/) {
4673                 # some changes can be done without hotplug
4674                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
4675                                     $vmid, $opt, $value);
4676             } elsif (is_valid_drivename($opt)) {
4677                 # some changes can be done without hotplug
4678                 my $drive = parse_drive($opt, $value);
4679                 if (drive_is_cloudinit($drive)) {
4680                     &$apply_pending_cloudinit($opt, $value);
4681                 }
4682                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
4683                                      $vmid, $opt, $value, 1);
4684             } elsif ($opt =~ m/^memory$/) { #dimms
4685                 die "skip\n" if !$hotplug_features->{memory};
4686                 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
4687             } elsif ($opt eq 'cpuunits') {
4688                 cgroups_write("cpu", $vmid, "cpu.shares", $conf->{pending}->{$opt});
4689             } elsif ($opt eq 'cpulimit') {
4690                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
4691                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", $cpulimit);
4692             } else {
4693                 die "skip\n";  # skip non-hot-pluggable options
4694             }
4695         };
4696         if (my $err = $@) {
4697             &$add_error($opt, $err) if $err ne "skip\n";
4698         } else {
4699             # save new config if hotplug was successful
4700             $conf->{$opt} = $value;
4701             delete $conf->{pending}->{$opt};
4702             PVE::QemuConfig->write_config($vmid, $conf);
4703             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4704         }
4705     }
4706 }
4707
4708 sub try_deallocate_drive {
4709     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
4710
4711     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
4712         my $volid = $drive->{file};
4713         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
4714             my $sid = PVE::Storage::parse_volume_id($volid);
4715             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
4716
4717             # check if the disk is really unused
4718             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
4719                 if is_volume_in_use($storecfg, $conf, $key, $volid);
4720             PVE::Storage::vdisk_free($storecfg, $volid);
4721             return 1;
4722         } else {
4723             # If vm is not owner of this disk remove from config
4724             return 1;
4725         }
4726     }
4727
4728     return undef;
4729 }
4730
4731 sub vmconfig_delete_or_detach_drive {
4732     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
4733
4734     my $drive = parse_drive($opt, $conf->{$opt});
4735
4736     my $rpcenv = PVE::RPCEnvironment::get();
4737     my $authuser = $rpcenv->get_user();
4738
4739     if ($force) {
4740         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
4741         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
4742     } else {
4743         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
4744     }
4745 }
4746
4747 sub vmconfig_apply_pending {
4748     my ($vmid, $conf, $storecfg) = @_;
4749
4750     # cold plug
4751
4752     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4753     while (my ($opt, $force) = each %$pending_delete_hash) {
4754         die "internal error" if $opt =~ m/^unused/;
4755         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4756         if (!defined($conf->{$opt})) {
4757             vmconfig_undelete_pending_option($conf, $opt);
4758             PVE::QemuConfig->write_config($vmid, $conf);
4759         } elsif (is_valid_drivename($opt)) {
4760             vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4761             vmconfig_undelete_pending_option($conf, $opt);
4762             delete $conf->{$opt};
4763             PVE::QemuConfig->write_config($vmid, $conf);
4764         } else {
4765             vmconfig_undelete_pending_option($conf, $opt);
4766             delete $conf->{$opt};
4767             PVE::QemuConfig->write_config($vmid, $conf);
4768         }
4769     }
4770
4771     $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4772
4773     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4774         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4775
4776         if (defined($conf->{$opt}) && ($conf->{$opt} eq $conf->{pending}->{$opt})) {
4777             # skip if nothing changed
4778         } elsif (is_valid_drivename($opt)) {
4779             vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
4780                 if defined($conf->{$opt});
4781             $conf->{$opt} = $conf->{pending}->{$opt};
4782         } else {
4783             $conf->{$opt} = $conf->{pending}->{$opt};
4784         }
4785
4786         delete $conf->{pending}->{$opt};
4787         PVE::QemuConfig->write_config($vmid, $conf);
4788     }
4789 }
4790
4791 my $safe_num_ne = sub {
4792     my ($a, $b) = @_;
4793
4794     return 0 if !defined($a) && !defined($b);
4795     return 1 if !defined($a);
4796     return 1 if !defined($b);
4797
4798     return $a != $b;
4799 };
4800
4801 my $safe_string_ne = sub {
4802     my ($a, $b) = @_;
4803
4804     return 0 if !defined($a) && !defined($b);
4805     return 1 if !defined($a);
4806     return 1 if !defined($b);
4807
4808     return $a ne $b;
4809 };
4810
4811 sub vmconfig_update_net {
4812     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value) = @_;
4813
4814     my $newnet = parse_net($value);
4815
4816     if ($conf->{$opt}) {
4817         my $oldnet = parse_net($conf->{$opt});
4818
4819         if (&$safe_string_ne($oldnet->{model}, $newnet->{model}) ||
4820             &$safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
4821             &$safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
4822             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
4823
4824             # for non online change, we try to hot-unplug
4825             die "skip\n" if !$hotplug;
4826             vm_deviceunplug($vmid, $conf, $opt);
4827         } else {
4828
4829             die "internal error" if $opt !~ m/net(\d+)/;
4830             my $iface = "tap${vmid}i$1";
4831
4832             if (&$safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
4833                 &$safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
4834                 &$safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
4835                 &$safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
4836                 PVE::Network::tap_unplug($iface);
4837                 PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
4838             } elsif (&$safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
4839                 # Rate can be applied on its own but any change above needs to
4840                 # include the rate in tap_plug since OVS resets everything.
4841                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
4842             }
4843
4844             if (&$safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
4845                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
4846             }
4847
4848             return 1;
4849         }
4850     }
4851
4852     if ($hotplug) {
4853         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet);
4854     } else {
4855         die "skip\n";
4856     }
4857 }
4858
4859 sub vmconfig_update_disk {
4860     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $force) = @_;
4861
4862     # fixme: do we need force?
4863
4864     my $drive = parse_drive($opt, $value);
4865
4866     if ($conf->{$opt}) {
4867
4868         if (my $old_drive = parse_drive($opt, $conf->{$opt}))  {
4869
4870             my $media = $drive->{media} || 'disk';
4871             my $oldmedia = $old_drive->{media} || 'disk';
4872             die "unable to change media type\n" if $media ne $oldmedia;
4873
4874             if (!drive_is_cdrom($old_drive)) {
4875
4876                 if ($drive->{file} ne $old_drive->{file}) {
4877
4878                     die "skip\n" if !$hotplug;
4879
4880                     # unplug and register as unused
4881                     vm_deviceunplug($vmid, $conf, $opt);
4882                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
4883
4884                 } else {
4885                     # update existing disk
4886
4887                     # skip non hotpluggable value
4888                     if (&$safe_string_ne($drive->{discard}, $old_drive->{discard}) ||
4889                         &$safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
4890                         &$safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
4891                         &$safe_string_ne($drive->{cache}, $old_drive->{cache})) {
4892                         die "skip\n";
4893                     }
4894
4895                     # apply throttle
4896                     if (&$safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
4897                         &$safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
4898                         &$safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
4899                         &$safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
4900                         &$safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
4901                         &$safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
4902                         &$safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
4903                         &$safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
4904                         &$safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
4905                         &$safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
4906                         &$safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
4907                         &$safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max}) ||
4908                         &$safe_num_ne($drive->{bps_max_length}, $old_drive->{bps_max_length}) ||
4909                         &$safe_num_ne($drive->{bps_rd_max_length}, $old_drive->{bps_rd_max_length}) ||
4910                         &$safe_num_ne($drive->{bps_wr_max_length}, $old_drive->{bps_wr_max_length}) ||
4911                         &$safe_num_ne($drive->{iops_max_length}, $old_drive->{iops_max_length}) ||
4912                         &$safe_num_ne($drive->{iops_rd_max_length}, $old_drive->{iops_rd_max_length}) ||
4913                         &$safe_num_ne($drive->{iops_wr_max_length}, $old_drive->{iops_wr_max_length})) {
4914
4915                         qemu_block_set_io_throttle($vmid,"drive-$opt",
4916                                                    ($drive->{mbps} || 0)*1024*1024,
4917                                                    ($drive->{mbps_rd} || 0)*1024*1024,
4918                                                    ($drive->{mbps_wr} || 0)*1024*1024,
4919                                                    $drive->{iops} || 0,
4920                                                    $drive->{iops_rd} || 0,
4921                                                    $drive->{iops_wr} || 0,
4922                                                    ($drive->{mbps_max} || 0)*1024*1024,
4923                                                    ($drive->{mbps_rd_max} || 0)*1024*1024,
4924                                                    ($drive->{mbps_wr_max} || 0)*1024*1024,
4925                                                    $drive->{iops_max} || 0,
4926                                                    $drive->{iops_rd_max} || 0,
4927                                                    $drive->{iops_wr_max} || 0,
4928                                                    $drive->{bps_max_length} || 1,
4929                                                    $drive->{bps_rd_max_length} || 1,
4930                                                    $drive->{bps_wr_max_length} || 1,
4931                                                    $drive->{iops_max_length} || 1,
4932                                                    $drive->{iops_rd_max_length} || 1,
4933                                                    $drive->{iops_wr_max_length} || 1);
4934
4935                     }
4936
4937                     return 1;
4938                 }
4939
4940             } else { # cdrom
4941
4942                 if ($drive->{file} eq 'none') {
4943                     vm_mon_cmd($vmid, "eject",force => JSON::true,device => "drive-$opt");
4944                     if (drive_is_cloudinit($old_drive)) {
4945                         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive);
4946                     }
4947                 } else {
4948                     my $path = get_iso_path($storecfg, $vmid, $drive->{file});
4949                     vm_mon_cmd($vmid, "eject", force => JSON::true,device => "drive-$opt"); # force eject if locked
4950                     vm_mon_cmd($vmid, "change", device => "drive-$opt",target => "$path") if $path;
4951                 }
4952
4953                 return 1;
4954             }
4955         }
4956     }
4957
4958     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
4959     # hotplug new disks
4960     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
4961     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive);
4962 }
4963
4964 sub vm_start {
4965     my ($storecfg, $vmid, $statefile, $skiplock, $migratedfrom, $paused,
4966         $forcemachine, $spice_ticket, $migration_network, $migration_type, $targetstorage) = @_;
4967
4968     PVE::QemuConfig->lock_config($vmid, sub {
4969         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
4970
4971         die "you can't start a vm if it's a template\n" if PVE::QemuConfig->is_template($conf);
4972
4973         PVE::QemuConfig->check_lock($conf) if !$skiplock;
4974
4975         die "VM $vmid already running\n" if check_running($vmid, undef, $migratedfrom);
4976
4977         if (!$statefile && scalar(keys %{$conf->{pending}})) {
4978             vmconfig_apply_pending($vmid, $conf, $storecfg);
4979             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4980         }
4981
4982         PVE::QemuServer::Cloudinit::generate_cloudinitconfig($conf, $vmid);
4983
4984         my $defaults = load_defaults();
4985
4986         # set environment variable useful inside network script
4987         $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
4988
4989         my $local_volumes = {};
4990
4991         if ($targetstorage) {
4992             foreach_drive($conf, sub {
4993                 my ($ds, $drive) = @_;
4994
4995                 return if drive_is_cdrom($drive);
4996
4997                 my $volid = $drive->{file};
4998
4999                 return if !$volid;
5000
5001                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
5002
5003                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5004                 return if $scfg->{shared};
5005                 $local_volumes->{$ds} = [$volid, $storeid, $volname];
5006             });
5007
5008             my $format = undef;
5009
5010             foreach my $opt (sort keys %$local_volumes) {
5011
5012                 my ($volid, $storeid, $volname) = @{$local_volumes->{$opt}};
5013                 my $drive = parse_drive($opt, $conf->{$opt});
5014
5015                 #if remote storage is specified, use default format
5016                 if ($targetstorage && $targetstorage ne "1") {
5017                     $storeid = $targetstorage;
5018                     my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
5019                     $format = $defFormat;
5020                 } else {
5021                     #else we use same format than original
5022                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5023                     $format = qemu_img_format($scfg, $volid);
5024                 }
5025
5026                 my $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $format, undef, ($drive->{size}/1024));
5027                 my $newdrive = $drive;
5028                 $newdrive->{format} = $format;
5029                 $newdrive->{file} = $newvolid;
5030                 my $drivestr = PVE::QemuServer::print_drive($vmid, $newdrive);
5031                 $local_volumes->{$opt} = $drivestr;
5032                 #pass drive to conf for command line
5033                 $conf->{$opt} = $drivestr;
5034             }
5035         }
5036
5037         my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine);
5038
5039         my $migrate_port = 0;
5040         my $migrate_uri;
5041         if ($statefile) {
5042             if ($statefile eq 'tcp') {
5043                 my $localip = "localhost";
5044                 my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
5045                 my $nodename = PVE::INotify::nodename();
5046
5047                 if (!defined($migration_type)) {
5048                     if (defined($datacenterconf->{migration}->{type})) {
5049                         $migration_type = $datacenterconf->{migration}->{type};
5050                     } else {
5051                         $migration_type = 'secure';
5052                     }
5053                 }
5054
5055                 if ($migration_type eq 'insecure') {
5056                     my $migrate_network_addr = PVE::Cluster::get_local_migration_ip($migration_network);
5057                     if ($migrate_network_addr) {
5058                         $localip = $migrate_network_addr;
5059                     } else {
5060                         $localip = PVE::Cluster::remote_node_ip($nodename, 1);
5061                     }
5062
5063                     $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5064                 }
5065
5066                 my $pfamily = PVE::Tools::get_host_address_family($nodename);
5067                 $migrate_port = PVE::Tools::next_migrate_port($pfamily);
5068                 $migrate_uri = "tcp:${localip}:${migrate_port}";
5069                 push @$cmd, '-incoming', $migrate_uri;
5070                 push @$cmd, '-S';
5071
5072             } elsif ($statefile eq 'unix') {
5073                 # should be default for secure migrations as a ssh TCP forward
5074                 # tunnel is not deterministic reliable ready and fails regurarly
5075                 # to set up in time, so use UNIX socket forwards
5076                 my $socket_addr = "/run/qemu-server/$vmid.migrate";
5077                 unlink $socket_addr;
5078
5079                 $migrate_uri = "unix:$socket_addr";
5080
5081                 push @$cmd, '-incoming', $migrate_uri;
5082                 push @$cmd, '-S';
5083
5084             } else {
5085                 push @$cmd, '-loadstate', $statefile;
5086             }
5087         } elsif ($paused) {
5088             push @$cmd, '-S';
5089         }
5090
5091         # host pci devices
5092         for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
5093           my $d = parse_hostpci($conf->{"hostpci$i"});
5094           next if !$d;
5095           my $pcidevices = $d->{pciid};
5096           foreach my $pcidevice (@$pcidevices) {
5097                 my $pciid = $pcidevice->{id}.".".$pcidevice->{function};
5098
5099                 my $info = pci_device_info("0000:$pciid");
5100                 die "IOMMU not present\n" if !check_iommu_support();
5101                 die "no pci device info for device '$pciid'\n" if !$info;
5102                 die "can't unbind/bind pci group to vfio '$pciid'\n" if !pci_dev_group_bind_to_vfio($pciid);
5103                 die "can't reset pci device '$pciid'\n" if $info->{has_fl_reset} and !pci_dev_reset($info);
5104           }
5105         }
5106
5107         PVE::Storage::activate_volumes($storecfg, $vollist);
5108
5109         if (!check_running($vmid, 1)) {
5110             eval {
5111                 run_command(['/bin/systemctl', 'stop', "$vmid.scope"],
5112                     outfunc => sub {}, errfunc => sub {});
5113             };
5114         }
5115
5116         my $cpuunits = defined($conf->{cpuunits}) ? $conf->{cpuunits}
5117                                                   : $defaults->{cpuunits};
5118
5119         my $start_timeout = $conf->{hugepages} ? 300 : 30;
5120         my %run_params = (timeout => $statefile ? undef : $start_timeout, umask => 0077);
5121
5122         my %properties = (
5123             Slice => 'qemu.slice',
5124             KillMode => 'none',
5125             CPUShares => $cpuunits
5126         );
5127
5128         if (my $cpulimit = $conf->{cpulimit}) {
5129             $properties{CPUQuota} = int($cpulimit * 100);
5130         }
5131         $properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
5132
5133         my $run_qemu = sub {
5134             PVE::Tools::run_fork sub {
5135                 PVE::Systemd::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
5136                 run_command($cmd, %run_params);
5137             };
5138         };
5139
5140         if ($conf->{hugepages}) {
5141
5142             my $code = sub {
5143                 my $hugepages_topology = PVE::QemuServer::Memory::hugepages_topology($conf);
5144                 my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
5145
5146                 PVE::QemuServer::Memory::hugepages_mount();
5147                 PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
5148
5149                 eval { $run_qemu->() };
5150                 if (my $err = $@) {
5151                     PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology);
5152                     die $err;
5153                 }
5154
5155                 PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology);
5156             };
5157             eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
5158
5159         } else {
5160             eval { $run_qemu->() };
5161         }
5162
5163         if (my $err = $@) {
5164             # deactivate volumes if start fails
5165             eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
5166             die "start failed: $err";
5167         }
5168
5169         print "migration listens on $migrate_uri\n" if $migrate_uri;
5170
5171         if ($statefile && $statefile ne 'tcp' && $statefile ne 'unix')  {
5172             eval { vm_mon_cmd_nocheck($vmid, "cont"); };
5173             warn $@ if $@;
5174         }
5175
5176         #start nbd server for storage migration
5177         if ($targetstorage) {
5178             my $nodename = PVE::INotify::nodename();
5179             my $migrate_network_addr = PVE::Cluster::get_local_migration_ip($migration_network);
5180             my $localip = $migrate_network_addr ? $migrate_network_addr : PVE::Cluster::remote_node_ip($nodename, 1);
5181             my $pfamily = PVE::Tools::get_host_address_family($nodename);
5182             $migrate_port = PVE::Tools::next_migrate_port($pfamily);
5183
5184             vm_mon_cmd_nocheck($vmid, "nbd-server-start", addr => { type => 'inet', data => { host => "${localip}", port => "${migrate_port}" } } );
5185
5186             $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5187
5188             foreach my $opt (sort keys %$local_volumes) {
5189                 my $volid = $local_volumes->{$opt};
5190                 vm_mon_cmd_nocheck($vmid, "nbd-server-add", device => "drive-$opt", writable => JSON::true );
5191                 my $migrate_storage_uri = "nbd:${localip}:${migrate_port}:exportname=drive-$opt";
5192                 print "storage migration listens on $migrate_storage_uri volume:$volid\n";
5193             }
5194         }
5195
5196         if ($migratedfrom) {
5197             eval {
5198                 set_migration_caps($vmid);
5199             };
5200             warn $@ if $@;
5201
5202             if ($spice_port) {
5203                 print "spice listens on port $spice_port\n";
5204                 if ($spice_ticket) {
5205                     vm_mon_cmd_nocheck($vmid, "set_password", protocol => 'spice', password => $spice_ticket);
5206                     vm_mon_cmd_nocheck($vmid, "expire_password", protocol => 'spice', time => "+30");
5207                 }
5208             }
5209
5210         } else {
5211             vm_mon_cmd_nocheck($vmid, "balloon", value => $conf->{balloon}*1024*1024)
5212                 if !$statefile && $conf->{balloon};
5213
5214             foreach my $opt (keys %$conf) {
5215                 next if $opt !~  m/^net\d+$/;
5216                 my $nicconf = parse_net($conf->{$opt});
5217                 qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
5218             }
5219         }
5220
5221         vm_mon_cmd_nocheck($vmid, 'qom-set',
5222                     path => "machine/peripheral/balloon0",
5223                     property => "guest-stats-polling-interval",
5224                     value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
5225
5226     });
5227 }
5228
5229 sub vm_mon_cmd {
5230     my ($vmid, $execute, %params) = @_;
5231
5232     my $cmd = { execute => $execute, arguments => \%params };
5233     vm_qmp_command($vmid, $cmd);
5234 }
5235
5236 sub vm_mon_cmd_nocheck {
5237     my ($vmid, $execute, %params) = @_;
5238
5239     my $cmd = { execute => $execute, arguments => \%params };
5240     vm_qmp_command($vmid, $cmd, 1);
5241 }
5242
5243 sub vm_qmp_command {
5244     my ($vmid, $cmd, $nocheck) = @_;
5245
5246     my $res;
5247
5248     my $timeout;
5249     if ($cmd->{arguments} && $cmd->{arguments}->{timeout}) {
5250         $timeout = $cmd->{arguments}->{timeout};
5251         delete $cmd->{arguments}->{timeout};
5252     }
5253
5254     eval {
5255         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
5256         my $sname = qmp_socket($vmid);
5257         if (-e $sname) { # test if VM is reasonambe new and supports qmp/qga
5258             my $qmpclient = PVE::QMPClient->new();
5259
5260             $res = $qmpclient->cmd($vmid, $cmd, $timeout);
5261         } else {
5262             die "unable to open monitor socket\n";
5263         }
5264     };
5265     if (my $err = $@) {
5266         syslog("err", "VM $vmid qmp command failed - $err");
5267         die $err;
5268     }
5269
5270     return $res;
5271 }
5272
5273 sub vm_human_monitor_command {
5274     my ($vmid, $cmdline) = @_;
5275
5276     my $res;
5277
5278     my $cmd = {
5279         execute => 'human-monitor-command',
5280         arguments => { 'command-line' => $cmdline},
5281     };
5282
5283     return vm_qmp_command($vmid, $cmd);
5284 }
5285
5286 sub vm_commandline {
5287     my ($storecfg, $vmid) = @_;
5288
5289     my $conf = PVE::QemuConfig->load_config($vmid);
5290
5291     my $defaults = load_defaults();
5292
5293     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults);
5294
5295     return PVE::Tools::cmd2string($cmd);
5296 }
5297
5298 sub vm_reset {
5299     my ($vmid, $skiplock) = @_;
5300
5301     PVE::QemuConfig->lock_config($vmid, sub {
5302
5303         my $conf = PVE::QemuConfig->load_config($vmid);
5304
5305         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5306
5307         vm_mon_cmd($vmid, "system_reset");
5308     });
5309 }
5310
5311 sub get_vm_volumes {
5312     my ($conf) = @_;
5313
5314     my $vollist = [];
5315     foreach_volid($conf, sub {
5316         my ($volid, $attr) = @_;
5317
5318         return if $volid =~ m|^/|;
5319
5320         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5321         return if !$sid;
5322
5323         push @$vollist, $volid;
5324     });
5325
5326     return $vollist;
5327 }
5328
5329 sub vm_stop_cleanup {
5330     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
5331
5332     eval {
5333
5334         if (!$keepActive) {
5335             my $vollist = get_vm_volumes($conf);
5336             PVE::Storage::deactivate_volumes($storecfg, $vollist);
5337         }
5338
5339         foreach my $ext (qw(mon qmp pid vnc qga)) {
5340             unlink "/var/run/qemu-server/${vmid}.$ext";
5341         }
5342
5343         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
5344     };
5345     warn $@ if $@; # avoid errors - just warn
5346 }
5347
5348 # Note: use $nockeck to skip tests if VM configuration file exists.
5349 # We need that when migration VMs to other nodes (files already moved)
5350 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
5351 sub vm_stop {
5352     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
5353
5354     $force = 1 if !defined($force) && !$shutdown;
5355
5356     if ($migratedfrom){
5357         my $pid = check_running($vmid, $nocheck, $migratedfrom);
5358         kill 15, $pid if $pid;
5359         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
5360         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
5361         return;
5362     }
5363
5364     PVE::QemuConfig->lock_config($vmid, sub {
5365
5366         my $pid = check_running($vmid, $nocheck);
5367         return if !$pid;
5368
5369         my $conf;
5370         if (!$nocheck) {
5371             $conf = PVE::QemuConfig->load_config($vmid);
5372             PVE::QemuConfig->check_lock($conf) if !$skiplock;
5373             if (!defined($timeout) && $shutdown && $conf->{startup}) {
5374                 my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
5375                 $timeout = $opts->{down} if $opts->{down};
5376             }
5377         }
5378
5379         $timeout = 60 if !defined($timeout);
5380
5381         eval {
5382             if ($shutdown) {
5383                 if (defined($conf) && parse_guest_agent($conf)->{enabled}) {
5384                     vm_qmp_command($vmid, { execute => "guest-shutdown" }, $nocheck);
5385                 } else {
5386                     vm_qmp_command($vmid, { execute => "system_powerdown" }, $nocheck);
5387                 }
5388             } else {
5389                 vm_qmp_command($vmid, { execute => "quit" }, $nocheck);
5390             }
5391         };
5392         my $err = $@;
5393
5394         if (!$err) {
5395             my $count = 0;
5396             while (($count < $timeout) && check_running($vmid, $nocheck)) {
5397                 $count++;
5398                 sleep 1;
5399             }
5400
5401             if ($count >= $timeout) {
5402                 if ($force) {
5403                     warn "VM still running - terminating now with SIGTERM\n";
5404                     kill 15, $pid;
5405                 } else {
5406                     die "VM quit/powerdown failed - got timeout\n";
5407                 }
5408             } else {
5409                 vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5410                 return;
5411             }
5412         } else {
5413             if ($force) {
5414                 warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
5415                 kill 15, $pid;
5416             } else {
5417                 die "VM quit/powerdown failed\n";
5418             }
5419         }
5420
5421         # wait again
5422         $timeout = 10;
5423
5424         my $count = 0;
5425         while (($count < $timeout) && check_running($vmid, $nocheck)) {
5426             $count++;
5427             sleep 1;
5428         }
5429
5430         if ($count >= $timeout) {
5431             warn "VM still running - terminating now with SIGKILL\n";
5432             kill 9, $pid;
5433             sleep 1;
5434         }
5435
5436         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5437    });
5438 }
5439
5440 sub vm_suspend {
5441     my ($vmid, $skiplock) = @_;
5442
5443     PVE::QemuConfig->lock_config($vmid, sub {
5444
5445         my $conf = PVE::QemuConfig->load_config($vmid);
5446
5447         PVE::QemuConfig->check_lock($conf)
5448             if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
5449
5450         vm_mon_cmd($vmid, "stop");
5451     });
5452 }
5453
5454 sub vm_resume {
5455     my ($vmid, $skiplock, $nocheck) = @_;
5456
5457     PVE::QemuConfig->lock_config($vmid, sub {
5458
5459         my $res = vm_mon_cmd($vmid, 'query-status');
5460         my $resume_cmd = 'cont';
5461
5462         if ($res->{status} && $res->{status} eq 'suspended') {
5463             $resume_cmd = 'system_wakeup';
5464         }
5465
5466         if (!$nocheck) {
5467
5468             my $conf = PVE::QemuConfig->load_config($vmid);
5469
5470             PVE::QemuConfig->check_lock($conf)
5471                 if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
5472
5473             vm_mon_cmd($vmid, $resume_cmd);
5474
5475         } else {
5476             vm_mon_cmd_nocheck($vmid, $resume_cmd);
5477         }
5478     });
5479 }
5480
5481 sub vm_sendkey {
5482     my ($vmid, $skiplock, $key) = @_;
5483
5484     PVE::QemuConfig->lock_config($vmid, sub {
5485
5486         my $conf = PVE::QemuConfig->load_config($vmid);
5487
5488         # there is no qmp command, so we use the human monitor command
5489         vm_human_monitor_command($vmid, "sendkey $key");
5490     });
5491 }
5492
5493 sub vm_destroy {
5494     my ($storecfg, $vmid, $skiplock) = @_;
5495
5496     PVE::QemuConfig->lock_config($vmid, sub {
5497
5498         my $conf = PVE::QemuConfig->load_config($vmid);
5499
5500         if (!check_running($vmid)) {
5501             destroy_vm($storecfg, $vmid, undef, $skiplock);
5502         } else {
5503             die "VM $vmid is running - destroy failed\n";
5504         }
5505     });
5506 }
5507
5508 # pci helpers
5509
5510 sub file_write {
5511     my ($filename, $buf) = @_;
5512
5513     my $fh = IO::File->new($filename, "w");
5514     return undef if !$fh;
5515
5516     my $res = print $fh $buf;
5517
5518     $fh->close();
5519
5520     return $res;
5521 }
5522
5523 sub pci_device_info {
5524     my ($name) = @_;
5525
5526     my $res;
5527
5528     return undef if $name !~ m/^([a-f0-9]{4}):([a-f0-9]{2}):([a-f0-9]{2})\.([a-f0-9])$/;
5529     my ($domain, $bus, $slot, $func) = ($1, $2, $3, $4);
5530
5531     my $irq = file_read_firstline("$pcisysfs/devices/$name/irq");
5532     return undef if !defined($irq) || $irq !~ m/^\d+$/;
5533
5534     my $vendor = file_read_firstline("$pcisysfs/devices/$name/vendor");
5535     return undef if !defined($vendor) || $vendor !~ s/^0x//;
5536
5537     my $product = file_read_firstline("$pcisysfs/devices/$name/device");
5538     return undef if !defined($product) || $product !~ s/^0x//;
5539
5540     $res = {
5541         name => $name,
5542         vendor => $vendor,
5543         product => $product,
5544         domain => $domain,
5545         bus => $bus,
5546         slot => $slot,
5547         func => $func,
5548         irq => $irq,
5549         has_fl_reset => -f "$pcisysfs/devices/$name/reset" || 0,
5550     };
5551
5552     return $res;
5553 }
5554
5555 sub pci_dev_reset {
5556     my ($dev) = @_;
5557
5558     my $name = $dev->{name};
5559
5560     my $fn = "$pcisysfs/devices/$name/reset";
5561
5562     return file_write($fn, "1");
5563 }
5564
5565 sub pci_dev_bind_to_vfio {
5566     my ($dev) = @_;
5567
5568     my $name = $dev->{name};
5569
5570     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
5571
5572     if (!-d $vfio_basedir) {
5573         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
5574     }
5575     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
5576
5577     my $testdir = "$vfio_basedir/$name";
5578     return 1 if -d $testdir;
5579
5580     my $data = "$dev->{vendor} $dev->{product}";
5581     return undef if !file_write("$vfio_basedir/new_id", $data);
5582
5583     my $fn = "$pcisysfs/devices/$name/driver/unbind";
5584     if (!file_write($fn, $name)) {
5585         return undef if -f $fn;
5586     }
5587
5588     $fn = "$vfio_basedir/bind";
5589     if (! -d $testdir) {
5590         return undef if !file_write($fn, $name);
5591     }
5592
5593     return -d $testdir;
5594 }
5595
5596 sub pci_dev_group_bind_to_vfio {
5597     my ($pciid) = @_;
5598
5599     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
5600
5601     if (!-d $vfio_basedir) {
5602         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
5603     }
5604     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
5605
5606     # get IOMMU group devices
5607     opendir(my $D, "$pcisysfs/devices/0000:$pciid/iommu_group/devices/") || die "Cannot open iommu_group: $!\n";
5608       my @devs = grep /^0000:/, readdir($D);
5609     closedir($D);
5610
5611     foreach my $pciid (@devs) {
5612         $pciid =~ m/^([:\.\da-f]+)$/ or die "PCI ID $pciid not valid!\n";
5613
5614         # pci bridges, switches or root ports are not supported
5615         # they have a pci_bus subdirectory so skip them
5616         next if (-e "$pcisysfs/devices/$pciid/pci_bus");
5617
5618         my $info = pci_device_info($1);
5619         pci_dev_bind_to_vfio($info) || die "Cannot bind $pciid to vfio\n";
5620     }
5621
5622     return 1;
5623 }
5624
5625 # vzdump restore implementaion
5626
5627 sub tar_archive_read_firstfile {
5628     my $archive = shift;
5629
5630     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
5631
5632     # try to detect archive type first
5633     my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
5634         die "unable to open file '$archive'\n";
5635     my $firstfile = <$fh>;
5636     kill 15, $pid;
5637     close $fh;
5638
5639     die "ERROR: archive contaions no data\n" if !$firstfile;
5640     chomp $firstfile;
5641
5642     return $firstfile;
5643 }
5644
5645 sub tar_restore_cleanup {
5646     my ($storecfg, $statfile) = @_;
5647
5648     print STDERR "starting cleanup\n";
5649
5650     if (my $fd = IO::File->new($statfile, "r")) {
5651         while (defined(my $line = <$fd>)) {
5652             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5653                 my $volid = $2;
5654                 eval {
5655                     if ($volid =~ m|^/|) {
5656                         unlink $volid || die 'unlink failed\n';
5657                     } else {
5658                         PVE::Storage::vdisk_free($storecfg, $volid);
5659                     }
5660                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
5661                 };
5662                 print STDERR "unable to cleanup '$volid' - $@" if $@;
5663             } else {
5664                 print STDERR "unable to parse line in statfile - $line";
5665             }
5666         }
5667         $fd->close();
5668     }
5669 }
5670
5671 sub restore_archive {
5672     my ($archive, $vmid, $user, $opts) = @_;
5673
5674     my $format = $opts->{format};
5675     my $comp;
5676
5677     if ($archive =~ m/\.tgz$/ || $archive =~ m/\.tar\.gz$/) {
5678         $format = 'tar' if !$format;
5679         $comp = 'gzip';
5680     } elsif ($archive =~ m/\.tar$/) {
5681         $format = 'tar' if !$format;
5682     } elsif ($archive =~ m/.tar.lzo$/) {
5683         $format = 'tar' if !$format;
5684         $comp = 'lzop';
5685     } elsif ($archive =~ m/\.vma$/) {
5686         $format = 'vma' if !$format;
5687     } elsif ($archive =~ m/\.vma\.gz$/) {
5688         $format = 'vma' if !$format;
5689         $comp = 'gzip';
5690     } elsif ($archive =~ m/\.vma\.lzo$/) {
5691         $format = 'vma' if !$format;
5692         $comp = 'lzop';
5693     } else {
5694         $format = 'vma' if !$format; # default
5695     }
5696
5697     # try to detect archive format
5698     if ($format eq 'tar') {
5699         return restore_tar_archive($archive, $vmid, $user, $opts);
5700     } else {
5701         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
5702     }
5703 }
5704
5705 sub restore_update_config_line {
5706     my ($outfd, $cookie, $vmid, $map, $line, $unique) = @_;
5707
5708     return if $line =~ m/^\#qmdump\#/;
5709     return if $line =~ m/^\#vzdump\#/;
5710     return if $line =~ m/^lock:/;
5711     return if $line =~ m/^unused\d+:/;
5712     return if $line =~ m/^parent:/;
5713     return if $line =~ m/^template:/; # restored VM is never a template
5714
5715     my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
5716     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
5717         # try to convert old 1.X settings
5718         my ($id, $ind, $ethcfg) = ($1, $2, $3);
5719         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
5720             my ($model, $macaddr) = split(/\=/, $devconfig);
5721             $macaddr = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if !$macaddr || $unique;
5722             my $net = {
5723                 model => $model,
5724                 bridge => "vmbr$ind",
5725                 macaddr => $macaddr,
5726             };
5727             my $netstr = print_net($net);
5728
5729             print $outfd "net$cookie->{netcount}: $netstr\n";
5730             $cookie->{netcount}++;
5731         }
5732     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
5733         my ($id, $netstr) = ($1, $2);
5734         my $net = parse_net($netstr);
5735         $net->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if $net->{macaddr};
5736         $netstr = print_net($net);
5737         print $outfd "$id: $netstr\n";
5738     } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk)\d+):\s*(\S+)\s*$/) {
5739         my $virtdev = $1;
5740         my $value = $3;
5741         my $di = parse_drive($virtdev, $value);
5742         if (defined($di->{backup}) && !$di->{backup}) {
5743             print $outfd "#$line";
5744         } elsif ($map->{$virtdev}) {
5745             delete $di->{format}; # format can change on restore
5746             $di->{file} = $map->{$virtdev};
5747             $value = print_drive($vmid, $di);
5748             print $outfd "$virtdev: $value\n";
5749         } else {
5750             print $outfd $line;
5751         }
5752     } elsif (($line =~ m/^vmgenid: (.*)/)) {
5753         my $vmgenid = $1;
5754         if ($vmgenid ne '0') {
5755             # always generate a new vmgenid if there was a valid one setup
5756             $vmgenid = generate_uuid();
5757         }
5758         print $outfd "vmgenid: $vmgenid\n";
5759     } elsif (($line =~ m/^(smbios1: )(.*)/) && $unique) {
5760         my ($uuid, $uuid_str);
5761         UUID::generate($uuid);
5762         UUID::unparse($uuid, $uuid_str);
5763         my $smbios1 = parse_smbios1($2);
5764         $smbios1->{uuid} = $uuid_str;
5765         print $outfd $1.print_smbios1($smbios1)."\n";
5766     } else {
5767         print $outfd $line;
5768     }
5769 }
5770
5771 sub scan_volids {
5772     my ($cfg, $vmid) = @_;
5773
5774     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid);
5775
5776     my $volid_hash = {};
5777     foreach my $storeid (keys %$info) {
5778         foreach my $item (@{$info->{$storeid}}) {
5779             next if !($item->{volid} && $item->{size});
5780             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
5781             $volid_hash->{$item->{volid}} = $item;
5782         }
5783     }
5784
5785     return $volid_hash;
5786 }
5787
5788 sub is_volume_in_use {
5789     my ($storecfg, $conf, $skip_drive, $volid) = @_;
5790
5791     my $path = PVE::Storage::path($storecfg, $volid);
5792
5793     my $scan_config = sub {
5794         my ($cref, $snapname) = @_;
5795
5796         foreach my $key (keys %$cref) {
5797             my $value = $cref->{$key};
5798             if (is_valid_drivename($key)) {
5799                 next if $skip_drive && $key eq $skip_drive;
5800                 my $drive = parse_drive($key, $value);
5801                 next if !$drive || !$drive->{file} || drive_is_cdrom($drive);
5802                 return 1 if $volid eq $drive->{file};
5803                 if ($drive->{file} =~ m!^/!) {
5804                     return 1 if $drive->{file} eq $path;
5805                 } else {
5806                     my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file}, 1);
5807                     next if !$storeid;
5808                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid, 1);
5809                     next if !$scfg;
5810                     return 1 if $path eq PVE::Storage::path($storecfg, $drive->{file}, $snapname);
5811                 }
5812             }
5813         }
5814
5815         return 0;
5816     };
5817
5818     return 1 if &$scan_config($conf);
5819
5820     undef $skip_drive;
5821
5822     foreach my $snapname (keys %{$conf->{snapshots}}) {
5823         return 1 if &$scan_config($conf->{snapshots}->{$snapname}, $snapname);
5824     }
5825
5826     return 0;
5827 }
5828
5829 sub update_disksize {
5830     my ($vmid, $conf, $volid_hash) = @_;
5831
5832     my $changes;
5833     my $prefix = "VM $vmid:";
5834
5835     # used and unused disks
5836     my $referenced = {};
5837
5838     # Note: it is allowed to define multiple storages with same path (alias), so
5839     # we need to check both 'volid' and real 'path' (two different volid can point
5840     # to the same path).
5841
5842     my $referencedpath = {};
5843
5844     # update size info
5845     foreach my $opt (keys %$conf) {
5846         if (is_valid_drivename($opt)) {
5847             my $drive = parse_drive($opt, $conf->{$opt});
5848             my $volid = $drive->{file};
5849             next if !$volid;
5850
5851             $referenced->{$volid} = 1;
5852             if ($volid_hash->{$volid} &&
5853                 (my $path = $volid_hash->{$volid}->{path})) {
5854                 $referencedpath->{$path} = 1;
5855             }
5856
5857             next if drive_is_cdrom($drive);
5858             next if !$volid_hash->{$volid};
5859
5860             $drive->{size} = $volid_hash->{$volid}->{size};
5861             my $new = print_drive($vmid, $drive);
5862             if ($new ne $conf->{$opt}) {
5863                 $changes = 1;
5864                 $conf->{$opt} = $new;
5865                 print "$prefix update disk '$opt' information.\n";
5866             }
5867         }
5868     }
5869
5870     # remove 'unusedX' entry if volume is used
5871     foreach my $opt (keys %$conf) {
5872         next if $opt !~ m/^unused\d+$/;
5873         my $volid = $conf->{$opt};
5874         my $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
5875         if ($referenced->{$volid} || ($path && $referencedpath->{$path})) {
5876             print "$prefix remove entry '$opt', its volume '$volid' is in use.\n";
5877             $changes = 1;
5878             delete $conf->{$opt};
5879         }
5880
5881         $referenced->{$volid} = 1;
5882         $referencedpath->{$path} = 1 if $path;
5883     }
5884
5885     foreach my $volid (sort keys %$volid_hash) {
5886         next if $volid =~ m/vm-$vmid-state-/;
5887         next if $referenced->{$volid};
5888         my $path = $volid_hash->{$volid}->{path};
5889         next if !$path; # just to be sure
5890         next if $referencedpath->{$path};
5891         $changes = 1;
5892         my $key = PVE::QemuConfig->add_unused_volume($conf, $volid);
5893         print "$prefix add unreferenced volume '$volid' as '$key' to config.\n";
5894         $referencedpath->{$path} = 1; # avoid to add more than once (aliases)
5895     }
5896
5897     return $changes;
5898 }
5899
5900 sub rescan {
5901     my ($vmid, $nolock, $dryrun) = @_;
5902
5903     my $cfg = PVE::Storage::config();
5904
5905     # FIXME: Remove once our RBD plugin can handle CT and VM on a single storage
5906     # see: https://pve.proxmox.com/pipermail/pve-devel/2018-July/032900.html
5907     foreach my $stor (keys %{$cfg->{ids}}) {
5908         delete($cfg->{ids}->{$stor}) if ! $cfg->{ids}->{$stor}->{content}->{images};
5909     }
5910
5911     print "rescan volumes...\n";
5912     my $volid_hash = scan_volids($cfg, $vmid);
5913
5914     my $updatefn =  sub {
5915         my ($vmid) = @_;
5916
5917         my $conf = PVE::QemuConfig->load_config($vmid);
5918
5919         PVE::QemuConfig->check_lock($conf);
5920
5921         my $vm_volids = {};
5922         foreach my $volid (keys %$volid_hash) {
5923             my $info = $volid_hash->{$volid};
5924             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
5925         }
5926
5927         my $changes = update_disksize($vmid, $conf, $vm_volids);
5928
5929         PVE::QemuConfig->write_config($vmid, $conf) if $changes && !$dryrun;
5930     };
5931
5932     if (defined($vmid)) {
5933         if ($nolock) {
5934             &$updatefn($vmid);
5935         } else {
5936             PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
5937         }
5938     } else {
5939         my $vmlist = config_list();
5940         foreach my $vmid (keys %$vmlist) {
5941             if ($nolock) {
5942                 &$updatefn($vmid);
5943             } else {
5944                 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
5945             }
5946         }
5947     }
5948 }
5949
5950 sub restore_vma_archive {
5951     my ($archive, $vmid, $user, $opts, $comp) = @_;
5952
5953     my $readfrom = $archive;
5954
5955     my $cfg = PVE::Storage::config();
5956     my $commands = [];
5957     my $bwlimit = $opts->{bwlimit};
5958
5959     my $dbg_cmdstring = '';
5960     my $add_pipe = sub {
5961         my ($cmd) = @_;
5962         push @$commands, $cmd;
5963         $dbg_cmdstring .= ' | ' if length($dbg_cmdstring);
5964         $dbg_cmdstring .= PVE::Tools::cmd2string($cmd);
5965         $readfrom = '-';
5966     };
5967
5968     my $input = undef;
5969     if ($archive eq '-') {
5970         $input = '<&STDIN';
5971     } else {
5972         # If we use a backup from a PVE defined storage we also consider that
5973         # storage's rate limit:
5974         my (undef, $volid) = PVE::Storage::path_to_volume_id($cfg, $archive);
5975         if (defined($volid)) {
5976             my ($sid, undef) = PVE::Storage::parse_volume_id($volid);
5977             my $readlimit = PVE::Storage::get_bandwidth_limit('restore', [$sid], $bwlimit);
5978             if ($readlimit) {
5979                 print STDERR "applying read rate limit: $readlimit\n";
5980                 my $cstream = ['cstream', '-t', $readlimit*1024, '--', $readfrom];
5981                 $add_pipe->($cstream);
5982             }
5983         }
5984     }
5985
5986     if ($comp) {
5987         my $cmd;
5988         if ($comp eq 'gzip') {
5989             $cmd = ['zcat', $readfrom];
5990         } elsif ($comp eq 'lzop') {
5991             $cmd = ['lzop', '-d', '-c', $readfrom];
5992         } else {
5993             die "unknown compression method '$comp'\n";
5994         }
5995         $add_pipe->($cmd);
5996     }
5997
5998     my $tmpdir = "/var/tmp/vzdumptmp$$";
5999     rmtree $tmpdir;
6000
6001     # disable interrupts (always do cleanups)
6002     local $SIG{INT} =
6003         local $SIG{TERM} =
6004         local $SIG{QUIT} =
6005         local $SIG{HUP} = sub { warn "got interrupt - ignored\n"; };
6006
6007     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
6008     POSIX::mkfifo($mapfifo, 0600);
6009     my $fifofh;
6010
6011     my $openfifo = sub {
6012         open($fifofh, '>', $mapfifo) || die $!;
6013     };
6014
6015     $add_pipe->(['vma', 'extract', '-v', '-r', $mapfifo, $readfrom, $tmpdir]);
6016
6017     my $oldtimeout;
6018     my $timeout = 5;
6019
6020     my $devinfo = {};
6021
6022     my $rpcenv = PVE::RPCEnvironment::get();
6023
6024     my $conffile = PVE::QemuConfig->config_file($vmid);
6025     my $tmpfn = "$conffile.$$.tmp";
6026
6027     # Note: $oldconf is undef if VM does not exists
6028     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
6029     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
6030
6031     my %storage_limits;
6032
6033     my $print_devmap = sub {
6034         my $virtdev_hash = {};
6035
6036         my $cfgfn = "$tmpdir/qemu-server.conf";
6037
6038         # we can read the config - that is already extracted
6039         my $fh = IO::File->new($cfgfn, "r") ||
6040             "unable to read qemu-server.conf - $!\n";
6041
6042         my $fwcfgfn = "$tmpdir/qemu-server.fw";
6043         if (-f $fwcfgfn) {
6044             my $pve_firewall_dir = '/etc/pve/firewall';
6045             mkdir $pve_firewall_dir; # make sure the dir exists
6046             PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
6047         }
6048
6049         while (defined(my $line = <$fh>)) {
6050             if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
6051                 my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
6052                 die "archive does not contain data for drive '$virtdev'\n"
6053                     if !$devinfo->{$devname};
6054                 if (defined($opts->{storage})) {
6055                     $storeid = $opts->{storage} || 'local';
6056                 } elsif (!$storeid) {
6057                     $storeid = 'local';
6058                 }
6059                 $format = 'raw' if !$format;
6060                 $devinfo->{$devname}->{devname} = $devname;
6061                 $devinfo->{$devname}->{virtdev} = $virtdev;
6062                 $devinfo->{$devname}->{format} = $format;
6063                 $devinfo->{$devname}->{storeid} = $storeid;
6064
6065                 # check permission on storage
6066                 my $pool = $opts->{pool}; # todo: do we need that?
6067                 if ($user ne 'root@pam') {
6068                     $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
6069                 }
6070
6071                 $storage_limits{$storeid} = $bwlimit;
6072
6073                 $virtdev_hash->{$virtdev} = $devinfo->{$devname};
6074             }
6075         }
6076
6077         foreach my $key (keys %storage_limits) {
6078             my $limit = PVE::Storage::get_bandwidth_limit('restore', [$key], $bwlimit);
6079             next if !$limit;
6080             print STDERR "rate limit for storage $key: $limit KiB/s\n";
6081             $storage_limits{$key} = $limit * 1024;
6082         }
6083
6084         foreach my $devname (keys %$devinfo) {
6085             die "found no device mapping information for device '$devname'\n"
6086                 if !$devinfo->{$devname}->{virtdev};
6087         }
6088
6089         # create empty/temp config
6090         if ($oldconf) {
6091             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
6092             foreach_drive($oldconf, sub {
6093                 my ($ds, $drive) = @_;
6094
6095                 return if drive_is_cdrom($drive);
6096
6097                 my $volid = $drive->{file};
6098
6099                 return if !$volid || $volid =~ m|^/|;
6100
6101                 my ($path, $owner) = PVE::Storage::path($cfg, $volid);
6102                 return if !$path || !$owner || ($owner != $vmid);
6103
6104                 # Note: only delete disk we want to restore
6105                 # other volumes will become unused
6106                 if ($virtdev_hash->{$ds}) {
6107                     eval { PVE::Storage::vdisk_free($cfg, $volid); };
6108                     if (my $err = $@) {
6109                         warn $err;
6110                     }
6111                 }
6112             });
6113
6114             # delete vmstate files
6115             # since after the restore we have no snapshots anymore
6116             foreach my $snapname (keys %{$oldconf->{snapshots}}) {
6117                 my $snap = $oldconf->{snapshots}->{$snapname};
6118                 if ($snap->{vmstate}) {
6119                     eval { PVE::Storage::vdisk_free($cfg, $snap->{vmstate}); };
6120                     if (my $err = $@) {
6121                         warn $err;
6122                     }
6123                 }
6124             }
6125         }
6126
6127         my $map = {};
6128         foreach my $virtdev (sort keys %$virtdev_hash) {
6129             my $d = $virtdev_hash->{$virtdev};
6130             my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
6131             my $storeid = $d->{storeid};
6132             my $scfg = PVE::Storage::storage_config($cfg, $storeid);
6133
6134             my $map_opts = '';
6135             if (my $limit = $storage_limits{$storeid}) {
6136                 $map_opts .= "throttling.bps=$limit:throttling.group=$storeid:";
6137             }
6138
6139             # test if requested format is supported
6140             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($cfg, $storeid);
6141             my $supported = grep { $_ eq $d->{format} } @$validFormats;
6142             $d->{format} = $defFormat if !$supported;
6143
6144             my $volid = PVE::Storage::vdisk_alloc($cfg, $storeid, $vmid,
6145                                                   $d->{format}, undef, $alloc_size);
6146             print STDERR "new volume ID is '$volid'\n";
6147             $d->{volid} = $volid;
6148             my $path = PVE::Storage::path($cfg, $volid);
6149
6150             PVE::Storage::activate_volumes($cfg,[$volid]);
6151
6152             my $write_zeros = 1;
6153             if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
6154                 $write_zeros = 0;
6155             }
6156
6157             print $fifofh "${map_opts}format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
6158
6159             print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
6160             $map->{$virtdev} = $volid;
6161         }
6162
6163         $fh->seek(0, 0) || die "seek failed - $!\n";
6164
6165         my $outfd = new IO::File ($tmpfn, "w") ||
6166             die "unable to write config for VM $vmid\n";
6167
6168         my $cookie = { netcount => 0 };
6169         while (defined(my $line = <$fh>)) {
6170             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
6171         }
6172
6173         $fh->close();
6174         $outfd->close();
6175     };
6176
6177     eval {
6178         # enable interrupts
6179         local $SIG{INT} =
6180             local $SIG{TERM} =
6181             local $SIG{QUIT} =
6182             local $SIG{HUP} =
6183             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6184         local $SIG{ALRM} = sub { die "got timeout\n"; };
6185
6186         $oldtimeout = alarm($timeout);
6187
6188         my $parser = sub {
6189             my $line = shift;
6190
6191             print "$line\n";
6192
6193             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
6194                 my ($dev_id, $size, $devname) = ($1, $2, $3);
6195                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
6196             } elsif ($line =~ m/^CTIME: /) {
6197                 # we correctly received the vma config, so we can disable
6198                 # the timeout now for disk allocation (set to 10 minutes, so
6199                 # that we always timeout if something goes wrong)
6200                 alarm(600);
6201                 &$print_devmap();
6202                 print $fifofh "done\n";
6203                 my $tmp = $oldtimeout || 0;
6204                 $oldtimeout = undef;
6205                 alarm($tmp);
6206                 close($fifofh);
6207             }
6208         };
6209
6210         print "restore vma archive: $dbg_cmdstring\n";
6211         run_command($commands, input => $input, outfunc => $parser, afterfork => $openfifo);
6212     };
6213     my $err = $@;
6214
6215     alarm($oldtimeout) if $oldtimeout;
6216
6217     my $vollist = [];
6218     foreach my $devname (keys %$devinfo) {
6219         my $volid = $devinfo->{$devname}->{volid};
6220         push @$vollist, $volid if $volid;
6221     }
6222
6223     PVE::Storage::deactivate_volumes($cfg, $vollist);
6224
6225     unlink $mapfifo;
6226
6227     if ($err) {
6228         rmtree $tmpdir;
6229         unlink $tmpfn;
6230
6231         foreach my $devname (keys %$devinfo) {
6232             my $volid = $devinfo->{$devname}->{volid};
6233             next if !$volid;
6234             eval {
6235                 if ($volid =~ m|^/|) {
6236                     unlink $volid || die 'unlink failed\n';
6237                 } else {
6238                     PVE::Storage::vdisk_free($cfg, $volid);
6239                 }
6240                 print STDERR "temporary volume '$volid' sucessfuly removed\n";
6241             };
6242             print STDERR "unable to cleanup '$volid' - $@" if $@;
6243         }
6244         die $err;
6245     }
6246
6247     rmtree $tmpdir;
6248
6249     rename($tmpfn, $conffile) ||
6250         die "unable to commit configuration file '$conffile'\n";
6251
6252     PVE::Cluster::cfs_update(); # make sure we read new file
6253
6254     eval { rescan($vmid, 1); };
6255     warn $@ if $@;
6256 }
6257
6258 sub restore_tar_archive {
6259     my ($archive, $vmid, $user, $opts) = @_;
6260
6261     if ($archive ne '-') {
6262         my $firstfile = tar_archive_read_firstfile($archive);
6263         die "ERROR: file '$archive' dos not lock like a QemuServer vzdump backup\n"
6264             if $firstfile ne 'qemu-server.conf';
6265     }
6266
6267     my $storecfg = PVE::Storage::config();
6268
6269     # destroy existing data - keep empty config
6270     my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
6271     destroy_vm($storecfg, $vmid, 1) if -f $vmcfgfn;
6272
6273     my $tocmd = "/usr/lib/qemu-server/qmextract";
6274
6275     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
6276     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
6277     $tocmd .= ' --prealloc' if $opts->{prealloc};
6278     $tocmd .= ' --info' if $opts->{info};
6279
6280     # tar option "xf" does not autodetect compression when read from STDIN,
6281     # so we pipe to zcat
6282     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
6283         PVE::Tools::shellquote("--to-command=$tocmd");
6284
6285     my $tmpdir = "/var/tmp/vzdumptmp$$";
6286     mkpath $tmpdir;
6287
6288     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
6289     local $ENV{VZDUMP_VMID} = $vmid;
6290     local $ENV{VZDUMP_USER} = $user;
6291
6292     my $conffile = PVE::QemuConfig->config_file($vmid);
6293     my $tmpfn = "$conffile.$$.tmp";
6294
6295     # disable interrupts (always do cleanups)
6296     local $SIG{INT} =
6297         local $SIG{TERM} =
6298         local $SIG{QUIT} =
6299         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
6300
6301     eval {
6302         # enable interrupts
6303         local $SIG{INT} =
6304             local $SIG{TERM} =
6305             local $SIG{QUIT} =
6306             local $SIG{HUP} =
6307             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6308
6309         if ($archive eq '-') {
6310             print "extracting archive from STDIN\n";
6311             run_command($cmd, input => "<&STDIN");
6312         } else {
6313             print "extracting archive '$archive'\n";
6314             run_command($cmd);
6315         }
6316
6317         return if $opts->{info};
6318
6319         # read new mapping
6320         my $map = {};
6321         my $statfile = "$tmpdir/qmrestore.stat";
6322         if (my $fd = IO::File->new($statfile, "r")) {
6323             while (defined (my $line = <$fd>)) {
6324                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
6325                     $map->{$1} = $2 if $1;
6326                 } else {
6327                     print STDERR "unable to parse line in statfile - $line\n";
6328                 }
6329             }
6330             $fd->close();
6331         }
6332
6333         my $confsrc = "$tmpdir/qemu-server.conf";
6334
6335         my $srcfd = new IO::File($confsrc, "r") ||
6336             die "unable to open file '$confsrc'\n";
6337
6338         my $outfd = new IO::File ($tmpfn, "w") ||
6339             die "unable to write config for VM $vmid\n";
6340
6341         my $cookie = { netcount => 0 };
6342         while (defined (my $line = <$srcfd>)) {
6343             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
6344         }
6345
6346         $srcfd->close();
6347         $outfd->close();
6348     };
6349     my $err = $@;
6350
6351     if ($err) {
6352
6353         unlink $tmpfn;
6354
6355         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
6356
6357         die $err;
6358     }
6359
6360     rmtree $tmpdir;
6361
6362     rename $tmpfn, $conffile ||
6363         die "unable to commit configuration file '$conffile'\n";
6364
6365     PVE::Cluster::cfs_update(); # make sure we read new file
6366
6367     eval { rescan($vmid, 1); };
6368     warn $@ if $@;
6369 };
6370
6371 sub foreach_storage_used_by_vm {
6372     my ($conf, $func) = @_;
6373
6374     my $sidhash = {};
6375
6376     foreach_drive($conf, sub {
6377         my ($ds, $drive) = @_;
6378         return if drive_is_cdrom($drive);
6379
6380         my $volid = $drive->{file};
6381
6382         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
6383         $sidhash->{$sid} = $sid if $sid;
6384     });
6385
6386     foreach my $sid (sort keys %$sidhash) {
6387         &$func($sid);
6388     }
6389 }
6390
6391 sub do_snapshots_with_qemu {
6392     my ($storecfg, $volid) = @_;
6393
6394     my $storage_name = PVE::Storage::parse_volume_id($volid);
6395
6396     if ($qemu_snap_storage->{$storecfg->{ids}->{$storage_name}->{type}}
6397         && !$storecfg->{ids}->{$storage_name}->{krbd}){
6398         return 1;
6399     }
6400
6401     if ($volid =~ m/\.(qcow2|qed)$/){
6402         return 1;
6403     }
6404
6405     return undef;
6406 }
6407
6408 sub qga_check_running {
6409     my ($vmid, $nowarn) = @_;
6410
6411     eval { vm_mon_cmd($vmid, "guest-ping", timeout => 3); };
6412     if ($@) {
6413         warn "Qemu Guest Agent is not running - $@" if !$nowarn;
6414         return 0;
6415     }
6416     return 1;
6417 }
6418
6419 sub template_create {
6420     my ($vmid, $conf, $disk) = @_;
6421
6422     my $storecfg = PVE::Storage::config();
6423
6424     foreach_drive($conf, sub {
6425         my ($ds, $drive) = @_;
6426
6427         return if drive_is_cdrom($drive);
6428         return if $disk && $ds ne $disk;
6429
6430         my $volid = $drive->{file};
6431         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
6432
6433         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
6434         $drive->{file} = $voliddst;
6435         $conf->{$ds} = print_drive($vmid, $drive);
6436         PVE::QemuConfig->write_config($vmid, $conf);
6437     });
6438 }
6439
6440 sub qemu_img_convert {
6441     my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized) = @_;
6442
6443     my $storecfg = PVE::Storage::config();
6444     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
6445     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
6446
6447     if ($src_storeid && $dst_storeid) {
6448
6449         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
6450
6451         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
6452         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6453
6454         my $src_format = qemu_img_format($src_scfg, $src_volname);
6455         my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
6456
6457         my $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
6458         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6459
6460         my $cmd = [];
6461         push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
6462         push @$cmd, '-l', "snapshot.name=$snapname" if($snapname && $src_format eq "qcow2");
6463         push @$cmd, '-t', 'none' if $dst_scfg->{type} eq 'zfspool';
6464         push @$cmd, '-T', 'none' if $src_scfg->{type} eq 'zfspool';
6465         push @$cmd, '-f', $src_format, '-O', $dst_format, $src_path;
6466         if ($is_zero_initialized) {
6467             push @$cmd, "zeroinit:$dst_path";
6468         } else {
6469             push @$cmd, $dst_path;
6470         }
6471
6472         my $parser = sub {
6473             my $line = shift;
6474             if($line =~ m/\((\S+)\/100\%\)/){
6475                 my $percent = $1;
6476                 my $transferred = int($size * $percent / 100);
6477                 my $remaining = $size - $transferred;
6478
6479                 print "transferred: $transferred bytes remaining: $remaining bytes total: $size bytes progression: $percent %\n";
6480             }
6481
6482         };
6483
6484         eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
6485         my $err = $@;
6486         die "copy failed: $err" if $err;
6487     }
6488 }
6489
6490 sub qemu_img_format {
6491     my ($scfg, $volname) = @_;
6492
6493     if ($scfg->{path} && $volname =~ m/\.($QEMU_FORMAT_RE)$/) {
6494         return $1;
6495     } else {
6496         return "raw";
6497     }
6498 }
6499
6500 sub qemu_drive_mirror {
6501     my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $skipcomplete, $qga) = @_;
6502
6503     $jobs = {} if !$jobs;
6504
6505     my $qemu_target;
6506     my $format;
6507     $jobs->{"drive-$drive"} = {};
6508
6509     if ($dst_volid =~ /^nbd:/) {
6510         $qemu_target = $dst_volid;
6511         $format = "nbd";
6512     } else {
6513         my $storecfg = PVE::Storage::config();
6514         my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
6515
6516         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6517
6518         $format = qemu_img_format($dst_scfg, $dst_volname);
6519
6520         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6521
6522         $qemu_target = $is_zero_initialized ? "zeroinit:$dst_path" : $dst_path;
6523     }
6524
6525     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $qemu_target };
6526     $opts->{format} = $format if $format;
6527
6528     print "drive mirror is starting for drive-$drive\n";
6529
6530     eval { vm_mon_cmd($vmid, "drive-mirror", %$opts); }; #if a job already run for this device,it's throw an error
6531
6532     if (my $err = $@) {
6533         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6534         die "mirroring error: $err";
6535     }
6536
6537     qemu_drive_mirror_monitor ($vmid, $vmiddst, $jobs, $skipcomplete, $qga);
6538 }
6539
6540 sub qemu_drive_mirror_monitor {
6541     my ($vmid, $vmiddst, $jobs, $skipcomplete, $qga) = @_;
6542
6543     eval {
6544         my $err_complete = 0;
6545
6546         while (1) {
6547             die "storage migration timed out\n" if $err_complete > 300;
6548
6549             my $stats = vm_mon_cmd($vmid, "query-block-jobs");
6550
6551             my $running_mirror_jobs = {};
6552             foreach my $stat (@$stats) {
6553                 next if $stat->{type} ne 'mirror';
6554                 $running_mirror_jobs->{$stat->{device}} = $stat;
6555             }
6556
6557             my $readycounter = 0;
6558
6559             foreach my $job (keys %$jobs) {
6560
6561                 if(defined($jobs->{$job}->{complete}) && !defined($running_mirror_jobs->{$job})) {
6562                     print "$job : finished\n";
6563                     delete $jobs->{$job};
6564                     next;
6565                 }
6566
6567                 die "$job: mirroring has been cancelled\n" if !defined($running_mirror_jobs->{$job});
6568
6569                 my $busy = $running_mirror_jobs->{$job}->{busy};
6570                 my $ready = $running_mirror_jobs->{$job}->{ready};
6571                 if (my $total = $running_mirror_jobs->{$job}->{len}) {
6572                     my $transferred = $running_mirror_jobs->{$job}->{offset} || 0;
6573                     my $remaining = $total - $transferred;
6574                     my $percent = sprintf "%.2f", ($transferred * 100 / $total);
6575
6576                     print "$job: transferred: $transferred bytes remaining: $remaining bytes total: $total bytes progression: $percent % busy: $busy ready: $ready \n";
6577                 }
6578
6579                 $readycounter++ if $running_mirror_jobs->{$job}->{ready};
6580             }
6581
6582             last if scalar(keys %$jobs) == 0;
6583
6584             if ($readycounter == scalar(keys %$jobs)) {
6585                 print "all mirroring jobs are ready \n";
6586                 last if $skipcomplete; #do the complete later
6587
6588                 if ($vmiddst && $vmiddst != $vmid) {
6589                     my $agent_running = $qga && qga_check_running($vmid);
6590                     if ($agent_running) {
6591                         print "freeze filesystem\n";
6592                         eval { PVE::QemuServer::vm_mon_cmd($vmid, "guest-fsfreeze-freeze"); };
6593                     } else {
6594                         print "suspend vm\n";
6595                         eval { PVE::QemuServer::vm_suspend($vmid, 1); };
6596                     }
6597
6598                     # if we clone a disk for a new target vm, we don't switch the disk
6599                     PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs);
6600
6601                     if ($agent_running) {
6602                         print "unfreeze filesystem\n";
6603                         eval { PVE::QemuServer::vm_mon_cmd($vmid, "guest-fsfreeze-thaw"); };
6604                     } else {
6605                         print "resume vm\n";
6606                         eval {  PVE::QemuServer::vm_resume($vmid, 1, 1); };
6607                     }
6608
6609                     last;
6610                 } else {
6611
6612                     foreach my $job (keys %$jobs) {
6613                         # try to switch the disk if source and destination are on the same guest
6614                         print "$job: Completing block job...\n";
6615
6616                         eval { vm_mon_cmd($vmid, "block-job-complete", device => $job) };
6617                         if ($@ =~ m/cannot be completed/) {
6618                             print "$job: Block job cannot be completed, try again.\n";
6619                             $err_complete++;
6620                         }else {
6621                             print "$job: Completed successfully.\n";
6622                             $jobs->{$job}->{complete} = 1;
6623                         }
6624                     }
6625                 }
6626             }
6627             sleep 1;
6628         }
6629     };
6630     my $err = $@;
6631
6632     if ($err) {
6633         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6634         die "mirroring error: $err";
6635     }
6636
6637 }
6638
6639 sub qemu_blockjobs_cancel {
6640     my ($vmid, $jobs) = @_;
6641
6642     foreach my $job (keys %$jobs) {
6643         print "$job: Cancelling block job\n";
6644         eval { vm_mon_cmd($vmid, "block-job-cancel", device => $job); };
6645         $jobs->{$job}->{cancel} = 1;
6646     }
6647
6648     while (1) {
6649         my $stats = vm_mon_cmd($vmid, "query-block-jobs");
6650
6651         my $running_jobs = {};
6652         foreach my $stat (@$stats) {
6653             $running_jobs->{$stat->{device}} = $stat;
6654         }
6655
6656         foreach my $job (keys %$jobs) {
6657
6658             if (defined($jobs->{$job}->{cancel}) && !defined($running_jobs->{$job})) {
6659                 print "$job: Done.\n";
6660                 delete $jobs->{$job};
6661             }
6662         }
6663
6664         last if scalar(keys %$jobs) == 0;
6665
6666         sleep 1;
6667     }
6668 }
6669
6670 sub clone_disk {
6671     my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
6672         $newvmid, $storage, $format, $full, $newvollist, $jobs, $skipcomplete, $qga) = @_;
6673
6674     my $newvolid;
6675
6676     if (!$full) {
6677         print "create linked clone of drive $drivename ($drive->{file})\n";
6678         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
6679         push @$newvollist, $newvolid;
6680     } else {
6681
6682         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6683         $storeid = $storage if $storage;
6684
6685         my $dst_format = resolve_dst_disk_format($storecfg, $storeid, $volname, $format);
6686         my ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 3);
6687
6688         print "create full clone of drive $drivename ($drive->{file})\n";
6689         my $name = undef;
6690         if (drive_is_cloudinit($drive)) {
6691             $name = "vm-$newvmid-cloudinit";
6692             # cloudinit only supports raw and qcow2 atm:
6693             if ($dst_format eq 'qcow2') {
6694                 $name .= '.qcow2';
6695             } elsif ($dst_format ne 'raw') {
6696                 die "clone: unhandled format for cloudinit image\n";
6697             }
6698         }
6699         $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $newvmid, $dst_format, $name, ($size/1024));
6700         push @$newvollist, $newvolid;
6701
6702         PVE::Storage::activate_volumes($storecfg, [$newvolid]);
6703
6704         my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
6705         if (!$running || $snapname) {
6706             qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit);
6707         } else {
6708
6709             my $kvmver = get_running_qemu_version ($vmid);
6710             if (!qemu_machine_feature_enabled (undef, $kvmver, 2, 7)) {
6711                 die "drive-mirror with iothread requires qemu version 2.7 or higher\n"
6712                     if $drive->{iothread};
6713             }
6714
6715             qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid, $sparseinit, $jobs, $skipcomplete, $qga);
6716         }
6717     }
6718
6719     my ($size) = PVE::Storage::volume_size_info($storecfg, $newvolid, 3);
6720
6721     my $disk = $drive;
6722     $disk->{format} = undef;
6723     $disk->{file} = $newvolid;
6724     $disk->{size} = $size;
6725
6726     return $disk;
6727 }
6728
6729 # this only works if VM is running
6730 sub get_current_qemu_machine {
6731     my ($vmid) = @_;
6732
6733     my $cmd = { execute => 'query-machines', arguments => {} };
6734     my $res = vm_qmp_command($vmid, $cmd);
6735
6736     my ($current, $default);
6737     foreach my $e (@$res) {
6738         $default = $e->{name} if $e->{'is-default'};
6739         $current = $e->{name} if $e->{'is-current'};
6740     }
6741
6742     # fallback to the default machine if current is not supported by qemu
6743     return $current || $default || 'pc';
6744 }
6745
6746 sub get_running_qemu_version {
6747     my ($vmid) = @_;
6748     my $cmd = { execute => 'query-version', arguments => {} };
6749     my $res = vm_qmp_command($vmid, $cmd);
6750     return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
6751 }
6752
6753 sub qemu_machine_feature_enabled {
6754     my ($machine, $kvmver, $version_major, $version_minor) = @_;
6755
6756     my $current_major;
6757     my $current_minor;
6758
6759     if ($machine && $machine =~ m/^((?:pc(-i440fx|-q35)?|virt)-(\d+)\.(\d+))/) {
6760
6761         $current_major = $3;
6762         $current_minor = $4;
6763
6764     } elsif ($kvmver =~ m/^(\d+)\.(\d+)/) {
6765
6766         $current_major = $1;
6767         $current_minor = $2;
6768     }
6769
6770     return 1 if $current_major > $version_major ||
6771                 ($current_major == $version_major &&
6772                  $current_minor >= $version_minor);
6773 }
6774
6775 sub qemu_machine_pxe {
6776     my ($vmid, $conf, $machine) = @_;
6777
6778     $machine =  PVE::QemuServer::get_current_qemu_machine($vmid) if !$machine;
6779
6780     if ($conf->{machine} && $conf->{machine} =~ m/\.pxe$/) {
6781         $machine .= '.pxe';
6782     }
6783
6784     return $machine;
6785 }
6786
6787 sub qemu_use_old_bios_files {
6788     my ($machine_type) = @_;
6789
6790     return if !$machine_type;
6791
6792     my $use_old_bios_files = undef;
6793
6794     if ($machine_type =~ m/^(\S+)\.pxe$/) {
6795         $machine_type = $1;
6796         $use_old_bios_files = 1;
6797     } else {
6798         my $kvmver = kvm_user_version();
6799         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
6800         # load new efi bios files on migration. So this hack is required to allow
6801         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
6802         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
6803         $use_old_bios_files = !qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 4);
6804     }
6805
6806     return ($use_old_bios_files, $machine_type);
6807 }
6808
6809 sub create_efidisk($$$$$) {
6810     my ($storecfg, $storeid, $vmid, $fmt, $arch) = @_;
6811
6812     my (undef, $ovmf_vars) = get_ovmf_files($arch);
6813     die "EFI vars default image not found\n" if ! -f $ovmf_vars;
6814
6815     my $vars_size = PVE::Tools::convert_size(-s $ovmf_vars, 'b' => 'kb');
6816     my $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $fmt, undef, $vars_size);
6817     PVE::Storage::activate_volumes($storecfg, [$volid]);
6818
6819     my $path = PVE::Storage::path($storecfg, $volid);
6820     eval {
6821         run_command(['/usr/bin/qemu-img', 'convert', '-n', '-f', 'raw', '-O', $fmt, $ovmf_vars, $path]);
6822     };
6823     die "Copying EFI vars image failed: $@" if $@;
6824
6825     return ($volid, $vars_size);
6826 }
6827
6828 sub lspci {
6829
6830     my $devices = {};
6831
6832     dir_glob_foreach("$pcisysfs/devices", '[a-f0-9]{4}:([a-f0-9]{2}:[a-f0-9]{2})\.([0-9])', sub {
6833             my (undef, $id, $function) = @_;
6834             my $res = { id => $id, function => $function};
6835             push @{$devices->{$id}}, $res;
6836     });
6837
6838     # Entries should be sorted by functions.
6839     foreach my $id (keys %$devices) {
6840         my $dev = $devices->{$id};
6841         $devices->{$id} = [ sort { $a->{function} <=> $b->{function} } @$dev ];
6842     }
6843
6844     return $devices;
6845 }
6846
6847 sub vm_iothreads_list {
6848     my ($vmid) = @_;
6849
6850     my $res = vm_mon_cmd($vmid, 'query-iothreads');
6851
6852     my $iothreads = {};
6853     foreach my $iothread (@$res) {
6854         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
6855     }
6856
6857     return $iothreads;
6858 }
6859
6860 sub scsihw_infos {
6861     my ($conf, $drive) = @_;
6862
6863     my $maxdev = 0;
6864
6865     if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)) {
6866         $maxdev = 7;
6867     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
6868         $maxdev = 1;
6869     } else {
6870         $maxdev = 256;
6871     }
6872
6873     my $controller = int($drive->{index} / $maxdev);
6874     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single') ? "virtioscsi" : "scsihw";
6875
6876     return ($maxdev, $controller, $controller_prefix);
6877 }
6878
6879 sub add_hyperv_enlightenments {
6880     my ($cpuFlags, $winversion, $machine_type, $kvmver, $bios, $gpu_passthrough) = @_;
6881
6882     return if $winversion < 6;
6883     return if $bios && $bios eq 'ovmf' && $winversion < 8;
6884
6885     push @$cpuFlags , 'hv_vendor_id=proxmox' if $gpu_passthrough;
6886
6887     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
6888         push @$cpuFlags , 'hv_spinlocks=0x1fff';
6889         push @$cpuFlags , 'hv_vapic';
6890         push @$cpuFlags , 'hv_time';
6891     } else {
6892         push @$cpuFlags , 'hv_spinlocks=0xffff';
6893     }
6894
6895     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 6)) {
6896         push @$cpuFlags , 'hv_reset';
6897         push @$cpuFlags , 'hv_vpindex';
6898         push @$cpuFlags , 'hv_runtime';
6899     }
6900
6901     if ($winversion >= 7) {
6902         push @$cpuFlags , 'hv_relaxed';
6903
6904         if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 12)) {
6905             push @$cpuFlags , 'hv_synic';
6906             push @$cpuFlags , 'hv_stimer';
6907         }
6908     }
6909 }
6910
6911 sub windows_version {
6912     my ($ostype) = @_;
6913
6914     return 0 if !$ostype;
6915
6916     my $winversion = 0;
6917
6918     if($ostype eq 'wxp' || $ostype eq 'w2k3' || $ostype eq 'w2k') {
6919         $winversion = 5;
6920     } elsif($ostype eq 'w2k8' || $ostype eq 'wvista') {
6921         $winversion = 6;
6922     } elsif ($ostype =~ m/^win(\d+)$/) {
6923         $winversion = $1;
6924     }
6925
6926     return $winversion;
6927 }
6928
6929 sub resolve_dst_disk_format {
6930         my ($storecfg, $storeid, $src_volname, $format) = @_;
6931         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
6932
6933         if (!$format) {
6934             # if no target format is specified, use the source disk format as hint
6935             if ($src_volname) {
6936                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6937                 $format = qemu_img_format($scfg, $src_volname);
6938             } else {
6939                 return $defFormat;
6940             }
6941         }
6942
6943         # test if requested format is supported - else use default
6944         my $supported = grep { $_ eq $format } @$validFormats;
6945         $format = $defFormat if !$supported;
6946         return $format;
6947 }
6948
6949 sub resolve_first_disk {
6950     my $conf = shift;
6951     my @disks = PVE::QemuServer::valid_drive_names();
6952     my $firstdisk;
6953     foreach my $ds (reverse @disks) {
6954         next if !$conf->{$ds};
6955         my $disk = PVE::QemuServer::parse_drive($ds, $conf->{$ds});
6956         next if PVE::QemuServer::drive_is_cdrom($disk);
6957         $firstdisk = $ds;
6958     }
6959     return $firstdisk;
6960 }
6961
6962 sub generate_uuid {
6963     my ($uuid, $uuid_str);
6964     UUID::generate($uuid);
6965     UUID::unparse($uuid, $uuid_str);
6966     return $uuid_str;
6967 }
6968
6969 sub generate_smbios1_uuid {
6970     return "uuid=".generate_uuid();
6971 }
6972
6973 sub nbd_stop {
6974     my ($vmid) = @_;
6975
6976     vm_mon_cmd($vmid, 'nbd-server-stop');
6977 }
6978
6979 # bash completion helper
6980
6981 sub complete_backup_archives {
6982     my ($cmdname, $pname, $cvalue) = @_;
6983
6984     my $cfg = PVE::Storage::config();
6985
6986     my $storeid;
6987
6988     if ($cvalue =~ m/^([^:]+):/) {
6989         $storeid = $1;
6990     }
6991
6992     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
6993
6994     my $res = [];
6995     foreach my $id (keys %$data) {
6996         foreach my $item (@{$data->{$id}}) {
6997             next if $item->{format} !~ m/^vma\.(gz|lzo)$/;
6998             push @$res, $item->{volid} if defined($item->{volid});
6999         }
7000     }
7001
7002     return $res;
7003 }
7004
7005 my $complete_vmid_full = sub {
7006     my ($running) = @_;
7007
7008     my $idlist = vmstatus();
7009
7010     my $res = [];
7011
7012     foreach my $id (keys %$idlist) {
7013         my $d = $idlist->{$id};
7014         if (defined($running)) {
7015             next if $d->{template};
7016             next if $running && $d->{status} ne 'running';
7017             next if !$running && $d->{status} eq 'running';
7018         }
7019         push @$res, $id;
7020
7021     }
7022     return $res;
7023 };
7024
7025 sub complete_vmid {
7026     return &$complete_vmid_full();
7027 }
7028
7029 sub complete_vmid_stopped {
7030     return &$complete_vmid_full(0);
7031 }
7032
7033 sub complete_vmid_running {
7034     return &$complete_vmid_full(1);
7035 }
7036
7037 sub complete_storage {
7038
7039     my $cfg = PVE::Storage::config();
7040     my $ids = $cfg->{ids};
7041
7042     my $res = [];
7043     foreach my $sid (keys %$ids) {
7044         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
7045         next if !$ids->{$sid}->{content}->{images};
7046         push @$res, $sid;
7047     }
7048
7049     return $res;
7050 }
7051
7052 1;