PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5 use POSIX;
   6 use IO::Handle;
   7 use IO::Select;
   8 use IO::File;
   9 use IO::Dir;
  10 use IO::Socket::UNIX;
  11 use File::Basename;
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use Digest::SHA;
  16 use Fcntl ':flock';
  17 use Cwd 'abs_path';
  18 use IPC::Open3;
  19 use JSON;
  20 use Fcntl;
  21 use PVE::SafeSyslog;
  22 use Storable qw(dclone);
  23 use PVE::Exception qw(raise raise_param_exc);
  24 use PVE::Storage;
  25 use PVE::Tools qw(run_command lock_file lock_file_full file_read_firstline dir_glob_foreach $IPV6RE);
  26 use PVE::JSONSchema qw(get_standard_option);
  27 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file cfs_lock_file);
  28 use PVE::INotify;
  29 use PVE::ProcFSTools;
  30 use PVE::QemuConfig;
  31 use PVE::QMPClient;
  32 use PVE::RPCEnvironment;
  33 use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr);
  34 use PVE::QemuServer::Memory;
  35 use PVE::QemuServer::USB qw(parse_usb_device);
  36 use PVE::QemuServer::Cloudinit;
  37 use PVE::Systemd;
  38 use Time::HiRes qw(gettimeofday);
  39 use File::Copy qw(copy);
  40 use URI::Escape;
  41
  42 my $EDK2_FW_BASE = '/usr/share/pve-edk2-firmware/';
  43 my $OVMF = {
  44     x86_64 => [
  45         "$EDK2_FW_BASE/OVMF_CODE.fd",
  46         "$EDK2_FW_BASE/OVMF_VARS.fd"
  47     ],
  48     aarch64 => [
  49         "$EDK2_FW_BASE/AAVMF_CODE.fd",
  50         "$EDK2_FW_BASE/AAVMF_VARS.fd"
  51     ],
  52 };
  53
  54 my $qemu_snap_storage = {rbd => 1, sheepdog => 1};
  55
  56 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  57
  58 my $QEMU_FORMAT_RE = qr/raw|cow|qcow|qcow2|qed|vmdk|cloop/;
  59
  60 # Note about locking: we use flock on the config file protect
  61 # against concurent actions.
  62 # Aditionaly, we have a 'lock' setting in the config file. This
  63 # can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
  64 # allowed when such lock is set. But you can ignore this kind of
  65 # lock with the --skiplock flag.
  66
  67 cfs_register_file('/qemu-server/',
  68                   \&parse_vm_config,
  69                   \&write_vm_config);
  70
  71 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
  72     description => "Some command save/restore state from this location.",
  73     type => 'string',
  74     maxLength => 128,
  75     optional => 1,
  76 });
  77
  78 PVE::JSONSchema::register_standard_option('pve-snapshot-name', {
  79     description => "The name of the snapshot.",
  80     type => 'string', format => 'pve-configid',
  81     maxLength => 40,
  82 });
  83
  84 PVE::JSONSchema::register_standard_option('pve-qm-image-format', {
  85     type => 'string',
  86     enum => [qw(raw cow qcow qed qcow2 vmdk cloop)],
  87     description => "The drive's backing file's data format.",
  88     optional => 1,
  89 });
  90
  91 PVE::JSONSchema::register_standard_option('pve-qemu-machine', {
  92         description => "Specifies the Qemu machine type.",
  93         type => 'string',
  94         pattern => '(pc|pc(-i440fx)?-\d+\.\d+(\.pxe)?|q35|pc-q35-\d+\.\d+(\.pxe)?|virt(?:-\d+\.\d+)?)',
  95         maxLength => 40,
  96         optional => 1,
  97 });
  98
  99 #no warnings 'redefine';
 100
 101 sub cgroups_write {
 102    my ($controller, $vmid, $option, $value) = @_;
 103
 104    my $path = "/sys/fs/cgroup/$controller/qemu.slice/$vmid.scope/$option";
 105    PVE::ProcFSTools::write_proc_entry($path, $value);
 106
 107 }
 108
 109 my $nodename = PVE::INotify::nodename();
 110
 111 mkdir "/etc/pve/nodes/$nodename";
 112 my $confdir = "/etc/pve/nodes/$nodename/qemu-server";
 113 mkdir $confdir;
 114
 115 my $var_run_tmpdir = "/var/run/qemu-server";
 116 mkdir $var_run_tmpdir;
 117
 118 my $lock_dir = "/var/lock/qemu-server";
 119 mkdir $lock_dir;
 120
 121 my $pcisysfs = "/sys/bus/pci";
 122
 123 my $cpu_vendor_list = {
 124     # Intel CPUs
 125     486 => 'GenuineIntel',
 126     pentium => 'GenuineIntel',
 127     pentium2  => 'GenuineIntel',
 128     pentium3  => 'GenuineIntel',
 129     coreduo => 'GenuineIntel',
 130     core2duo => 'GenuineIntel',
 131     Conroe  => 'GenuineIntel',
 132     Penryn  => 'GenuineIntel',
 133     Nehalem  => 'GenuineIntel',
 134     'Nehalem-IBRS'  => 'GenuineIntel',
 135     Westmere => 'GenuineIntel',
 136     'Westmere-IBRS' => 'GenuineIntel',
 137     SandyBridge => 'GenuineIntel',
 138     'SandyBridge-IBRS' => 'GenuineIntel',
 139     IvyBridge => 'GenuineIntel',
 140     'IvyBridge-IBRS' => 'GenuineIntel',
 141     Haswell => 'GenuineIntel',
 142     'Haswell-IBRS' => 'GenuineIntel',
 143     'Haswell-noTSX' => 'GenuineIntel',
 144     'Haswell-noTSX-IBRS' => 'GenuineIntel',
 145     Broadwell => 'GenuineIntel',
 146     'Broadwell-IBRS' => 'GenuineIntel',
 147     'Broadwell-noTSX' => 'GenuineIntel',
 148     'Broadwell-noTSX-IBRS' => 'GenuineIntel',
 149     'Skylake-Client' => 'GenuineIntel',
 150     'Skylake-Client-IBRS' => 'GenuineIntel',
 151     'Skylake-Server' => 'GenuineIntel',
 152     'Skylake-Server-IBRS' => 'GenuineIntel',
 153
 154     # AMD CPUs
 155     athlon => 'AuthenticAMD',
 156     phenom  => 'AuthenticAMD',
 157     Opteron_G1  => 'AuthenticAMD',
 158     Opteron_G2  => 'AuthenticAMD',
 159     Opteron_G3  => 'AuthenticAMD',
 160     Opteron_G4  => 'AuthenticAMD',
 161     Opteron_G5  => 'AuthenticAMD',
 162     EPYC => 'AuthenticAMD',
 163     'EPYC-IBPB' => 'AuthenticAMD',
 164
 165     # generic types, use vendor from host node
 166     host => 'default',
 167     kvm32 => 'default',
 168     kvm64 => 'default',
 169     qemu32 => 'default',
 170     qemu64 => 'default',
 171     max => 'default',
 172 };
 173
 174 my $cpu_flag = qr/[+-](pcid|spec-ctrl|ibpb|ssbd|virt-ssbd|amd-ssbd|amd-no-ssb|pdpe1gb)/;
 175
 176 my $cpu_fmt = {
 177     cputype => {
 178         description => "Emulated CPU type.",
 179         type => 'string',
 180         enum => [ sort { "\L$a" cmp "\L$b" } keys %$cpu_vendor_list ],
 181         default => 'kvm64',
 182         default_key => 1,
 183     },
 184     hidden => {
 185         description => "Do not identify as a KVM virtual machine.",
 186         type => 'boolean',
 187         optional => 1,
 188         default => 0
 189     },
 190     flags => {
 191         description => "List of additional CPU flags separated by ';'."
 192                      . " Use '+FLAG' to enable, '-FLAG' to disable a flag."
 193                      . " Currently supported flags: 'pcid', 'spec-ctrl', 'ibpb', 'ssbd', 'virt-ssbd', 'amd-ssbd', 'amd-no-ssb', 'pdpe1gb'.",
 194         format_description => '+FLAG[;-FLAG...]',
 195         type => 'string',
 196         pattern => qr/$cpu_flag(;$cpu_flag)*/,
 197         optional => 1,
 198     },
 199 };
 200
 201 my $watchdog_fmt = {
 202     model => {
 203         default_key => 1,
 204         type => 'string',
 205         enum => [qw(i6300esb ib700)],
 206         description => "Watchdog type to emulate.",
 207         default => 'i6300esb',
 208         optional => 1,
 209     },
 210     action => {
 211         type => 'string',
 212         enum => [qw(reset shutdown poweroff pause debug none)],
 213         description => "The action to perform if after activation the guest fails to poll the watchdog in time.",
 214         optional => 1,
 215     },
 216 };
 217 PVE::JSONSchema::register_format('pve-qm-watchdog', $watchdog_fmt);
 218
 219 my $agent_fmt = {
 220     enabled => {
 221         description => "Enable/disable Qemu GuestAgent.",
 222         type => 'boolean',
 223         default => 0,
 224         default_key => 1,
 225     },
 226     fstrim_cloned_disks => {
 227         description => "Run fstrim after cloning/moving a disk.",
 228         type => 'boolean',
 229         optional => 1,
 230         default => 0
 231     },
 232 };
 233
 234 my $vga_fmt = {
 235     type => {
 236         description => "Select the VGA type.",
 237         type => 'string',
 238         default => 'std',
 239         optional => 1,
 240         default_key => 1,
 241         enum => [qw(cirrus qxl qxl2 qxl3 qxl4 serial0 serial1 serial2 serial3 std virtio vmware)],
 242     },
 243     memory => {
 244         description => "Sets the VGA memory (in MiB). Has no effect with serial display.",
 245         type => 'integer',
 246         optional => 1,
 247         minimum => 4,
 248         maximum => 512,
 249     },
 250 };
 251
 252 my $confdesc = {
 253     onboot => {
 254         optional => 1,
 255         type => 'boolean',
 256         description => "Specifies whether a VM will be started during system bootup.",
 257         default => 0,
 258     },
 259     autostart => {
 260         optional => 1,
 261         type => 'boolean',
 262         description => "Automatic restart after crash (currently ignored).",
 263         default => 0,
 264     },
 265     hotplug => {
 266         optional => 1,
 267         type => 'string', format => 'pve-hotplug-features',
 268         description => "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
 269         default => 'network,disk,usb',
 270     },
 271     reboot => {
 272         optional => 1,
 273         type => 'boolean',
 274         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 275         default => 1,
 276     },
 277     lock => {
 278         optional => 1,
 279         type => 'string',
 280         description => "Lock/unlock the VM.",
 281         enum => [qw(migrate backup snapshot rollback)],
 282     },
 283     cpulimit => {
 284         optional => 1,
 285         type => 'number',
 286         description => "Limit of CPU usage.",
 287         verbose_description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
 288         minimum => 0,
 289         maximum => 128,
 290         default => 0,
 291     },
 292     cpuunits => {
 293         optional => 1,
 294         type => 'integer',
 295         description => "CPU weight for a VM.",
 296         verbose_description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.",
 297         minimum => 2,
 298         maximum => 262144,
 299         default => 1024,
 300     },
 301     memory => {
 302         optional => 1,
 303         type => 'integer',
 304         description => "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
 305         minimum => 16,
 306         default => 512,
 307     },
 308     balloon => {
 309         optional => 1,
 310         type => 'integer',
 311         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 312         minimum => 0,
 313     },
 314     shares => {
 315         optional => 1,
 316         type => 'integer',
 317         description => "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning. Auto-ballooning is done by pvestatd.",
 318         minimum => 0,
 319         maximum => 50000,
 320         default => 1000,
 321     },
 322     keyboard => {
 323         optional => 1,
 324         type => 'string',
 325         description => "Keybord layout for vnc server. Default is read from the '/etc/pve/datacenter.cfg' configuration file.".
 326                        "It should not be necessary to set it.",
 327         enum => PVE::Tools::kvmkeymaplist(),
 328         default => undef,
 329     },
 330     name => {
 331         optional => 1,
 332         type => 'string', format => 'dns-name',
 333         description => "Set a name for the VM. Only used on the configuration web interface.",
 334     },
 335     scsihw => {
 336         optional => 1,
 337         type => 'string',
 338         description => "SCSI controller model",
 339         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 340         default => 'lsi',
 341     },
 342     description => {
 343         optional => 1,
 344         type => 'string',
 345         description => "Description for the VM. Only used on the configuration web interface. This is saved as comment inside the configuration file.",
 346     },
 347     ostype => {
 348         optional => 1,
 349         type => 'string',
 350         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 l24 l26 solaris)],
 351         description => "Specify guest operating system.",
 352         verbose_description => <<EODESC,
 353 Specify guest operating system. This is used to enable special
 354 optimization/features for specific operating systems:
 355
 356 [horizontal]
 357 other;; unspecified OS
 358 wxp;; Microsoft Windows XP
 359 w2k;; Microsoft Windows 2000
 360 w2k3;; Microsoft Windows 2003
 361 w2k8;; Microsoft Windows 2008
 362 wvista;; Microsoft Windows Vista
 363 win7;; Microsoft Windows 7
 364 win8;; Microsoft Windows 8/2012/2012r2
 365 win10;; Microsoft Windows 10/2016
 366 l24;; Linux 2.4 Kernel
 367 l26;; Linux 2.6/3.X Kernel
 368 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
 369 EODESC
 370     },
 371     boot => {
 372         optional => 1,
 373         type => 'string',
 374         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n).",
 375         pattern => '[acdn]{1,4}',
 376         default => 'cdn',
 377     },
 378     bootdisk => {
 379         optional => 1,
 380         type => 'string', format => 'pve-qm-bootdisk',
 381         description => "Enable booting from specified disk.",
 382         pattern => '(ide|sata|scsi|virtio)\d+',
 383     },
 384     smp => {
 385         optional => 1,
 386         type => 'integer',
 387         description => "The number of CPUs. Please use option -sockets instead.",
 388         minimum => 1,
 389         default => 1,
 390     },
 391     sockets => {
 392         optional => 1,
 393         type => 'integer',
 394         description => "The number of CPU sockets.",
 395         minimum => 1,
 396         default => 1,
 397     },
 398     cores => {
 399         optional => 1,
 400         type => 'integer',
 401         description => "The number of cores per socket.",
 402         minimum => 1,
 403         default => 1,
 404     },
 405     numa => {
 406         optional => 1,
 407         type => 'boolean',
 408         description => "Enable/disable NUMA.",
 409         default => 0,
 410     },
 411     hugepages => {
 412         optional => 1,
 413         type => 'string',
 414         description => "Enable/disable hugepages memory.",
 415         enum => [qw(any 2 1024)],
 416     },
 417     vcpus => {
 418         optional => 1,
 419         type => 'integer',
 420         description => "Number of hotplugged vcpus.",
 421         minimum => 1,
 422         default => 0,
 423     },
 424     acpi => {
 425         optional => 1,
 426         type => 'boolean',
 427         description => "Enable/disable ACPI.",
 428         default => 1,
 429     },
 430     agent => {
 431         optional => 1,
 432         description => "Enable/disable Qemu GuestAgent and its properties.",
 433         type => 'string',
 434         format => $agent_fmt,
 435     },
 436     kvm => {
 437         optional => 1,
 438         type => 'boolean',
 439         description => "Enable/disable KVM hardware virtualization.",
 440         default => 1,
 441     },
 442     tdf => {
 443         optional => 1,
 444         type => 'boolean',
 445         description => "Enable/disable time drift fix.",
 446         default => 0,
 447     },
 448     localtime => {
 449         optional => 1,
 450         type => 'boolean',
 451         description => "Set the real time clock to local time. This is enabled by default if ostype indicates a Microsoft OS.",
 452     },
 453     freeze => {
 454         optional => 1,
 455         type => 'boolean',
 456         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 457     },
 458     vga => {
 459         optional => 1,
 460         type => 'string', format => $vga_fmt,
 461         description => "Configure the VGA hardware.",
 462         verbose_description => "Configure the VGA Hardware. If you want to use ".
 463             "high resolution modes (>= 1280x1024x16) you may need to increase " .
 464             "the vga memory option. Since QEMU 2.9 the default VGA display type " .
 465             "is 'std' for all OS types besides some Windows versions (XP and " .
 466             "older) which use 'cirrus'. The 'qxl' option enables the SPICE " .
 467             "display server. For win* OS you can select how many independent " .
 468             "displays you want, Linux guests can add displays them self.\n".
 469             "You can also run without any graphic card, using a serial device as terminal.",
 470     },
 471     watchdog => {
 472         optional => 1,
 473         type => 'string', format => 'pve-qm-watchdog',
 474         description => "Create a virtual hardware watchdog device.",
 475         verbose_description => "Create a virtual hardware watchdog device. Once enabled" .
 476             " (by a guest action), the watchdog must be periodically polled " .
 477             "by an agent inside the guest or else the watchdog will reset " .
 478             "the guest (or execute the respective action specified)",
 479     },
 480     startdate => {
 481         optional => 1,
 482         type => 'string',
 483         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 484         description => "Set the initial date of the real time clock. Valid format for date are: 'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 485         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 486         default => 'now',
 487     },
 488     startup =>  get_standard_option('pve-startup-order'),
 489     template => {
 490         optional => 1,
 491         type => 'boolean',
 492         description => "Enable/disable Template.",
 493         default => 0,
 494     },
 495     args => {
 496         optional => 1,
 497         type => 'string',
 498         description => "Arbitrary arguments passed to kvm.",
 499         verbose_description => <<EODESCR,
 500 Arbitrary arguments passed to kvm, for example:
 501
 502 args: -no-reboot -no-hpet
 503
 504 NOTE: this option is for experts only.
 505 EODESCR
 506     },
 507     tablet => {
 508         optional => 1,
 509         type => 'boolean',
 510         default => 1,
 511         description => "Enable/disable the USB tablet device.",
 512         verbose_description => "Enable/disable the USB tablet device. This device is " .
 513             "usually needed to allow absolute mouse positioning with VNC. " .
 514             "Else the mouse runs out of sync with normal VNC clients. " .
 515             "If you're running lots of console-only guests on one host, " .
 516             "you may consider disabling this to save some context switches. " .
 517             "This is turned off by default if you use spice (-vga=qxl).",
 518     },
 519     migrate_speed => {
 520         optional => 1,
 521         type => 'integer',
 522         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 523         minimum => 0,
 524         default => 0,
 525     },
 526     migrate_downtime => {
 527         optional => 1,
 528         type => 'number',
 529         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 530         minimum => 0,
 531         default => 0.1,
 532     },
 533     cdrom => {
 534         optional => 1,
 535         type => 'string', format => 'pve-qm-ide',
 536         typetext => '<volume>',
 537         description => "This is an alias for option -ide2",
 538     },
 539     cpu => {
 540         optional => 1,
 541         description => "Emulated CPU type.",
 542         type => 'string',
 543         format => $cpu_fmt,
 544     },
 545     parent => get_standard_option('pve-snapshot-name', {
 546         optional => 1,
 547         description => "Parent snapshot name. This is used internally, and should not be modified.",
 548     }),
 549     snaptime => {
 550         optional => 1,
 551         description => "Timestamp for snapshots.",
 552         type => 'integer',
 553         minimum => 0,
 554     },
 555     vmstate => {
 556         optional => 1,
 557         type => 'string', format => 'pve-volume-id',
 558         description => "Reference to a volume which stores the VM state. This is used internally for snapshots.",
 559     },
 560     vmstatestorage => get_standard_option('pve-storage-id', {
 561         description => "Default storage for VM state volumes/files.",
 562         optional => 1,
 563     }),
 564     runningmachine => get_standard_option('pve-qemu-machine', {
 565         description => "Specifies the Qemu machine type of the running vm. This is used internally for snapshots.",
 566     }),
 567     machine => get_standard_option('pve-qemu-machine'),
 568     arch => {
 569         description => "Virtual processor architecture. Defaults to the host.",
 570         optional => 1,
 571         type => 'string',
 572         enum => [qw(x86_64 aarch64)],
 573     },
 574     smbios1 => {
 575         description => "Specify SMBIOS type 1 fields.",
 576         type => 'string', format => 'pve-qm-smbios1',
 577         maxLength => 256,
 578         optional => 1,
 579     },
 580     protection => {
 581         optional => 1,
 582         type => 'boolean',
 583         description => "Sets the protection flag of the VM. This will disable the remove VM and remove disk operations.",
 584         default => 0,
 585     },
 586     bios => {
 587         optional => 1,
 588         type => 'string',
 589         enum => [ qw(seabios ovmf) ],
 590         description => "Select BIOS implementation.",
 591         default => 'seabios',
 592     },
 593     vmgenid => {
 594         type => 'string',
 595         pattern => '(?:[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}|[01])',
 596         format_description => 'UUID',
 597         description => "Set VM Generation ID. Use '1' to autogenerate on create or update, pass '0' to disable explicitly.",
 598         verbose_description => "The VM generation ID (vmgenid) device exposes a".
 599             " 128-bit integer value identifier to the guest OS. This allows to".
 600             " notify the guest operating system when the virtual machine is".
 601             " executed with a different configuration (e.g. snapshot execution".
 602             " or creation from a template). The guest operating system notices".
 603             " the change, and is then able to react as appropriate by marking".
 604             " its copies of distributed databases as dirty, re-initializing its".
 605             " random number generator, etc.\n".
 606             "Note that auto-creation only works when done throug API/CLI create".
 607             " or update methods, but not when manually editing the config file.",
 608         default => "1 (autogenerated)",
 609         optional => 1,
 610     },
 611 };
 612
 613 my $confdesc_cloudinit = {
 614     citype => {
 615         optional => 1,
 616         type => 'string',
 617         description => 'Specifies the cloud-init configuration format. The default depends on the configured operating system type (`ostype`. We use the `nocloud` format for Linux, and `configdrive2` for windows.',
 618         enum => ['configdrive2', 'nocloud'],
 619     },
 620     ciuser => {
 621         optional => 1,
 622         type => 'string',
 623         description => "cloud-init: User name to change ssh keys and password for instead of the image's configured default user.",
 624     },
 625     cipassword => {
 626         optional => 1,
 627         type => 'string',
 628         description => 'cloud-init: Password to assign the user. Using this is generally not recommended. Use ssh keys instead. Also note that older cloud-init versions do not support hashed passwords.',
 629     },
 630     searchdomain => {
 631         optional => 1,
 632         type => 'string',
 633         description => "cloud-init: Sets DNS search domains for a container. Create will automatically use the setting from the host if neither searchdomain nor nameserver are set.",
 634     },
 635     nameserver => {
 636         optional => 1,
 637         type => 'string', format => 'address-list',
 638         description => "cloud-init: Sets DNS server IP address for a container. Create will automatically use the setting from the host if neither searchdomain nor nameserver are set.",
 639     },
 640     sshkeys => {
 641         optional => 1,
 642         type => 'string',
 643         format => 'urlencoded',
 644         description => "cloud-init: Setup public SSH keys (one key per line, OpenSSH format).",
 645     },
 646 };
 647
 648 # what about other qemu settings ?
 649 #cpu => 'string',
 650 #machine => 'string',
 651 #fda => 'file',
 652 #fdb => 'file',
 653 #mtdblock => 'file',
 654 #sd => 'file',
 655 #pflash => 'file',
 656 #snapshot => 'bool',
 657 #bootp => 'file',
 658 ##tftp => 'dir',
 659 ##smb => 'dir',
 660 #kernel => 'file',
 661 #append => 'string',
 662 #initrd => 'file',
 663 ##soundhw => 'string',
 664
 665 while (my ($k, $v) = each %$confdesc) {
 666     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 667 }
 668
 669 my $MAX_IDE_DISKS = 4;
 670 my $MAX_SCSI_DISKS = 14;
 671 my $MAX_VIRTIO_DISKS = 16;
 672 my $MAX_SATA_DISKS = 6;
 673 my $MAX_USB_DEVICES = 5;
 674 my $MAX_NETS = 32;
 675 my $MAX_UNUSED_DISKS = 256;
 676 my $MAX_HOSTPCI_DEVICES = 4;
 677 my $MAX_SERIAL_PORTS = 4;
 678 my $MAX_PARALLEL_PORTS = 3;
 679 my $MAX_NUMA = 8;
 680
 681 my $numa_fmt = {
 682     cpus => {
 683         type => "string",
 684         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 685         description => "CPUs accessing this NUMA node.",
 686         format_description => "id[-id];...",
 687     },
 688     memory => {
 689         type => "number",
 690         description => "Amount of memory this NUMA node provides.",
 691         optional => 1,
 692     },
 693     hostnodes => {
 694         type => "string",
 695         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 696         description => "Host NUMA nodes to use.",
 697         format_description => "id[-id];...",
 698         optional => 1,
 699     },
 700     policy => {
 701         type => 'string',
 702         enum => [qw(preferred bind interleave)],
 703         description => "NUMA allocation policy.",
 704         optional => 1,
 705     },
 706 };
 707 PVE::JSONSchema::register_format('pve-qm-numanode', $numa_fmt);
 708 my $numadesc = {
 709     optional => 1,
 710     type => 'string', format => $numa_fmt,
 711     description => "NUMA topology.",
 712 };
 713 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 714
 715 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 716     $confdesc->{"numa$i"} = $numadesc;
 717 }
 718
 719 my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000',  'pcnet',  'virtio',
 720                       'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3',
 721                       'e1000-82540em', 'e1000-82544gc', 'e1000-82545em'];
 722 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 723
 724 my $net_fmt_bridge_descr = <<__EOD__;
 725 Bridge to attach the network device to. The Proxmox VE standard bridge
 726 is called 'vmbr0'.
 727
 728 If you do not specify a bridge, we create a kvm user (NATed) network
 729 device, which provides DHCP and DNS services. The following addresses
 730 are used:
 731
 732  10.0.2.2   Gateway
 733  10.0.2.3   DNS Server
 734  10.0.2.4   SMB Server
 735
 736 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 737 __EOD__
 738
 739 my $net_fmt = {
 740     macaddr => {
 741         type => 'string',
 742         pattern => qr/[0-9a-f]{2}(?::[0-9a-f]{2}){5}/i,
 743         description => "MAC address. That address must be unique withing your network. This is automatically generated if not specified.",
 744         format_description => "XX:XX:XX:XX:XX:XX",
 745         optional => 1,
 746     },
 747     model => {
 748         type => 'string',
 749         description => "Network Card Model. The 'virtio' model provides the best performance with very low CPU overhead. If your guest does not support this driver, it is usually best to use 'e1000'.",
 750         enum => $nic_model_list,
 751         default_key => 1,
 752     },
 753     (map { $_ => { keyAlias => 'model', alias => 'macaddr' }} @$nic_model_list),
 754     bridge => {
 755         type => 'string',
 756         description => $net_fmt_bridge_descr,
 757         format_description => 'bridge',
 758         optional => 1,
 759     },
 760     queues => {
 761         type => 'integer',
 762         minimum => 0, maximum => 16,
 763         description => 'Number of packet queues to be used on the device.',
 764         optional => 1,
 765     },
 766     rate => {
 767         type => 'number',
 768         minimum => 0,
 769         description => "Rate limit in mbps (megabytes per second) as floating point number.",
 770         optional => 1,
 771     },
 772     tag => {
 773         type => 'integer',
 774         minimum => 1, maximum => 4094,
 775         description => 'VLAN tag to apply to packets on this interface.',
 776         optional => 1,
 777     },
 778     trunks => {
 779         type => 'string',
 780         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 781         description => 'VLAN trunks to pass through this interface.',
 782         format_description => 'vlanid[;vlanid...]',
 783         optional => 1,
 784     },
 785     firewall => {
 786         type => 'boolean',
 787         description => 'Whether this interface should be protected by the firewall.',
 788         optional => 1,
 789     },
 790     link_down => {
 791         type => 'boolean',
 792         description => 'Whether this interface should be disconnected (like pulling the plug).',
 793         optional => 1,
 794     },
 795 };
 796
 797 my $netdesc = {
 798     optional => 1,
 799     type => 'string', format => $net_fmt,
 800     description => "Specify network devices.",
 801 };
 802
 803 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 804
 805 my $ipconfig_fmt = {
 806     ip => {
 807         type => 'string',
 808         format => 'pve-ipv4-config',
 809         format_description => 'IPv4Format/CIDR',
 810         description => 'IPv4 address in CIDR format.',
 811         optional => 1,
 812         default => 'dhcp',
 813     },
 814     gw => {
 815         type => 'string',
 816         format => 'ipv4',
 817         format_description => 'GatewayIPv4',
 818         description => 'Default gateway for IPv4 traffic.',
 819         optional => 1,
 820         requires => 'ip',
 821     },
 822     ip6 => {
 823         type => 'string',
 824         format => 'pve-ipv6-config',
 825         format_description => 'IPv6Format/CIDR',
 826         description => 'IPv6 address in CIDR format.',
 827         optional => 1,
 828         default => 'dhcp',
 829     },
 830     gw6 => {
 831         type => 'string',
 832         format => 'ipv6',
 833         format_description => 'GatewayIPv6',
 834         description => 'Default gateway for IPv6 traffic.',
 835         optional => 1,
 836         requires => 'ip6',
 837     },
 838 };
 839 PVE::JSONSchema::register_format('pve-qm-ipconfig', $ipconfig_fmt);
 840 my $ipconfigdesc = {
 841     optional => 1,
 842     type => 'string', format => 'pve-qm-ipconfig',
 843     description => <<'EODESCR',
 844 cloud-init: Specify IP addresses and gateways for the corresponding interface.
 845
 846 IP addresses use CIDR notation, gateways are optional but need an IP of the same type specified.
 847
 848 The special string 'dhcp' can be used for IP addresses to use DHCP, in which case no explicit gateway should be provided.
 849 For IPv6 the special string 'auto' can be used to use stateless autoconfiguration.
 850
 851 If cloud-init is enabled and neither an IPv4 nor an IPv6 address is specified, it defaults to using dhcp on IPv4.
 852 EODESCR
 853 };
 854 PVE::JSONSchema::register_standard_option("pve-qm-ipconfig", $netdesc);
 855
 856 for (my $i = 0; $i < $MAX_NETS; $i++)  {
 857     $confdesc->{"net$i"} = $netdesc;
 858     $confdesc_cloudinit->{"ipconfig$i"} = $ipconfigdesc;
 859 }
 860
 861 foreach my $key (keys %$confdesc_cloudinit) {
 862     $confdesc->{$key} = $confdesc_cloudinit->{$key};
 863 }
 864
 865 PVE::JSONSchema::register_format('pve-volume-id-or-qm-path', \&verify_volume_id_or_qm_path);
 866 sub verify_volume_id_or_qm_path {
 867     my ($volid, $noerr) = @_;
 868
 869     if ($volid eq 'none' || $volid eq 'cdrom' || $volid =~ m|^/|) {
 870         return $volid;
 871     }
 872
 873     # if its neither 'none' nor 'cdrom' nor a path, check if its a volume-id
 874     $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') };
 875     if ($@) {
 876         return undef if $noerr;
 877         die $@;
 878     }
 879     return $volid;
 880 }
 881
 882 my $drivename_hash;
 883
 884 my %drivedesc_base = (
 885     volume => { alias => 'file' },
 886     file => {
 887         type => 'string',
 888         format => 'pve-volume-id-or-qm-path',
 889         default_key => 1,
 890         format_description => 'volume',
 891         description => "The drive's backing volume.",
 892     },
 893     media => {
 894         type => 'string',
 895         enum => [qw(cdrom disk)],
 896         description => "The drive's media type.",
 897         default => 'disk',
 898         optional => 1
 899     },
 900     cyls => {
 901         type => 'integer',
 902         description => "Force the drive's physical geometry to have a specific cylinder count.",
 903         optional => 1
 904     },
 905     heads => {
 906         type => 'integer',
 907         description => "Force the drive's physical geometry to have a specific head count.",
 908         optional => 1
 909     },
 910     secs => {
 911         type => 'integer',
 912         description => "Force the drive's physical geometry to have a specific sector count.",
 913         optional => 1
 914     },
 915     trans => {
 916         type => 'string',
 917         enum => [qw(none lba auto)],
 918         description => "Force disk geometry bios translation mode.",
 919         optional => 1,
 920     },
 921     snapshot => {
 922         type => 'boolean',
 923         description => "Controls qemu's snapshot mode feature."
 924             . " If activated, changes made to the disk are temporary and will"
 925             . " be discarded when the VM is shutdown.",
 926         optional => 1,
 927     },
 928     cache => {
 929         type => 'string',
 930         enum => [qw(none writethrough writeback unsafe directsync)],
 931         description => "The drive's cache mode",
 932         optional => 1,
 933     },
 934     format => get_standard_option('pve-qm-image-format'),
 935     size => {
 936         type => 'string',
 937         format => 'disk-size',
 938         format_description => 'DiskSize',
 939         description => "Disk size. This is purely informational and has no effect.",
 940         optional => 1,
 941     },
 942     backup => {
 943         type => 'boolean',
 944         description => "Whether the drive should be included when making backups.",
 945         optional => 1,
 946     },
 947     replicate => {
 948         type => 'boolean',
 949         description => 'Whether the drive should considered for replication jobs.',
 950         optional => 1,
 951         default => 1,
 952     },
 953     rerror => {
 954         type => 'string',
 955         enum => [qw(ignore report stop)],
 956         description => 'Read error action.',
 957         optional => 1,
 958     },
 959     werror => {
 960         type => 'string',
 961         enum => [qw(enospc ignore report stop)],
 962         description => 'Write error action.',
 963         optional => 1,
 964     },
 965     aio => {
 966         type => 'string',
 967         enum => [qw(native threads)],
 968         description => 'AIO type to use.',
 969         optional => 1,
 970     },
 971     discard => {
 972         type => 'string',
 973         enum => [qw(ignore on)],
 974         description => 'Controls whether to pass discard/trim requests to the underlying storage.',
 975         optional => 1,
 976     },
 977     detect_zeroes => {
 978         type => 'boolean',
 979         description => 'Controls whether to detect and try to optimize writes of zeroes.',
 980         optional => 1,
 981     },
 982     serial => {
 983         type => 'string',
 984         format => 'urlencoded',
 985         format_description => 'serial',
 986         maxLength => 20*3, # *3 since it's %xx url enoded
 987         description => "The drive's reported serial number, url-encoded, up to 20 bytes long.",
 988         optional => 1,
 989     },
 990     shared => {
 991         type => 'boolean',
 992         description => 'Mark this locally-managed volume as available on all nodes',
 993         verbose_description => "Mark this locally-managed volume as available on all nodes.\n\nWARNING: This option does not share the volume automatically, it assumes it is shared already!",
 994         optional => 1,
 995         default => 0,
 996     }
 997 );
 998
 999 my %iothread_fmt = ( iothread => {
1000         type => 'boolean',
1001         description => "Whether to use iothreads for this drive",
1002         optional => 1,
1003 });
1004
1005 my %model_fmt = (
1006     model => {
1007         type => 'string',
1008         format => 'urlencoded',
1009         format_description => 'model',
1010         maxLength => 40*3, # *3 since it's %xx url enoded
1011         description => "The drive's reported model name, url-encoded, up to 40 bytes long.",
1012         optional => 1,
1013     },
1014 );
1015
1016 my %queues_fmt = (
1017     queues => {
1018         type => 'integer',
1019         description => "Number of queues.",
1020         minimum => 2,
1021         optional => 1
1022     }
1023 );
1024
1025 my %scsiblock_fmt = (
1026     scsiblock => {
1027         type => 'boolean',
1028         description => "whether to use scsi-block for full passthrough of host block device\n\nWARNING: can lead to I/O errors in combination with low memory or high memory fragmentation on host",
1029         optional => 1,
1030         default => 0,
1031     },
1032 );
1033
1034 my %ssd_fmt = (
1035     ssd => {
1036         type => 'boolean',
1037         description => "Whether to expose this drive as an SSD, rather than a rotational hard disk.",
1038         optional => 1,
1039     },
1040 );
1041
1042 my $add_throttle_desc = sub {
1043     my ($key, $type, $what, $unit, $longunit, $minimum) = @_;
1044     my $d = {
1045         type => $type,
1046         format_description => $unit,
1047         description => "Maximum $what in $longunit.",
1048         optional => 1,
1049     };
1050     $d->{minimum} = $minimum if defined($minimum);
1051     $drivedesc_base{$key} = $d;
1052 };
1053 # throughput: (leaky bucket)
1054 $add_throttle_desc->('bps',     'integer', 'r/w speed',   'bps',  'bytes per second');
1055 $add_throttle_desc->('bps_rd',  'integer', 'read speed',  'bps',  'bytes per second');
1056 $add_throttle_desc->('bps_wr',  'integer', 'write speed', 'bps',  'bytes per second');
1057 $add_throttle_desc->('mbps',    'number',  'r/w speed',   'mbps', 'megabytes per second');
1058 $add_throttle_desc->('mbps_rd', 'number',  'read speed',  'mbps', 'megabytes per second');
1059 $add_throttle_desc->('mbps_wr', 'number',  'write speed', 'mbps', 'megabytes per second');
1060 $add_throttle_desc->('iops',    'integer', 'r/w I/O',     'iops', 'operations per second');
1061 $add_throttle_desc->('iops_rd', 'integer', 'read I/O',    'iops', 'operations per second');
1062 $add_throttle_desc->('iops_wr', 'integer', 'write I/O',   'iops', 'operations per second');
1063
1064 # pools: (pool of IO before throttling starts taking effect)
1065 $add_throttle_desc->('mbps_max',    'number',  'unthrottled r/w pool',       'mbps', 'megabytes per second');
1066 $add_throttle_desc->('mbps_rd_max', 'number',  'unthrottled read pool',      'mbps', 'megabytes per second');
1067 $add_throttle_desc->('mbps_wr_max', 'number',  'unthrottled write pool',     'mbps', 'megabytes per second');
1068 $add_throttle_desc->('iops_max',    'integer', 'unthrottled r/w I/O pool',   'iops', 'operations per second');
1069 $add_throttle_desc->('iops_rd_max', 'integer', 'unthrottled read I/O pool',  'iops', 'operations per second');
1070 $add_throttle_desc->('iops_wr_max', 'integer', 'unthrottled write I/O pool', 'iops', 'operations per second');
1071
1072 # burst lengths
1073 $add_throttle_desc->('bps_max_length',     'integer', 'length of I/O bursts',       'seconds', 'seconds', 1);
1074 $add_throttle_desc->('bps_rd_max_length',  'integer', 'length of read I/O bursts',  'seconds', 'seconds', 1);
1075 $add_throttle_desc->('bps_wr_max_length',  'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
1076 $add_throttle_desc->('iops_max_length',    'integer', 'length of I/O bursts',       'seconds', 'seconds', 1);
1077 $add_throttle_desc->('iops_rd_max_length', 'integer', 'length of read I/O bursts',  'seconds', 'seconds', 1);
1078 $add_throttle_desc->('iops_wr_max_length', 'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
1079
1080 # legacy support
1081 $drivedesc_base{'bps_rd_length'} = { alias => 'bps_rd_max_length' };
1082 $drivedesc_base{'bps_wr_length'} = { alias => 'bps_wr_max_length' };
1083 $drivedesc_base{'iops_rd_length'} = { alias => 'iops_rd_max_length' };
1084 $drivedesc_base{'iops_wr_length'} = { alias => 'iops_wr_max_length' };
1085
1086 my $ide_fmt = {
1087     %drivedesc_base,
1088     %model_fmt,
1089     %ssd_fmt,
1090 };
1091 PVE::JSONSchema::register_format("pve-qm-ide", $ide_fmt);
1092
1093 my $idedesc = {
1094     optional => 1,
1095     type => 'string', format => $ide_fmt,
1096     description => "Use volume as IDE hard disk or CD-ROM (n is 0 to " .($MAX_IDE_DISKS -1) . ").",
1097 };
1098 PVE::JSONSchema::register_standard_option("pve-qm-ide", $idedesc);
1099
1100 my $scsi_fmt = {
1101     %drivedesc_base,
1102     %iothread_fmt,
1103     %queues_fmt,
1104     %scsiblock_fmt,
1105     %ssd_fmt,
1106 };
1107 my $scsidesc = {
1108     optional => 1,
1109     type => 'string', format => $scsi_fmt,
1110     description => "Use volume as SCSI hard disk or CD-ROM (n is 0 to " . ($MAX_SCSI_DISKS - 1) . ").",
1111 };
1112 PVE::JSONSchema::register_standard_option("pve-qm-scsi", $scsidesc);
1113
1114 my $sata_fmt = {
1115     %drivedesc_base,
1116     %ssd_fmt,
1117 };
1118 my $satadesc = {
1119     optional => 1,
1120     type => 'string', format => $sata_fmt,
1121     description => "Use volume as SATA hard disk or CD-ROM (n is 0 to " . ($MAX_SATA_DISKS - 1). ").",
1122 };
1123 PVE::JSONSchema::register_standard_option("pve-qm-sata", $satadesc);
1124
1125 my $virtio_fmt = {
1126     %drivedesc_base,
1127     %iothread_fmt,
1128 };
1129 my $virtiodesc = {
1130     optional => 1,
1131     type => 'string', format => $virtio_fmt,
1132     description => "Use volume as VIRTIO hard disk (n is 0 to " . ($MAX_VIRTIO_DISKS - 1) . ").",
1133 };
1134 PVE::JSONSchema::register_standard_option("pve-qm-virtio", $virtiodesc);
1135
1136 my $alldrive_fmt = {
1137     %drivedesc_base,
1138     %iothread_fmt,
1139     %model_fmt,
1140     %queues_fmt,
1141     %scsiblock_fmt,
1142     %ssd_fmt,
1143 };
1144
1145 my $efidisk_fmt = {
1146     volume => { alias => 'file' },
1147     file => {
1148         type => 'string',
1149         format => 'pve-volume-id-or-qm-path',
1150         default_key => 1,
1151         format_description => 'volume',
1152         description => "The drive's backing volume.",
1153     },
1154     format => get_standard_option('pve-qm-image-format'),
1155     size => {
1156         type => 'string',
1157         format => 'disk-size',
1158         format_description => 'DiskSize',
1159         description => "Disk size. This is purely informational and has no effect.",
1160         optional => 1,
1161     },
1162 };
1163
1164 my $efidisk_desc = {
1165     optional => 1,
1166     type => 'string', format => $efidisk_fmt,
1167     description => "Configure a Disk for storing EFI vars",
1168 };
1169
1170 PVE::JSONSchema::register_standard_option("pve-qm-efidisk", $efidisk_desc);
1171
1172 my $usb_fmt = {
1173     host => {
1174         default_key => 1,
1175         type => 'string', format => 'pve-qm-usb-device',
1176         format_description => 'HOSTUSBDEVICE|spice',
1177         description => <<EODESCR,
1178 The Host USB device or port or the value 'spice'. HOSTUSBDEVICE syntax is:
1179
1180  'bus-port(.port)*' (decimal numbers) or
1181  'vendor_id:product_id' (hexadeciaml numbers) or
1182  'spice'
1183
1184 You can use the 'lsusb -t' command to list existing usb devices.
1185
1186 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1187
1188 The value 'spice' can be used to add a usb redirection devices for spice.
1189 EODESCR
1190     },
1191     usb3 => {
1192         optional => 1,
1193         type => 'boolean',
1194         description => "Specifies whether if given host option is a USB3 device or port (this does currently not work reliably with spice redirection and is then ignored).",
1195         default => 0,
1196     },
1197 };
1198
1199 my $usbdesc = {
1200     optional => 1,
1201     type => 'string', format => $usb_fmt,
1202     description => "Configure an USB device (n is 0 to 4).",
1203 };
1204 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
1205
1206 # NOTE: the match-groups of this regex are used in parse_hostpci
1207 my $PCIRE = qr/([a-f0-9]{2}:[a-f0-9]{2})(?:\.([a-f0-9]))?/;
1208 my $hostpci_fmt = {
1209     host => {
1210         default_key => 1,
1211         type => 'string',
1212         pattern => qr/$PCIRE(;$PCIRE)*/,
1213         format_description => 'HOSTPCIID[;HOSTPCIID2...]',
1214         description => <<EODESCR,
1215 Host PCI device pass through. The PCI ID of a host's PCI device or a list
1216 of PCI virtual functions of the host. HOSTPCIID syntax is:
1217
1218 'bus:dev.func' (hexadecimal numbers)
1219
1220 You can us the 'lspci' command to list existing PCI devices.
1221 EODESCR
1222     },
1223     rombar => {
1224         type => 'boolean',
1225         description =>  "Specify whether or not the device's ROM will be visible in the guest's memory map.",
1226         optional => 1,
1227         default => 1,
1228     },
1229     romfile => {
1230         type => 'string',
1231         pattern => '[^,;]+',
1232         format_description => 'string',
1233         description => "Custom pci device rom filename (must be located in /usr/share/kvm/).",
1234         optional => 1,
1235     },
1236     pcie => {
1237         type => 'boolean',
1238         description =>  "Choose the PCI-express bus (needs the 'q35' machine model).",
1239         optional => 1,
1240         default => 0,
1241     },
1242     'x-vga' => {
1243         type => 'boolean',
1244         description =>  "Enable vfio-vga device support.",
1245         optional => 1,
1246         default => 0,
1247     },
1248 };
1249 PVE::JSONSchema::register_format('pve-qm-hostpci', $hostpci_fmt);
1250
1251 my $hostpcidesc = {
1252         optional => 1,
1253         type => 'string', format => 'pve-qm-hostpci',
1254         description => "Map host PCI devices into guest.",
1255         verbose_description =>  <<EODESCR,
1256 Map host PCI devices into guest.
1257
1258 NOTE: This option allows direct access to host hardware. So it is no longer
1259 possible to migrate such machines - use with special care.
1260
1261 CAUTION: Experimental! User reported problems with this option.
1262 EODESCR
1263 };
1264 PVE::JSONSchema::register_standard_option("pve-qm-hostpci", $hostpcidesc);
1265
1266 my $serialdesc = {
1267         optional => 1,
1268         type => 'string',
1269         pattern => '(/dev/.+|socket)',
1270         description =>  "Create a serial device inside the VM (n is 0 to 3)",
1271         verbose_description =>  <<EODESCR,
1272 Create a serial device inside the VM (n is 0 to 3), and pass through a
1273 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1274 host side (use 'qm terminal' to open a terminal connection).
1275
1276 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines - use with special care.
1277
1278 CAUTION: Experimental! User reported problems with this option.
1279 EODESCR
1280 };
1281
1282 my $paralleldesc= {
1283         optional => 1,
1284         type => 'string',
1285         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
1286         description =>  "Map host parallel devices (n is 0 to 2).",
1287         verbose_description =>  <<EODESCR,
1288 Map host parallel devices (n is 0 to 2).
1289
1290 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1291
1292 CAUTION: Experimental! User reported problems with this option.
1293 EODESCR
1294 };
1295
1296 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
1297     $confdesc->{"parallel$i"} = $paralleldesc;
1298 }
1299
1300 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
1301     $confdesc->{"serial$i"} = $serialdesc;
1302 }
1303
1304 for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
1305     $confdesc->{"hostpci$i"} = $hostpcidesc;
1306 }
1307
1308 for (my $i = 0; $i < $MAX_IDE_DISKS; $i++)  {
1309     $drivename_hash->{"ide$i"} = 1;
1310     $confdesc->{"ide$i"} = $idedesc;
1311 }
1312
1313 for (my $i = 0; $i < $MAX_SATA_DISKS; $i++)  {
1314     $drivename_hash->{"sata$i"} = 1;
1315     $confdesc->{"sata$i"} = $satadesc;
1316 }
1317
1318 for (my $i = 0; $i < $MAX_SCSI_DISKS; $i++)  {
1319     $drivename_hash->{"scsi$i"} = 1;
1320     $confdesc->{"scsi$i"} = $scsidesc ;
1321 }
1322
1323 for (my $i = 0; $i < $MAX_VIRTIO_DISKS; $i++)  {
1324     $drivename_hash->{"virtio$i"} = 1;
1325     $confdesc->{"virtio$i"} = $virtiodesc;
1326 }
1327
1328 $drivename_hash->{efidisk0} = 1;
1329 $confdesc->{efidisk0} = $efidisk_desc;
1330
1331 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
1332     $confdesc->{"usb$i"} = $usbdesc;
1333 }
1334
1335 my $unuseddesc = {
1336     optional => 1,
1337     type => 'string', format => 'pve-volume-id',
1338     description => "Reference to unused volumes. This is used internally, and should not be modified manually.",
1339 };
1340
1341 for (my $i = 0; $i < $MAX_UNUSED_DISKS; $i++)  {
1342     $confdesc->{"unused$i"} = $unuseddesc;
1343 }
1344
1345 my $kvm_api_version = 0;
1346
1347 sub kvm_version {
1348     return $kvm_api_version if $kvm_api_version;
1349
1350     open my $fh, '<', '/dev/kvm'
1351         or return undef;
1352
1353     # 0xae00 => KVM_GET_API_VERSION
1354     $kvm_api_version = ioctl($fh, 0xae00, 0);
1355
1356     return $kvm_api_version;
1357 }
1358
1359 my $kvm_user_version;
1360
1361 sub kvm_user_version {
1362
1363     return $kvm_user_version if $kvm_user_version;
1364
1365     $kvm_user_version = 'unknown';
1366
1367     my $code = sub {
1368         my $line = shift;
1369         if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1370             $kvm_user_version = $2;
1371         }
1372     };
1373
1374     eval { run_command("kvm -version", outfunc => $code); };
1375     warn $@ if $@;
1376
1377     return $kvm_user_version;
1378
1379 }
1380
1381 my $kernel_has_vhost_net = -c '/dev/vhost-net';
1382
1383 sub valid_drive_names {
1384     # order is important - used to autoselect boot disk
1385     return ((map { "ide$_" } (0 .. ($MAX_IDE_DISKS - 1))),
1386             (map { "scsi$_" } (0 .. ($MAX_SCSI_DISKS - 1))),
1387             (map { "virtio$_" } (0 .. ($MAX_VIRTIO_DISKS - 1))),
1388             (map { "sata$_" } (0 .. ($MAX_SATA_DISKS - 1))),
1389             'efidisk0');
1390 }
1391
1392 sub is_valid_drivename {
1393     my $dev = shift;
1394
1395     return defined($drivename_hash->{$dev});
1396 }
1397
1398 sub option_exists {
1399     my $key = shift;
1400     return defined($confdesc->{$key});
1401 }
1402
1403 sub nic_models {
1404     return $nic_model_list;
1405 }
1406
1407 sub os_list_description {
1408
1409     return {
1410         other => 'Other',
1411         wxp => 'Windows XP',
1412         w2k => 'Windows 2000',
1413         w2k3 =>, 'Windows 2003',
1414         w2k8 => 'Windows 2008',
1415         wvista => 'Windows Vista',
1416         win7 => 'Windows 7',
1417         win8 => 'Windows 8/2012',
1418         win10 => 'Windows 10/2016',
1419         l24 => 'Linux 2.4',
1420         l26 => 'Linux 2.6',
1421     };
1422 }
1423
1424 my $cdrom_path;
1425
1426 sub get_cdrom_path {
1427
1428     return  $cdrom_path if $cdrom_path;
1429
1430     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
1431     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
1432     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
1433 }
1434
1435 sub get_iso_path {
1436     my ($storecfg, $vmid, $cdrom) = @_;
1437
1438     if ($cdrom eq 'cdrom') {
1439         return get_cdrom_path();
1440     } elsif ($cdrom eq 'none') {
1441         return '';
1442     } elsif ($cdrom =~ m|^/|) {
1443         return $cdrom;
1444     } else {
1445         return PVE::Storage::path($storecfg, $cdrom);
1446     }
1447 }
1448
1449 # try to convert old style file names to volume IDs
1450 sub filename_to_volume_id {
1451     my ($vmid, $file, $media) = @_;
1452
1453      if (!($file eq 'none' || $file eq 'cdrom' ||
1454           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1455
1456         return undef if $file =~ m|/|;
1457
1458         if ($media && $media eq 'cdrom') {
1459             $file = "local:iso/$file";
1460         } else {
1461             $file = "local:$vmid/$file";
1462         }
1463     }
1464
1465     return $file;
1466 }
1467
1468 sub verify_media_type {
1469     my ($opt, $vtype, $media) = @_;
1470
1471     return if !$media;
1472
1473     my $etype;
1474     if ($media eq 'disk') {
1475         $etype = 'images';
1476     } elsif ($media eq 'cdrom') {
1477         $etype = 'iso';
1478     } else {
1479         die "internal error";
1480     }
1481
1482     return if ($vtype eq $etype);
1483
1484     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1485 }
1486
1487 sub cleanup_drive_path {
1488     my ($opt, $storecfg, $drive) = @_;
1489
1490     # try to convert filesystem paths to volume IDs
1491
1492     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1493         ($drive->{file} !~ m|^/dev/.+|) &&
1494         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1495         ($drive->{file} !~ m/^\d+$/)) {
1496         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1497         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"}) if !$vtype;
1498         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1499         verify_media_type($opt, $vtype, $drive->{media});
1500         $drive->{file} = $volid;
1501     }
1502
1503     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1504 }
1505
1506 sub parse_hotplug_features {
1507     my ($data) = @_;
1508
1509     my $res = {};
1510
1511     return $res if $data eq '0';
1512
1513     $data = $confdesc->{hotplug}->{default} if $data eq '1';
1514
1515     foreach my $feature (PVE::Tools::split_list($data)) {
1516         if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
1517             $res->{$1} = 1;
1518         } else {
1519             die "invalid hotplug feature '$feature'\n";
1520         }
1521     }
1522     return $res;
1523 }
1524
1525 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1526 sub pve_verify_hotplug_features {
1527     my ($value, $noerr) = @_;
1528
1529     return $value if parse_hotplug_features($value);
1530
1531     return undef if $noerr;
1532
1533     die "unable to parse hotplug option\n";
1534 }
1535
1536 # ideX = [volume=]volume-id[,media=d][,cyls=c,heads=h,secs=s[,trans=t]]
1537 #        [,snapshot=on|off][,cache=on|off][,format=f][,backup=yes|no]
1538 #        [,rerror=ignore|report|stop][,werror=enospc|ignore|report|stop]
1539 #        [,aio=native|threads][,discard=ignore|on][,detect_zeroes=on|off]
1540 #        [,iothread=on][,serial=serial][,model=model]
1541
1542 sub parse_drive {
1543     my ($key, $data) = @_;
1544
1545     my ($interface, $index);
1546
1547     if ($key =~ m/^([^\d]+)(\d+)$/) {
1548         $interface = $1;
1549         $index = $2;
1550     } else {
1551         return undef;
1552     }
1553
1554     my $desc = $key =~ /^unused\d+$/ ? $alldrive_fmt
1555                                      : $confdesc->{$key}->{format};
1556     if (!$desc) {
1557         warn "invalid drive key: $key\n";
1558         return undef;
1559     }
1560     my $res = eval { PVE::JSONSchema::parse_property_string($desc, $data) };
1561     return undef if !$res;
1562     $res->{interface} = $interface;
1563     $res->{index} = $index;
1564
1565     my $error = 0;
1566     foreach my $opt (qw(bps bps_rd bps_wr)) {
1567         if (my $bps = defined(delete $res->{$opt})) {
1568             if (defined($res->{"m$opt"})) {
1569                 warn "both $opt and m$opt specified\n";
1570                 ++$error;
1571                 next;
1572             }
1573             $res->{"m$opt"} = sprintf("%.3f", $bps / (1024*1024.0));
1574         }
1575     }
1576
1577     # can't use the schema's 'requires' because of the mbps* => bps* "transforming aliases"
1578     for my $requirement (
1579         [mbps_max => 'mbps'],
1580         [mbps_rd_max => 'mbps_rd'],
1581         [mbps_wr_max => 'mbps_wr'],
1582         [miops_max => 'miops'],
1583         [miops_rd_max => 'miops_rd'],
1584         [miops_wr_max => 'miops_wr'],
1585         [bps_max_length => 'mbps_max'],
1586         [bps_rd_max_length => 'mbps_rd_max'],
1587         [bps_wr_max_length => 'mbps_wr_max'],
1588         [iops_max_length => 'iops_max'],
1589         [iops_rd_max_length => 'iops_rd_max'],
1590         [iops_wr_max_length => 'iops_wr_max']) {
1591         my ($option, $requires) = @$requirement;
1592         if ($res->{$option} && !$res->{$requires}) {
1593             warn "$option requires $requires\n";
1594             ++$error;
1595         }
1596     }
1597
1598     return undef if $error;
1599
1600     return undef if $res->{mbps_rd} && $res->{mbps};
1601     return undef if $res->{mbps_wr} && $res->{mbps};
1602     return undef if $res->{iops_rd} && $res->{iops};
1603     return undef if $res->{iops_wr} && $res->{iops};
1604
1605     if ($res->{media} && ($res->{media} eq 'cdrom')) {
1606         return undef if $res->{snapshot} || $res->{trans} || $res->{format};
1607         return undef if $res->{heads} || $res->{secs} || $res->{cyls};
1608         return undef if $res->{interface} eq 'virtio';
1609     }
1610
1611     if (my $size = $res->{size}) {
1612         return undef if !defined($res->{size} = PVE::JSONSchema::parse_size($size));
1613     }
1614
1615     return $res;
1616 }
1617
1618 sub print_drive {
1619     my ($vmid, $drive) = @_;
1620     my $data = { %$drive };
1621     delete $data->{$_} for qw(index interface);
1622     return PVE::JSONSchema::print_property_string($data, $alldrive_fmt);
1623 }
1624
1625 sub scsi_inquiry {
1626     my($fh, $noerr) = @_;
1627
1628     my $SG_IO = 0x2285;
1629     my $SG_GET_VERSION_NUM = 0x2282;
1630
1631     my $versionbuf = "\x00" x 8;
1632     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1633     if (!$ret) {
1634         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1635         return undef;
1636     }
1637     my $version = unpack("I", $versionbuf);
1638     if ($version < 30000) {
1639         die "scsi generic interface too old\n"  if !$noerr;
1640         return undef;
1641     }
1642
1643     my $buf = "\x00" x 36;
1644     my $sensebuf = "\x00" x 8;
1645     my $cmd = pack("C x3 C x1", 0x12, 36);
1646
1647     # see /usr/include/scsi/sg.h
1648     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1649
1650     my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1651                       length($sensebuf), 0, length($buf), $buf,
1652                       $cmd, $sensebuf, 6000);
1653
1654     $ret = ioctl($fh, $SG_IO, $packet);
1655     if (!$ret) {
1656         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1657         return undef;
1658     }
1659
1660     my @res = unpack($sg_io_hdr_t, $packet);
1661     if ($res[17] || $res[18]) {
1662         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1663         return undef;
1664     }
1665
1666     my $res = {};
1667     (my $byte0, my $byte1, $res->{vendor},
1668      $res->{product}, $res->{revision}) = unpack("C C x6 A8 A16 A4", $buf);
1669
1670     $res->{removable} = $byte1 & 128 ? 1 : 0;
1671     $res->{type} = $byte0 & 31;
1672
1673     return $res;
1674 }
1675
1676 sub path_is_scsi {
1677     my ($path) = @_;
1678
1679     my $fh = IO::File->new("+<$path") || return undef;
1680     my $res = scsi_inquiry($fh, 1);
1681     close($fh);
1682
1683     return $res;
1684 }
1685
1686 sub machine_type_is_q35 {
1687     my ($conf) = @_;
1688
1689     return $conf->{machine} && ($conf->{machine} =~ m/q35/) ? 1 : 0;
1690 }
1691
1692 sub print_tabletdevice_full {
1693     my ($conf) = @_;
1694
1695     my $q35 = machine_type_is_q35($conf);
1696
1697     # we use uhci for old VMs because tablet driver was buggy in older qemu
1698     my $usbbus = $q35 ? "ehci" : "uhci";
1699
1700     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1701 }
1702
1703 sub print_drivedevice_full {
1704     my ($storecfg, $conf, $vmid, $drive, $bridges) = @_;
1705
1706     my $device = '';
1707     my $maxdev = 0;
1708
1709     if ($drive->{interface} eq 'virtio') {
1710         my $pciaddr = print_pci_addr("$drive->{interface}$drive->{index}", $bridges);
1711         $device = "virtio-blk-pci,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}$pciaddr";
1712         $device .= ",iothread=iothread-$drive->{interface}$drive->{index}" if $drive->{iothread};
1713     } elsif ($drive->{interface} eq 'scsi') {
1714
1715         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1716         my $unit = $drive->{index} % $maxdev;
1717         my $devicetype = 'hd';
1718         my $path = '';
1719         if (drive_is_cdrom($drive)) {
1720             $devicetype = 'cd';
1721         } else {
1722             if ($drive->{file} =~ m|^/|) {
1723                 $path = $drive->{file};
1724                 if (my $info = path_is_scsi($path)) {
1725                     if ($info->{type} == 0 && $drive->{scsiblock}) {
1726                         $devicetype = 'block';
1727                     } elsif ($info->{type} == 1) { # tape
1728                         $devicetype = 'generic';
1729                     }
1730                 }
1731             } else {
1732                  $path = PVE::Storage::path($storecfg, $drive->{file});
1733             }
1734
1735             if($path =~ m/^iscsi\:\/\//){
1736                 $devicetype = 'generic';
1737             }
1738         }
1739
1740         if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)){
1741             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1742         } else {
1743             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0,lun=$drive->{index},drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1744         }
1745
1746         if ($drive->{ssd} && ($devicetype eq 'block' || $devicetype eq 'hd')) {
1747             $device .= ",rotation_rate=1";
1748         }
1749
1750     } elsif ($drive->{interface} eq 'ide' || $drive->{interface} eq 'sata') {
1751         my $maxdev = ($drive->{interface} eq 'sata') ? $MAX_SATA_DISKS : 2;
1752         my $controller = int($drive->{index} / $maxdev);
1753         my $unit = $drive->{index} % $maxdev;
1754         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1755
1756         $device = "ide-$devicetype";
1757         if ($drive->{interface} eq 'ide') {
1758             $device .= ",bus=ide.$controller,unit=$unit";
1759         } else {
1760             $device .= ",bus=ahci$controller.$unit";
1761         }
1762         $device .= ",drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1763
1764         if ($devicetype eq 'hd') {
1765             if (my $model = $drive->{model}) {
1766                 $model = URI::Escape::uri_unescape($model);
1767                 $device .= ",model=$model";
1768             }
1769             if ($drive->{ssd}) {
1770                 $device .= ",rotation_rate=1";
1771             }
1772         }
1773     } elsif ($drive->{interface} eq 'usb') {
1774         die "implement me";
1775         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1776     } else {
1777         die "unsupported interface type";
1778     }
1779
1780     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1781
1782     if (my $serial = $drive->{serial}) {
1783         $serial = URI::Escape::uri_unescape($serial);
1784         $device .= ",serial=$serial";
1785     }
1786
1787
1788     return $device;
1789 }
1790
1791 sub get_initiator_name {
1792     my $initiator;
1793
1794     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return undef;
1795     while (defined(my $line = <$fh>)) {
1796         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1797         $initiator = $1;
1798         last;
1799     }
1800     $fh->close();
1801
1802     return $initiator;
1803 }
1804
1805 sub print_drive_full {
1806     my ($storecfg, $vmid, $drive) = @_;
1807
1808     my $path;
1809     my $volid = $drive->{file};
1810     my $format;
1811
1812     if (drive_is_cdrom($drive)) {
1813         $path = get_iso_path($storecfg, $vmid, $volid);
1814     } else {
1815         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1816         if ($storeid) {
1817             $path = PVE::Storage::path($storecfg, $volid);
1818             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
1819             $format = qemu_img_format($scfg, $volname);
1820         } else {
1821             $path = $volid;
1822             $format = "raw";
1823         }
1824    }
1825
1826     my $opts = '';
1827     my @qemu_drive_options = qw(heads secs cyls trans media format cache rerror werror aio discard);
1828     foreach my $o (@qemu_drive_options) {
1829         $opts .= ",$o=$drive->{$o}" if defined($drive->{$o});
1830     }
1831
1832     # snapshot only accepts on|off
1833     if (defined($drive->{snapshot})) {
1834         my $v = $drive->{snapshot} ? 'on' : 'off';
1835         $opts .= ",snapshot=$v";
1836     }
1837
1838     foreach my $type (['', '-total'], [_rd => '-read'], [_wr => '-write']) {
1839         my ($dir, $qmpname) = @$type;
1840         if (my $v = $drive->{"mbps$dir"}) {
1841             $opts .= ",throttling.bps$qmpname=".int($v*1024*1024);
1842         }
1843         if (my $v = $drive->{"mbps${dir}_max"}) {
1844             $opts .= ",throttling.bps$qmpname-max=".int($v*1024*1024);
1845         }
1846         if (my $v = $drive->{"bps${dir}_max_length"}) {
1847             $opts .= ",throttling.bps$qmpname-max-length=$v";
1848         }
1849         if (my $v = $drive->{"iops${dir}"}) {
1850             $opts .= ",throttling.iops$qmpname=$v";
1851         }
1852         if (my $v = $drive->{"iops${dir}_max"}) {
1853             $opts .= ",throttling.iops$qmpname-max=$v";
1854         }
1855         if (my $v = $drive->{"iops${dir}_max_length"}) {
1856             $opts .= ",throttling.iops$qmpname-max-length=$v";
1857         }
1858     }
1859
1860     $opts .= ",format=$format" if $format && !$drive->{format};
1861
1862     my $cache_direct = 0;
1863
1864     if (my $cache = $drive->{cache}) {
1865         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1866     } elsif (!drive_is_cdrom($drive)) {
1867         $opts .= ",cache=none";
1868         $cache_direct = 1;
1869     }
1870
1871     # aio native works only with O_DIRECT
1872     if (!$drive->{aio}) {
1873         if($cache_direct) {
1874             $opts .= ",aio=native";
1875         } else {
1876             $opts .= ",aio=threads";
1877         }
1878     }
1879
1880     if (!drive_is_cdrom($drive)) {
1881         my $detectzeroes;
1882         if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
1883             $detectzeroes = 'off';
1884         } elsif ($drive->{discard}) {
1885             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
1886         } else {
1887             # This used to be our default with discard not being specified:
1888             $detectzeroes = 'on';
1889         }
1890         $opts .= ",detect-zeroes=$detectzeroes" if $detectzeroes;
1891     }
1892
1893     my $pathinfo = $path ? "file=$path," : '';
1894
1895     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1896 }
1897
1898 sub print_netdevice_full {
1899     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files) = @_;
1900
1901     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
1902
1903     my $device = $net->{model};
1904     if ($net->{model} eq 'virtio') {
1905          $device = 'virtio-net-pci';
1906      };
1907
1908     my $pciaddr = print_pci_addr("$netid", $bridges);
1909     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1910     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
1911         #Consider we have N queues, the number of vectors needed is 2*N + 2 (plus one config interrupt and control vq)
1912         my $vectors = $net->{queues} * 2 + 2;
1913         $tmpstr .= ",vectors=$vectors,mq=on";
1914     }
1915     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
1916
1917     if ($use_old_bios_files) {
1918         my $romfile;
1919         if ($device eq 'virtio-net-pci') {
1920             $romfile = 'pxe-virtio.rom';
1921         } elsif ($device eq 'e1000') {
1922             $romfile = 'pxe-e1000.rom';
1923         } elsif ($device eq 'ne2k') {
1924             $romfile = 'pxe-ne2k_pci.rom';
1925         } elsif ($device eq 'pcnet') {
1926             $romfile = 'pxe-pcnet.rom';
1927         } elsif ($device eq 'rtl8139') {
1928             $romfile = 'pxe-rtl8139.rom';
1929         }
1930         $tmpstr .= ",romfile=$romfile" if $romfile;
1931     }
1932
1933     return $tmpstr;
1934 }
1935
1936 sub print_netdev_full {
1937     my ($vmid, $conf, $net, $netid, $hotplug) = @_;
1938
1939     my $i = '';
1940     if ($netid =~ m/^net(\d+)$/) {
1941         $i = int($1);
1942     }
1943
1944     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
1945
1946     my $ifname = "tap${vmid}i$i";
1947
1948     # kvm uses TUNSETIFF ioctl, and that limits ifname length
1949     die "interface name '$ifname' is too long (max 15 character)\n"
1950         if length($ifname) >= 16;
1951
1952     my $vhostparam = '';
1953     $vhostparam = ',vhost=on' if $kernel_has_vhost_net && $net->{model} eq 'virtio';
1954
1955     my $vmname = $conf->{name} || "vm$vmid";
1956
1957     my $netdev = "";
1958     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
1959
1960     if ($net->{bridge}) {
1961         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script,downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1962     } else {
1963         $netdev = "type=user,id=$netid,hostname=$vmname";
1964     }
1965
1966     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
1967
1968     return $netdev;
1969 }
1970
1971
1972 sub print_cpu_device {
1973     my ($conf, $id) = @_;
1974
1975     my $kvm = $conf->{kvm} // 1;
1976     my $cpu = $kvm ? "kvm64" : "qemu64";
1977     if (my $cputype = $conf->{cpu}) {
1978         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
1979             or die "Cannot parse cpu description: $cputype\n";
1980         $cpu = $cpuconf->{cputype};
1981     }
1982
1983     my $cores = $conf->{cores} || 1;
1984
1985     my $current_core = ($id - 1) % $cores;
1986     my $current_socket = int(($id - 1 - $current_core)/$cores);
1987
1988     return "$cpu-x86_64-cpu,id=cpu$id,socket-id=$current_socket,core-id=$current_core,thread-id=0";
1989 }
1990
1991 my $vga_map = {
1992     'cirrus' => 'cirrus-vga',
1993     'std' => 'VGA',
1994     'vmware' => 'vmware-svga',
1995     'virtio' => 'virtio-vga',
1996 };
1997
1998 sub print_vga_device {
1999     my ($conf, $vga, $id, $qxlnum, $bridges) = @_;
2000
2001     my $type = $vga_map->{$vga->{type}};
2002     my $vgamem_mb = $vga->{memory};
2003     if ($qxlnum) {
2004         $type = $id ? 'qxl' : 'qxl-vga';
2005     }
2006     die "no devicetype for $vga->{type}\n" if !$type;
2007
2008     my $memory = "";
2009     if ($vgamem_mb) {
2010         if ($vga->{type} eq 'virtio') {
2011             my $bytes = PVE::Tools::convert_size($vgamem_mb, "mb" => "b");
2012             $memory = ",max_hostmem=$bytes";
2013         } elsif ($qxlnum) {
2014             # from https://www.spice-space.org/multiple-monitors.html
2015             $memory = ",vgamem_mb=$vga->{memory}";
2016             my $ram = $vgamem_mb * 4;
2017             my $vram = $vgamem_mb * 2;
2018             $memory .= ",ram_size_mb=$ram,vram_size_mb=$vram";
2019         } else {
2020             $memory = ",vgamem_mb=$vga->{memory}";
2021         }
2022     } elsif ($qxlnum && $id) {
2023         $memory = ",ram_size=67108864,vram_size=33554432";
2024     }
2025
2026     my $q35 = machine_type_is_q35($conf);
2027     my $vgaid = "vga" . ($id // '');
2028     my $pciaddr;
2029
2030     if ($q35 && $vgaid eq 'vga') {
2031         # the first display uses pcie.0 bus on q35 machines
2032         $pciaddr = print_pcie_addr($vgaid, $bridges);
2033     } else {
2034         $pciaddr = print_pci_addr($vgaid, $bridges);
2035     }
2036
2037     return "$type,id=${vgaid}${memory}${pciaddr}";
2038 }
2039
2040 sub drive_is_cloudinit {
2041     my ($drive) = @_;
2042     return $drive->{file} =~ m@[:/]vm-\d+-cloudinit(?:\.$QEMU_FORMAT_RE)?$@;
2043 }
2044
2045 sub drive_is_cdrom {
2046     my ($drive, $exclude_cloudinit) = @_;
2047
2048     return 0 if $exclude_cloudinit && drive_is_cloudinit($drive);
2049
2050     return $drive && $drive->{media} && ($drive->{media} eq 'cdrom');
2051
2052 }
2053
2054 sub parse_number_sets {
2055     my ($set) = @_;
2056     my $res = [];
2057     foreach my $part (split(/;/, $set)) {
2058         if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
2059             die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
2060             push @$res, [ $1, $2 ];
2061         } else {
2062             die "invalid range: $part\n";
2063         }
2064     }
2065     return $res;
2066 }
2067
2068 sub parse_numa {
2069     my ($data) = @_;
2070
2071     my $res = PVE::JSONSchema::parse_property_string($numa_fmt, $data);
2072     $res->{cpus} = parse_number_sets($res->{cpus}) if defined($res->{cpus});
2073     $res->{hostnodes} = parse_number_sets($res->{hostnodes}) if defined($res->{hostnodes});
2074     return $res;
2075 }
2076
2077 sub parse_hostpci {
2078     my ($value) = @_;
2079
2080     return undef if !$value;
2081
2082     my $res = PVE::JSONSchema::parse_property_string($hostpci_fmt, $value);
2083
2084     my @idlist = split(/;/, $res->{host});
2085     delete $res->{host};
2086     foreach my $id (@idlist) {
2087         if ($id =~ /^$PCIRE$/) {
2088             if (defined($2)) {
2089                 push @{$res->{pciid}}, { id => $1, function => $2 };
2090             } else {
2091                 my $pcidevices = lspci($1);
2092                 $res->{pciid} = $pcidevices->{$1};
2093             }
2094         } else {
2095             # should have been caught by parse_property_string already
2096             die "failed to parse PCI id: $id\n";
2097         }
2098     }
2099     return $res;
2100 }
2101
2102 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
2103 sub parse_net {
2104     my ($data) = @_;
2105
2106     my $res = eval { PVE::JSONSchema::parse_property_string($net_fmt, $data) };
2107     if ($@) {
2108         warn $@;
2109         return undef;
2110     }
2111     if (!defined($res->{macaddr})) {
2112         my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
2113         $res->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix});
2114     }
2115     return $res;
2116 }
2117
2118 # ipconfigX ip=cidr,gw=ip,ip6=cidr,gw6=ip
2119 sub parse_ipconfig {
2120     my ($data) = @_;
2121
2122     my $res = eval { PVE::JSONSchema::parse_property_string($ipconfig_fmt, $data) };
2123     if ($@) {
2124         warn $@;
2125         return undef;
2126     }
2127
2128     if ($res->{gw} && !$res->{ip}) {
2129         warn 'gateway specified without specifying an IP address';
2130         return undef;
2131     }
2132     if ($res->{gw6} && !$res->{ip6}) {
2133         warn 'IPv6 gateway specified without specifying an IPv6 address';
2134         return undef;
2135     }
2136     if ($res->{gw} && $res->{ip} eq 'dhcp') {
2137         warn 'gateway specified together with DHCP';
2138         return undef;
2139     }
2140     if ($res->{gw6} && $res->{ip6} !~ /^$IPV6RE/) {
2141         # gw6 + auto/dhcp
2142         warn "IPv6 gateway specified together with $res->{ip6} address";
2143         return undef;
2144     }
2145
2146     if (!$res->{ip} && !$res->{ip6}) {
2147         return { ip => 'dhcp', ip6 => 'dhcp' };
2148     }
2149
2150     return $res;
2151 }
2152
2153 sub print_net {
2154     my $net = shift;
2155
2156     return PVE::JSONSchema::print_property_string($net, $net_fmt);
2157 }
2158
2159 sub add_random_macs {
2160     my ($settings) = @_;
2161
2162     foreach my $opt (keys %$settings) {
2163         next if $opt !~ m/^net(\d+)$/;
2164         my $net = parse_net($settings->{$opt});
2165         next if !$net;
2166         $settings->{$opt} = print_net($net);
2167     }
2168 }
2169
2170 sub vm_is_volid_owner {
2171     my ($storecfg, $vmid, $volid) = @_;
2172
2173     if ($volid !~  m|^/|) {
2174         my ($path, $owner);
2175         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
2176         if ($owner && ($owner == $vmid)) {
2177             return 1;
2178         }
2179     }
2180
2181     return undef;
2182 }
2183
2184 sub split_flagged_list {
2185     my $text = shift || '';
2186     $text =~ s/[,;]/ /g;
2187     $text =~ s/^\s+//;
2188     return { map { /^(!?)(.*)$/ && ($2, $1) } ($text =~ /\S+/g) };
2189 }
2190
2191 sub join_flagged_list {
2192     my ($how, $lst) = @_;
2193     join $how, map { $lst->{$_} . $_ } keys %$lst;
2194 }
2195
2196 sub vmconfig_delete_pending_option {
2197     my ($conf, $key, $force) = @_;
2198
2199     delete $conf->{pending}->{$key};
2200     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
2201     $pending_delete_hash->{$key} = $force ? '!' : '';
2202     $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
2203 }
2204
2205 sub vmconfig_undelete_pending_option {
2206     my ($conf, $key) = @_;
2207
2208     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
2209     delete $pending_delete_hash->{$key};
2210
2211     if (%$pending_delete_hash) {
2212         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
2213     } else {
2214         delete $conf->{pending}->{delete};
2215     }
2216 }
2217
2218 sub vmconfig_register_unused_drive {
2219     my ($storecfg, $vmid, $conf, $drive) = @_;
2220
2221     if (drive_is_cloudinit($drive)) {
2222         eval { PVE::Storage::vdisk_free($storecfg, $drive->{file}) };
2223         warn $@ if $@;
2224     } elsif (!drive_is_cdrom($drive)) {
2225         my $volid = $drive->{file};
2226         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
2227             PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
2228         }
2229     }
2230 }
2231
2232 sub vmconfig_cleanup_pending {
2233     my ($conf) = @_;
2234
2235     # remove pending changes when nothing changed
2236     my $changes;
2237     foreach my $opt (keys %{$conf->{pending}}) {
2238         if (defined($conf->{$opt}) && ($conf->{pending}->{$opt} eq  $conf->{$opt})) {
2239             $changes = 1;
2240             delete $conf->{pending}->{$opt};
2241         }
2242     }
2243
2244     my $current_delete_hash = split_flagged_list($conf->{pending}->{delete});
2245     my $pending_delete_hash = {};
2246     while (my ($opt, $force) = each %$current_delete_hash) {
2247         if (defined($conf->{$opt})) {
2248             $pending_delete_hash->{$opt} = $force;
2249         } else {
2250             $changes = 1;
2251         }
2252     }
2253
2254     if (%$pending_delete_hash) {
2255         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
2256     } else {
2257         delete $conf->{pending}->{delete};
2258     }
2259
2260     return $changes;
2261 }
2262
2263 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str]
2264 my $smbios1_fmt = {
2265     uuid => {
2266         type => 'string',
2267         pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
2268         format_description => 'UUID',
2269         description => "Set SMBIOS1 UUID.",
2270         optional => 1,
2271     },
2272     version => {
2273         type => 'string',
2274         pattern => '\S+',
2275         format_description => 'string',
2276         description => "Set SMBIOS1 version.",
2277         optional => 1,
2278     },
2279     serial => {
2280         type => 'string',
2281         pattern => '\S+',
2282         format_description => 'string',
2283         description => "Set SMBIOS1 serial number.",
2284         optional => 1,
2285     },
2286     manufacturer => {
2287         type => 'string',
2288         pattern => '\S+',
2289         format_description => 'string',
2290         description => "Set SMBIOS1 manufacturer.",
2291         optional => 1,
2292     },
2293     product => {
2294         type => 'string',
2295         pattern => '\S+',
2296         format_description => 'string',
2297         description => "Set SMBIOS1 product ID.",
2298         optional => 1,
2299     },
2300     sku => {
2301         type => 'string',
2302         pattern => '\S+',
2303         format_description => 'string',
2304         description => "Set SMBIOS1 SKU string.",
2305         optional => 1,
2306     },
2307     family => {
2308         type => 'string',
2309         pattern => '\S+',
2310         format_description => 'string',
2311         description => "Set SMBIOS1 family string.",
2312         optional => 1,
2313     },
2314 };
2315
2316 sub parse_smbios1 {
2317     my ($data) = @_;
2318
2319     my $res = eval { PVE::JSONSchema::parse_property_string($smbios1_fmt, $data) };
2320     warn $@ if $@;
2321     return $res;
2322 }
2323
2324 sub print_smbios1 {
2325     my ($smbios1) = @_;
2326     return PVE::JSONSchema::print_property_string($smbios1, $smbios1_fmt);
2327 }
2328
2329 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_fmt);
2330
2331 PVE::JSONSchema::register_format('pve-qm-bootdisk', \&verify_bootdisk);
2332 sub verify_bootdisk {
2333     my ($value, $noerr) = @_;
2334
2335     return $value if is_valid_drivename($value);
2336
2337     return undef if $noerr;
2338
2339     die "invalid boot disk '$value'\n";
2340 }
2341
2342 sub parse_watchdog {
2343     my ($value) = @_;
2344
2345     return undef if !$value;
2346
2347     my $res = eval { PVE::JSONSchema::parse_property_string($watchdog_fmt, $value) };
2348     warn $@ if $@;
2349     return $res;
2350 }
2351
2352 sub parse_guest_agent {
2353     my ($value) = @_;
2354
2355     return {} if !defined($value->{agent});
2356
2357     my $res = eval { PVE::JSONSchema::parse_property_string($agent_fmt, $value->{agent}) };
2358     warn $@ if $@;
2359
2360     # if the agent is disabled ignore the other potentially set properties
2361     return {} if !$res->{enabled};
2362     return $res;
2363 }
2364
2365 sub parse_vga {
2366     my ($value) = @_;
2367
2368     return {} if !$value;
2369     my $res = eval { PVE::JSONSchema::parse_property_string($vga_fmt, $value) };
2370     warn $@ if $@;
2371     return $res;
2372 }
2373
2374 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
2375 sub verify_usb_device {
2376     my ($value, $noerr) = @_;
2377
2378     return $value if parse_usb_device($value);
2379
2380     return undef if $noerr;
2381
2382     die "unable to parse usb device\n";
2383 }
2384
2385 # add JSON properties for create and set function
2386 sub json_config_properties {
2387     my $prop = shift;
2388
2389     foreach my $opt (keys %$confdesc) {
2390         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate' || $opt eq 'runningmachine';
2391         $prop->{$opt} = $confdesc->{$opt};
2392     }
2393
2394     return $prop;
2395 }
2396
2397 # return copy of $confdesc_cloudinit to generate documentation
2398 sub cloudinit_config_properties {
2399
2400     return dclone($confdesc_cloudinit);
2401 }
2402
2403 sub check_type {
2404     my ($key, $value) = @_;
2405
2406     die "unknown setting '$key'\n" if !$confdesc->{$key};
2407
2408     my $type = $confdesc->{$key}->{type};
2409
2410     if (!defined($value)) {
2411         die "got undefined value\n";
2412     }
2413
2414     if ($value =~ m/[\n\r]/) {
2415         die "property contains a line feed\n";
2416     }
2417
2418     if ($type eq 'boolean') {
2419         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
2420         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
2421         die "type check ('boolean') failed - got '$value'\n";
2422     } elsif ($type eq 'integer') {
2423         return int($1) if $value =~ m/^(\d+)$/;
2424         die "type check ('integer') failed - got '$value'\n";
2425     } elsif ($type eq 'number') {
2426         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
2427         die "type check ('number') failed - got '$value'\n";
2428     } elsif ($type eq 'string') {
2429         if (my $fmt = $confdesc->{$key}->{format}) {
2430             PVE::JSONSchema::check_format($fmt, $value);
2431             return $value;
2432         }
2433         $value =~ s/^\"(.*)\"$/$1/;
2434         return $value;
2435     } else {
2436         die "internal error"
2437     }
2438 }
2439
2440 sub check_iommu_support{
2441     #fixme : need to check IOMMU support
2442     #http://www.linux-kvm.org/page/How_to_assign_devices_with_VT-d_in_KVM
2443
2444     my $iommu=1;
2445     return $iommu;
2446
2447 }
2448
2449 sub touch_config {
2450     my ($vmid) = @_;
2451
2452     my $conf = PVE::QemuConfig->config_file($vmid);
2453     utime undef, undef, $conf;
2454 }
2455
2456 sub destroy_vm {
2457     my ($storecfg, $vmid, $keep_empty_config, $skiplock) = @_;
2458
2459     my $conffile = PVE::QemuConfig->config_file($vmid);
2460
2461     my $conf = PVE::QemuConfig->load_config($vmid);
2462
2463     PVE::QemuConfig->check_lock($conf) if !$skiplock;
2464
2465     if ($conf->{template}) {
2466         # check if any base image is still used by a linked clone
2467         foreach_drive($conf, sub {
2468                 my ($ds, $drive) = @_;
2469
2470                 return if drive_is_cdrom($drive);
2471
2472                 my $volid = $drive->{file};
2473
2474                 return if !$volid || $volid =~ m|^/|;
2475
2476                 die "base volume '$volid' is still in use by linked cloned\n"
2477                     if PVE::Storage::volume_is_base_and_used($storecfg, $volid);
2478
2479         });
2480     }
2481
2482     # only remove disks owned by this VM
2483     foreach_drive($conf, sub {
2484         my ($ds, $drive) = @_;
2485
2486         return if drive_is_cdrom($drive, 1);
2487
2488         my $volid = $drive->{file};
2489
2490         return if !$volid || $volid =~ m|^/|;
2491
2492         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
2493         return if !$path || !$owner || ($owner != $vmid);
2494
2495         eval {
2496             PVE::Storage::vdisk_free($storecfg, $volid);
2497         };
2498         warn "Could not remove disk '$volid', check manually: $@" if $@;
2499
2500     });
2501
2502     if ($keep_empty_config) {
2503         PVE::Tools::file_set_contents($conffile, "memory: 128\n");
2504     } else {
2505         unlink $conffile;
2506     }
2507
2508     # also remove unused disk
2509     eval {
2510         my $dl = PVE::Storage::vdisk_list($storecfg, undef, $vmid);
2511
2512         eval {
2513             PVE::Storage::foreach_volid($dl, sub {
2514                 my ($volid, $sid, $volname, $d) = @_;
2515                 PVE::Storage::vdisk_free($storecfg, $volid);
2516             });
2517         };
2518         warn $@ if $@;
2519
2520     };
2521     warn $@ if $@;
2522 }
2523
2524 sub parse_vm_config {
2525     my ($filename, $raw) = @_;
2526
2527     return undef if !defined($raw);
2528
2529     my $res = {
2530         digest => Digest::SHA::sha1_hex($raw),
2531         snapshots => {},
2532         pending => {},
2533     };
2534
2535     $filename =~ m|/qemu-server/(\d+)\.conf$|
2536         || die "got strange filename '$filename'";
2537
2538     my $vmid = $1;
2539
2540     my $conf = $res;
2541     my $descr;
2542     my $section = '';
2543
2544     my @lines = split(/\n/, $raw);
2545     foreach my $line (@lines) {
2546         next if $line =~ m/^\s*$/;
2547
2548         if ($line =~ m/^\[PENDING\]\s*$/i) {
2549             $section = 'pending';
2550             if (defined($descr)) {
2551                 $descr =~ s/\s+$//;
2552                 $conf->{description} = $descr;
2553             }
2554             $descr = undef;
2555             $conf = $res->{$section} = {};
2556             next;
2557
2558         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2559             $section = $1;
2560             if (defined($descr)) {
2561                 $descr =~ s/\s+$//;
2562                 $conf->{description} = $descr;
2563             }
2564             $descr = undef;
2565             $conf = $res->{snapshots}->{$section} = {};
2566             next;
2567         }
2568
2569         if ($line =~ m/^\#(.*)\s*$/) {
2570             $descr = '' if !defined($descr);
2571             $descr .= PVE::Tools::decode_text($1) . "\n";
2572             next;
2573         }
2574
2575         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2576             $descr = '' if !defined($descr);
2577             $descr .= PVE::Tools::decode_text($2);
2578         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2579             $conf->{snapstate} = $1;
2580         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2581             my $key = $1;
2582             my $value = $2;
2583             $conf->{$key} = $value;
2584         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2585             my $value = $1;
2586             if ($section eq 'pending') {
2587                 $conf->{delete} = $value; # we parse this later
2588             } else {
2589                 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
2590             }
2591         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(.+?)\s*$/) {
2592             my $key = $1;
2593             my $value = $2;
2594             eval { $value = check_type($key, $value); };
2595             if ($@) {
2596                 warn "vm $vmid - unable to parse value of '$key' - $@";
2597             } else {
2598                 $key = 'ide2' if $key eq 'cdrom';
2599                 my $fmt = $confdesc->{$key}->{format};
2600                 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2601                     my $v = parse_drive($key, $value);
2602                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2603                         $v->{file} = $volid;
2604                         $value = print_drive($vmid, $v);
2605                     } else {
2606                         warn "vm $vmid - unable to parse value of '$key'\n";
2607                         next;
2608                     }
2609                 }
2610
2611                 $conf->{$key} = $value;
2612             }
2613         }
2614     }
2615
2616     if (defined($descr)) {
2617         $descr =~ s/\s+$//;
2618         $conf->{description} = $descr;
2619     }
2620     delete $res->{snapstate}; # just to be sure
2621
2622     return $res;
2623 }
2624
2625 sub write_vm_config {
2626     my ($filename, $conf) = @_;
2627
2628     delete $conf->{snapstate}; # just to be sure
2629
2630     if ($conf->{cdrom}) {
2631         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2632         $conf->{ide2} = $conf->{cdrom};
2633         delete $conf->{cdrom};
2634     }
2635
2636     # we do not use 'smp' any longer
2637     if ($conf->{sockets}) {
2638         delete $conf->{smp};
2639     } elsif ($conf->{smp}) {
2640         $conf->{sockets} = $conf->{smp};
2641         delete $conf->{cores};
2642         delete $conf->{smp};
2643     }
2644
2645     my $used_volids = {};
2646
2647     my $cleanup_config = sub {
2648         my ($cref, $pending, $snapname) = @_;
2649
2650         foreach my $key (keys %$cref) {
2651             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2652                 $key eq 'snapstate' || $key eq 'pending';
2653             my $value = $cref->{$key};
2654             if ($key eq 'delete') {
2655                 die "propertry 'delete' is only allowed in [PENDING]\n"
2656                     if !$pending;
2657                 # fixme: check syntax?
2658                 next;
2659             }
2660             eval { $value = check_type($key, $value); };
2661             die "unable to parse value of '$key' - $@" if $@;
2662
2663             $cref->{$key} = $value;
2664
2665             if (!$snapname && is_valid_drivename($key)) {
2666                 my $drive = parse_drive($key, $value);
2667                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2668             }
2669         }
2670     };
2671
2672     &$cleanup_config($conf);
2673
2674     &$cleanup_config($conf->{pending}, 1);
2675
2676     foreach my $snapname (keys %{$conf->{snapshots}}) {
2677         die "internal error" if $snapname eq 'pending';
2678         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2679     }
2680
2681     # remove 'unusedX' settings if we re-add a volume
2682     foreach my $key (keys %$conf) {
2683         my $value = $conf->{$key};
2684         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2685             delete $conf->{$key};
2686         }
2687     }
2688
2689     my $generate_raw_config = sub {
2690         my ($conf, $pending) = @_;
2691
2692         my $raw = '';
2693
2694         # add description as comment to top of file
2695         if (defined(my $descr = $conf->{description})) {
2696             if ($descr) {
2697                 foreach my $cl (split(/\n/, $descr)) {
2698                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2699                 }
2700             } else {
2701                 $raw .= "#\n" if $pending;
2702             }
2703         }
2704
2705         foreach my $key (sort keys %$conf) {
2706             next if $key eq 'digest' || $key eq 'description' || $key eq 'pending' || $key eq 'snapshots';
2707             $raw .= "$key: $conf->{$key}\n";
2708         }
2709         return $raw;
2710     };
2711
2712     my $raw = &$generate_raw_config($conf);
2713
2714     if (scalar(keys %{$conf->{pending}})){
2715         $raw .= "\n[PENDING]\n";
2716         $raw .= &$generate_raw_config($conf->{pending}, 1);
2717     }
2718
2719     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2720         $raw .= "\n[$snapname]\n";
2721         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2722     }
2723
2724     return $raw;
2725 }
2726
2727 sub load_defaults {
2728
2729     my $res = {};
2730
2731     # we use static defaults from our JSON schema configuration
2732     foreach my $key (keys %$confdesc) {
2733         if (defined(my $default = $confdesc->{$key}->{default})) {
2734             $res->{$key} = $default;
2735         }
2736     }
2737
2738     return $res;
2739 }
2740
2741 sub config_list {
2742     my $vmlist = PVE::Cluster::get_vmlist();
2743     my $res = {};
2744     return $res if !$vmlist || !$vmlist->{ids};
2745     my $ids = $vmlist->{ids};
2746
2747     foreach my $vmid (keys %$ids) {
2748         my $d = $ids->{$vmid};
2749         next if !$d->{node} || $d->{node} ne $nodename;
2750         next if !$d->{type} || $d->{type} ne 'qemu';
2751         $res->{$vmid}->{exists} = 1;
2752     }
2753     return $res;
2754 }
2755
2756 # test if VM uses local resources (to prevent migration)
2757 sub check_local_resources {
2758     my ($conf, $noerr) = @_;
2759
2760     my $loc_res = 0;
2761
2762     $loc_res = 1 if $conf->{hostusb}; # old syntax
2763     $loc_res = 1 if $conf->{hostpci}; # old syntax
2764
2765     foreach my $k (keys %$conf) {
2766         next if $k =~ m/^usb/ && ($conf->{$k} eq 'spice');
2767         # sockets are safe: they will recreated be on the target side post-migrate
2768         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2769         $loc_res = 1 if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2770     }
2771
2772     die "VM uses local resources\n" if $loc_res && !$noerr;
2773
2774     return $loc_res;
2775 }
2776
2777 # check if used storages are available on all nodes (use by migrate)
2778 sub check_storage_availability {
2779     my ($storecfg, $conf, $node) = @_;
2780
2781     foreach_drive($conf, sub {
2782         my ($ds, $drive) = @_;
2783
2784         my $volid = $drive->{file};
2785         return if !$volid;
2786
2787         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2788         return if !$sid;
2789
2790         # check if storage is available on both nodes
2791         my $scfg = PVE::Storage::storage_check_node($storecfg, $sid);
2792         PVE::Storage::storage_check_node($storecfg, $sid, $node);
2793    });
2794 }
2795
2796 # list nodes where all VM images are available (used by has_feature API)
2797 sub shared_nodes {
2798     my ($conf, $storecfg) = @_;
2799
2800     my $nodelist = PVE::Cluster::get_nodelist();
2801     my $nodehash = { map { $_ => 1 } @$nodelist };
2802     my $nodename = PVE::INotify::nodename();
2803
2804     foreach_drive($conf, sub {
2805         my ($ds, $drive) = @_;
2806
2807         my $volid = $drive->{file};
2808         return if !$volid;
2809
2810         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2811         if ($storeid) {
2812             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2813             if ($scfg->{disable}) {
2814                 $nodehash = {};
2815             } elsif (my $avail = $scfg->{nodes}) {
2816                 foreach my $node (keys %$nodehash) {
2817                     delete $nodehash->{$node} if !$avail->{$node};
2818                 }
2819             } elsif (!$scfg->{shared}) {
2820                 foreach my $node (keys %$nodehash) {
2821                     delete $nodehash->{$node} if $node ne $nodename
2822                 }
2823             }
2824         }
2825     });
2826
2827     return $nodehash
2828 }
2829
2830 sub check_cmdline {
2831     my ($pidfile, $pid) = @_;
2832
2833     my $fh = IO::File->new("/proc/$pid/cmdline", "r");
2834     if (defined($fh)) {
2835         my $line = <$fh>;
2836         $fh->close;
2837         return undef if !$line;
2838         my @param = split(/\0/, $line);
2839
2840         my $cmd = $param[0];
2841         return if !$cmd || ($cmd !~ m|kvm$| && $cmd !~ m@(?:^|/)qemu-system-[^/]+$@);
2842
2843         for (my $i = 0; $i < scalar (@param); $i++) {
2844             my $p = $param[$i];
2845             next if !$p;
2846             if (($p eq '-pidfile') || ($p eq '--pidfile')) {
2847                 my $p = $param[$i+1];
2848                 return 1 if $p && ($p eq $pidfile);
2849                 return undef;
2850             }
2851         }
2852     }
2853     return undef;
2854 }
2855
2856 sub check_running {
2857     my ($vmid, $nocheck, $node) = @_;
2858
2859     my $filename = PVE::QemuConfig->config_file($vmid, $node);
2860
2861     die "unable to find configuration file for VM $vmid - no such machine\n"
2862         if !$nocheck && ! -f $filename;
2863
2864     my $pidfile = pidfile_name($vmid);
2865
2866     if (my $fd = IO::File->new("<$pidfile")) {
2867         my $st = stat($fd);
2868         my $line = <$fd>;
2869         close($fd);
2870
2871         my $mtime = $st->mtime;
2872         if ($mtime > time()) {
2873             warn "file '$filename' modified in future\n";
2874         }
2875
2876         if ($line =~ m/^(\d+)$/) {
2877             my $pid = $1;
2878             if (check_cmdline($pidfile, $pid)) {
2879                 if (my $pinfo = PVE::ProcFSTools::check_process_running($pid)) {
2880                     return $pid;
2881                 }
2882             }
2883         }
2884     }
2885
2886     return undef;
2887 }
2888
2889 sub vzlist {
2890
2891     my $vzlist = config_list();
2892
2893     my $fd = IO::Dir->new($var_run_tmpdir) || return $vzlist;
2894
2895     while (defined(my $de = $fd->read)) {
2896         next if $de !~ m/^(\d+)\.pid$/;
2897         my $vmid = $1;
2898         next if !defined($vzlist->{$vmid});
2899         if (my $pid = check_running($vmid)) {
2900             $vzlist->{$vmid}->{pid} = $pid;
2901         }
2902     }
2903
2904     return $vzlist;
2905 }
2906
2907 sub disksize {
2908     my ($storecfg, $conf) = @_;
2909
2910     my $bootdisk = $conf->{bootdisk};
2911     return undef if !$bootdisk;
2912     return undef if !is_valid_drivename($bootdisk);
2913
2914     return undef if !$conf->{$bootdisk};
2915
2916     my $drive = parse_drive($bootdisk, $conf->{$bootdisk});
2917     return undef if !defined($drive);
2918
2919     return undef if drive_is_cdrom($drive);
2920
2921     my $volid = $drive->{file};
2922     return undef if !$volid;
2923
2924     return $drive->{size};
2925 }
2926
2927 our $vmstatus_return_properties = {
2928     vmid => get_standard_option('pve-vmid'),
2929     status => {
2930         description => "Qemu process status.",
2931         type => 'string',
2932         enum => ['stopped', 'running'],
2933     },
2934     maxmem => {
2935         description => "Maximum memory in bytes.",
2936         type => 'integer',
2937         optional => 1,
2938         renderer => 'bytes',
2939     },
2940     maxdisk => {
2941         description => "Root disk size in bytes.",
2942         type => 'integer',
2943         optional => 1,
2944         renderer => 'bytes',
2945     },
2946     name => {
2947         description => "VM name.",
2948         type => 'string',
2949         optional => 1,
2950     },
2951     qmpstatus => {
2952         description => "Qemu QMP agent status.",
2953         type => 'string',
2954         optional => 1,
2955     },
2956     pid => {
2957         description => "PID of running qemu process.",
2958         type => 'integer',
2959         optional => 1,
2960     },
2961     uptime => {
2962         description => "Uptime.",
2963         type => 'integer',
2964         optional => 1,
2965         renderer => 'duration',
2966     },
2967     cpus => {
2968         description => "Maximum usable CPUs.",
2969         type => 'number',
2970         optional => 1,
2971     },
2972 };
2973
2974 my $last_proc_pid_stat;
2975
2976 # get VM status information
2977 # This must be fast and should not block ($full == false)
2978 # We only query KVM using QMP if $full == true (this can be slow)
2979 sub vmstatus {
2980     my ($opt_vmid, $full) = @_;
2981
2982     my $res = {};
2983
2984     my $storecfg = PVE::Storage::config();
2985
2986     my $list = vzlist();
2987     my $defaults = load_defaults();
2988
2989     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
2990
2991     my $cpucount = $cpuinfo->{cpus} || 1;
2992
2993     foreach my $vmid (keys %$list) {
2994         next if $opt_vmid && ($vmid ne $opt_vmid);
2995
2996         my $cfspath = PVE::QemuConfig->cfs_config_path($vmid);
2997         my $conf = PVE::Cluster::cfs_read_file($cfspath) || {};
2998
2999         my $d = { vmid => $vmid };
3000         $d->{pid} = $list->{$vmid}->{pid};
3001
3002         # fixme: better status?
3003         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
3004
3005         my $size = disksize($storecfg, $conf);
3006         if (defined($size)) {
3007             $d->{disk} = 0; # no info available
3008             $d->{maxdisk} = $size;
3009         } else {
3010             $d->{disk} = 0;
3011             $d->{maxdisk} = 0;
3012         }
3013
3014         $d->{cpus} = ($conf->{sockets} || $defaults->{sockets})
3015             * ($conf->{cores} || $defaults->{cores});
3016         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
3017         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
3018
3019         $d->{name} = $conf->{name} || "VM $vmid";
3020         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024)
3021             : $defaults->{memory}*(1024*1024);
3022
3023         if ($conf->{balloon}) {
3024             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
3025             $d->{shares} = defined($conf->{shares}) ? $conf->{shares}
3026                 : $defaults->{shares};
3027         }
3028
3029         $d->{uptime} = 0;
3030         $d->{cpu} = 0;
3031         $d->{mem} = 0;
3032
3033         $d->{netout} = 0;
3034         $d->{netin} = 0;
3035
3036         $d->{diskread} = 0;
3037         $d->{diskwrite} = 0;
3038
3039         $d->{template} = PVE::QemuConfig->is_template($conf);
3040
3041         $d->{serial} = 1 if conf_has_serial($conf);
3042
3043         $res->{$vmid} = $d;
3044     }
3045
3046     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
3047     foreach my $dev (keys %$netdev) {
3048         next if $dev !~ m/^tap([1-9]\d*)i/;
3049         my $vmid = $1;
3050         my $d = $res->{$vmid};
3051         next if !$d;
3052
3053         $d->{netout} += $netdev->{$dev}->{receive};
3054         $d->{netin} += $netdev->{$dev}->{transmit};
3055
3056         if ($full) {
3057             $d->{nics}->{$dev}->{netout} = $netdev->{$dev}->{receive};
3058             $d->{nics}->{$dev}->{netin} = $netdev->{$dev}->{transmit};
3059         }
3060
3061     }
3062
3063     my $ctime = gettimeofday;
3064
3065     foreach my $vmid (keys %$list) {
3066
3067         my $d = $res->{$vmid};
3068         my $pid = $d->{pid};
3069         next if !$pid;
3070
3071         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
3072         next if !$pstat; # not running
3073
3074         my $used = $pstat->{utime} + $pstat->{stime};
3075
3076         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
3077
3078         if ($pstat->{vsize}) {
3079             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
3080         }
3081
3082         my $old = $last_proc_pid_stat->{$pid};
3083         if (!$old) {
3084             $last_proc_pid_stat->{$pid} = {
3085                 time => $ctime,
3086                 used => $used,
3087                 cpu => 0,
3088             };
3089             next;
3090         }
3091
3092         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
3093
3094         if ($dtime > 1000) {
3095             my $dutime = $used -  $old->{used};
3096
3097             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
3098             $last_proc_pid_stat->{$pid} = {
3099                 time => $ctime,
3100                 used => $used,
3101                 cpu => $d->{cpu},
3102             };
3103         } else {
3104             $d->{cpu} = $old->{cpu};
3105         }
3106     }
3107
3108     return $res if !$full;
3109
3110     my $qmpclient = PVE::QMPClient->new();
3111
3112     my $ballooncb = sub {
3113         my ($vmid, $resp) = @_;
3114
3115         my $info = $resp->{'return'};
3116         return if !$info->{max_mem};
3117
3118         my $d = $res->{$vmid};
3119
3120         # use memory assigned to VM
3121         $d->{maxmem} = $info->{max_mem};
3122         $d->{balloon} = $info->{actual};
3123
3124         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
3125             $d->{mem} = $info->{total_mem} - $info->{free_mem};
3126             $d->{freemem} = $info->{free_mem};
3127         }
3128
3129         $d->{ballooninfo} = $info;
3130     };
3131
3132     my $blockstatscb = sub {
3133         my ($vmid, $resp) = @_;
3134         my $data = $resp->{'return'} || [];
3135         my $totalrdbytes = 0;
3136         my $totalwrbytes = 0;
3137
3138         for my $blockstat (@$data) {
3139             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
3140             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
3141
3142             $blockstat->{device} =~ s/drive-//;
3143             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
3144         }
3145         $res->{$vmid}->{diskread} = $totalrdbytes;
3146         $res->{$vmid}->{diskwrite} = $totalwrbytes;
3147     };
3148
3149     my $statuscb = sub {
3150         my ($vmid, $resp) = @_;
3151
3152         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
3153         # this fails if ballon driver is not loaded, so this must be
3154         # the last commnand (following command are aborted if this fails).
3155         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
3156
3157         my $status = 'unknown';
3158         if (!defined($status = $resp->{'return'}->{status})) {
3159             warn "unable to get VM status\n";
3160             return;
3161         }
3162
3163         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
3164     };
3165
3166     foreach my $vmid (keys %$list) {
3167         next if $opt_vmid && ($vmid ne $opt_vmid);
3168         next if !$res->{$vmid}->{pid}; # not running
3169         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
3170     }
3171
3172     $qmpclient->queue_execute(undef, 2);
3173
3174     foreach my $vmid (keys %$list) {
3175         next if $opt_vmid && ($vmid ne $opt_vmid);
3176         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
3177     }
3178
3179     return $res;
3180 }
3181
3182 sub foreach_drive {
3183     my ($conf, $func, @param) = @_;
3184
3185     foreach my $ds (valid_drive_names()) {
3186         next if !defined($conf->{$ds});
3187
3188         my $drive = parse_drive($ds, $conf->{$ds});
3189         next if !$drive;
3190
3191         &$func($ds, $drive, @param);
3192     }
3193 }
3194
3195 sub foreach_volid {
3196     my ($conf, $func, @param) = @_;
3197
3198     my $volhash = {};
3199
3200     my $test_volid = sub {
3201         my ($volid, $is_cdrom, $replicate, $shared, $snapname) = @_;
3202
3203         return if !$volid;
3204
3205         $volhash->{$volid}->{cdrom} //= 1;
3206         $volhash->{$volid}->{cdrom} = 0 if !$is_cdrom;
3207
3208         $volhash->{$volid}->{replicate} //= 0;
3209         $volhash->{$volid}->{replicate} = 1 if $replicate;
3210
3211         $volhash->{$volid}->{shared} //= 0;
3212         $volhash->{$volid}->{shared} = 1 if $shared;
3213
3214         $volhash->{$volid}->{referenced_in_config} //= 0;
3215         $volhash->{$volid}->{referenced_in_config} = 1 if !defined($snapname);
3216
3217         $volhash->{$volid}->{referenced_in_snapshot}->{$snapname} = 1
3218             if defined($snapname);
3219     };
3220
3221     foreach_drive($conf, sub {
3222         my ($ds, $drive) = @_;
3223         $test_volid->($drive->{file}, drive_is_cdrom($drive), $drive->{replicate} // 1, $drive->{shared}, undef);
3224     });
3225
3226     foreach my $snapname (keys %{$conf->{snapshots}}) {
3227         my $snap = $conf->{snapshots}->{$snapname};
3228         $test_volid->($snap->{vmstate}, 0, 1, $snapname);
3229         foreach_drive($snap, sub {
3230             my ($ds, $drive) = @_;
3231             $test_volid->($drive->{file}, drive_is_cdrom($drive), $drive->{replicate} // 1, $drive->{shared}, $snapname);
3232         });
3233     }
3234
3235     foreach my $volid (keys %$volhash) {
3236         &$func($volid, $volhash->{$volid}, @param);
3237     }
3238 }
3239
3240 sub conf_has_serial {
3241     my ($conf) = @_;
3242
3243     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3244         if ($conf->{"serial$i"}) {
3245             return 1;
3246         }
3247     }
3248
3249     return 0;
3250 }
3251
3252 sub vga_conf_has_spice {
3253     my ($vga) = @_;
3254
3255     my $vgaconf = parse_vga($vga);
3256     my $vgatype = $vgaconf->{type};
3257     return 0 if !$vgatype || $vgatype !~ m/^qxl([234])?$/;
3258
3259     return $1 || 1;
3260 }
3261
3262 my $host_arch; # FIXME: fix PVE::Tools::get_host_arch
3263 sub get_host_arch() {
3264     $host_arch = (POSIX::uname())[4] if !$host_arch;
3265     return $host_arch;
3266 }
3267
3268 sub is_native($) {
3269     my ($arch) = @_;
3270     return get_host_arch() eq $arch;
3271 }
3272
3273 my $default_machines = {
3274     x86_64 => 'pc',
3275     aarch64 => 'virt',
3276 };
3277
3278 sub get_basic_machine_info {
3279     my ($conf, $forcemachine) = @_;
3280
3281     my $arch = $conf->{arch} // get_host_arch();
3282     my $machine = $forcemachine || $conf->{machine} || $default_machines->{$arch};
3283     return ($arch, $machine);
3284 }
3285
3286 sub get_ovmf_files($) {
3287     my ($arch) = @_;
3288
3289     my $ovmf = $OVMF->{$arch}
3290         or die "no OVMF images known for architecture '$arch'\n";
3291
3292     return @$ovmf;
3293 }
3294
3295 my $Arch2Qemu = {
3296     aarch64 => '/usr/bin/qemu-system-aarch64',
3297     x86_64 => '/usr/bin/qemu-system-x86_64',
3298 };
3299 sub get_command_for_arch($) {
3300     my ($arch) = @_;
3301     return '/usr/bin/kvm' if is_native($arch);
3302
3303     my $cmd = $Arch2Qemu->{$arch}
3304         or die "don't know how to emulate architecture '$arch'\n";
3305     return $cmd;
3306 }
3307
3308 sub get_cpu_options {
3309     my ($conf, $arch, $kvm, $machine_type, $kvm_off, $kvmver, $winversion, $gpu_passthrough) = @_;
3310
3311     my $cpuFlags = [];
3312     my $ostype = $conf->{ostype};
3313
3314     my $cpu = $kvm ? "kvm64" : "qemu64";
3315     if ($arch eq 'aarch64') {
3316         $cpu = 'cortex-a57';
3317     }
3318     if (my $cputype = $conf->{cpu}) {
3319         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
3320             or die "Cannot parse cpu description: $cputype\n";
3321         $cpu = $cpuconf->{cputype};
3322         $kvm_off = 1 if $cpuconf->{hidden};
3323
3324         if (defined(my $flags = $cpuconf->{flags})) {
3325             push @$cpuFlags, split(";", $flags);
3326         }
3327     }
3328
3329     push @$cpuFlags , '+lahf_lm' if $cpu eq 'kvm64';
3330
3331     push @$cpuFlags , '-x2apic'
3332         if $conf->{ostype} && $conf->{ostype} eq 'solaris';
3333
3334     push @$cpuFlags, '+sep' if $cpu eq 'kvm64' || $cpu eq 'kvm32';
3335
3336     push @$cpuFlags, '-rdtscp' if $cpu =~ m/^Opteron/;
3337
3338     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3339
3340         push @$cpuFlags , '+kvm_pv_unhalt' if $kvm;
3341         push @$cpuFlags , '+kvm_pv_eoi' if $kvm;
3342     }
3343
3344     add_hyperv_enlightenments($cpuFlags, $winversion, $machine_type, $kvmver, $conf->{bios}, $gpu_passthrough) if $kvm;
3345
3346     push @$cpuFlags, 'enforce' if $cpu ne 'host' && $kvm;
3347
3348     push @$cpuFlags, 'kvm=off' if $kvm_off;
3349
3350     if (my $cpu_vendor = $cpu_vendor_list->{$cpu}) {
3351         push @$cpuFlags, "vendor=${cpu_vendor}"
3352             if $cpu_vendor ne 'default';
3353     } elsif ($arch ne 'aarch64') {
3354         die "internal error"; # should not happen
3355     }
3356
3357     $cpu .= "," . join(',', @$cpuFlags) if scalar(@$cpuFlags);
3358
3359     return ('-cpu', $cpu);
3360 }
3361
3362 sub config_to_command {
3363     my ($storecfg, $vmid, $conf, $defaults, $forcemachine) = @_;
3364
3365     my $cmd = [];
3366     my $globalFlags = [];
3367     my $machineFlags = [];
3368     my $rtcFlags = [];
3369     my $devices = [];
3370     my $pciaddr = '';
3371     my $bridges = {};
3372     my $kvmver = kvm_user_version();
3373     my $vernum = 0; # unknown
3374     my $ostype = $conf->{ostype};
3375     my $winversion = windows_version($ostype);
3376     my $kvm = $conf->{kvm};
3377
3378     my ($arch, $machine_type) = get_basic_machine_info($conf, $forcemachine);
3379     $kvm //= 1 if is_native($arch);
3380
3381     if ($kvm) {
3382         die "KVM virtualisation configured, but not available. Either disable in VM configuration or enable in BIOS.\n"
3383             if !defined kvm_version();
3384     }
3385
3386     if ($kvmver =~ m/^(\d+)\.(\d+)$/) {
3387         $vernum = $1*1000000+$2*1000;
3388     } elsif ($kvmver =~ m/^(\d+)\.(\d+)\.(\d+)$/) {
3389         $vernum = $1*1000000+$2*1000+$3;
3390     }
3391
3392     die "detected old qemu-kvm binary ($kvmver)\n" if $vernum < 15000;
3393
3394     my $have_ovz = -f '/proc/vz/vestat';
3395
3396     my $q35 = machine_type_is_q35($conf);
3397     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
3398     my $use_old_bios_files = undef;
3399     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3400
3401     my $cpuunits = defined($conf->{cpuunits}) ?
3402             $conf->{cpuunits} : $defaults->{cpuunits};
3403
3404     push @$cmd, get_command_for_arch($arch);
3405
3406     push @$cmd, '-id', $vmid;
3407
3408     my $vmname = $conf->{name} || "vm$vmid";
3409
3410     push @$cmd, '-name', $vmname;
3411
3412     my $use_virtio = 0;
3413
3414     my $qmpsocket = qmp_socket($vmid);
3415     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server,nowait";
3416     push @$cmd, '-mon', "chardev=qmp,mode=control";
3417
3418     if (qemu_machine_feature_enabled($machine_type, $kvmver, 2, 12)) {
3419         my $eventsocket = qmp_socket($vmid, 0, 'event');
3420         push @$cmd, '-chardev', "socket,id=qmp-event,path=$eventsocket,server,nowait";
3421         push @$cmd, '-mon', "chardev=qmp-event,mode=control";
3422     }
3423
3424     push @$cmd, '-pidfile' , pidfile_name($vmid);
3425
3426     push @$cmd, '-daemonize';
3427
3428     if ($conf->{smbios1}) {
3429         push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
3430     }
3431
3432     if ($conf->{vmgenid}) {
3433         push @$devices, '-device', 'vmgenid,guid='.$conf->{vmgenid};
3434     }
3435
3436     my ($ovmf_code, $ovmf_vars) = get_ovmf_files($arch);
3437     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3438         die "uefi base image not found\n" if ! -f $ovmf_code;
3439
3440         my $path;
3441         my $format;
3442         if (my $efidisk = $conf->{efidisk0}) {
3443             my $d = PVE::JSONSchema::parse_property_string($efidisk_fmt, $efidisk);
3444             my ($storeid, $volname) = PVE::Storage::parse_volume_id($d->{file}, 1);
3445             $format = $d->{format};
3446             if ($storeid) {
3447                 $path = PVE::Storage::path($storecfg, $d->{file});
3448                 if (!defined($format)) {
3449                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
3450                     $format = qemu_img_format($scfg, $volname);
3451                 }
3452             } else {
3453                 $path = $d->{file};
3454                 die "efidisk format must be specified\n"
3455                     if !defined($format);
3456             }
3457         } else {
3458             warn "no efidisk configured! Using temporary efivars disk.\n";
3459             $path = "/tmp/$vmid-ovmf.fd";
3460             PVE::Tools::file_copy($ovmf_vars, $path, -s $ovmf_vars);
3461             $format = 'raw';
3462         }
3463
3464         push @$cmd, '-drive', "if=pflash,unit=0,format=raw,readonly,file=$ovmf_code";
3465         push @$cmd, '-drive', "if=pflash,unit=1,format=$format,id=drive-efidisk0,file=$path";
3466     }
3467
3468
3469     # add usb controllers
3470     my @usbcontrollers = PVE::QemuServer::USB::get_usb_controllers($conf, $bridges, $q35, $usbdesc->{format}, $MAX_USB_DEVICES);
3471     push @$devices, @usbcontrollers if @usbcontrollers;
3472     my $vga = parse_vga($conf->{vga});
3473
3474     my $qxlnum = vga_conf_has_spice($conf->{vga});
3475     $vga->{type} = 'qxl' if $qxlnum;
3476
3477     if (!$vga->{type}) {
3478         if (qemu_machine_feature_enabled($machine_type, $kvmver, 2, 9)) {
3479             $vga->{type} = (!$winversion || $winversion >= 6) ? 'std' : 'cirrus';
3480         } else {
3481             $vga->{type} = ($winversion >= 6) ? 'std' : 'cirrus';
3482         }
3483     }
3484
3485     # enable absolute mouse coordinates (needed by vnc)
3486     my $tablet;
3487     if (defined($conf->{tablet})) {
3488         $tablet = $conf->{tablet};
3489     } else {
3490         $tablet = $defaults->{tablet};
3491         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
3492         $tablet = 0 if $vga->{type} =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
3493     }
3494
3495     push @$devices, '-device', print_tabletdevice_full($conf) if $tablet;
3496
3497     my $kvm_off = 0;
3498     my $gpu_passthrough;
3499
3500     # host pci devices
3501     for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
3502         my $d = parse_hostpci($conf->{"hostpci$i"});
3503         next if !$d;
3504
3505         my $pcie = $d->{pcie};
3506         if($pcie){
3507             die "q35 machine model is not enabled" if !$q35;
3508             $pciaddr = print_pcie_addr("hostpci$i");
3509         }else{
3510             $pciaddr = print_pci_addr("hostpci$i", $bridges);
3511         }
3512
3513         my $rombar = defined($d->{rombar}) && !$d->{rombar} ? ',rombar=0' : '';
3514         my $romfile = $d->{romfile};
3515
3516         my $xvga = '';
3517         if ($d->{'x-vga'}) {
3518             $xvga = ',x-vga=on';
3519             $kvm_off = 1;
3520             $vga->{type} = 'none';
3521             $gpu_passthrough = 1;
3522
3523             if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3524                 $xvga = "";
3525             }
3526         }
3527         my $pcidevices = $d->{pciid};
3528         my $multifunction = 1 if @$pcidevices > 1;
3529
3530         my $j=0;
3531         foreach my $pcidevice (@$pcidevices) {
3532
3533             my $id = "hostpci$i";
3534             $id .= ".$j" if $multifunction;
3535             my $addr = $pciaddr;
3536             $addr .= ".$j" if $multifunction;
3537             my $devicestr = "vfio-pci,host=$pcidevice->{id}.$pcidevice->{function},id=$id$addr";
3538
3539             if($j == 0){
3540                 $devicestr .= "$rombar$xvga";
3541                 $devicestr .= ",multifunction=on" if $multifunction;
3542                 $devicestr .= ",romfile=/usr/share/kvm/$romfile" if $romfile;
3543             }
3544
3545             push @$devices, '-device', $devicestr;
3546             $j++;
3547         }
3548     }
3549
3550     # usb devices
3551     my @usbdevices = PVE::QemuServer::USB::get_usb_devices($conf, $usbdesc->{format}, $MAX_USB_DEVICES);
3552     push @$devices, @usbdevices if @usbdevices;
3553     # serial devices
3554     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3555         if (my $path = $conf->{"serial$i"}) {
3556             if ($path eq 'socket') {
3557                 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
3558                 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server,nowait";
3559                 push @$devices, '-device', "isa-serial,chardev=serial$i";
3560             } else {
3561                 die "no such serial device\n" if ! -c $path;
3562                 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
3563                 push @$devices, '-device', "isa-serial,chardev=serial$i";
3564             }
3565         }
3566     }
3567
3568     # parallel devices
3569     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
3570         if (my $path = $conf->{"parallel$i"}) {
3571             die "no such parallel device\n" if ! -c $path;
3572             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
3573             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
3574             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
3575         }
3576     }
3577
3578
3579     my $sockets = 1;
3580     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3581     $sockets = $conf->{sockets} if  $conf->{sockets};
3582
3583     my $cores = $conf->{cores} || 1;
3584
3585     my $maxcpus = $sockets * $cores;
3586
3587     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
3588
3589     my $allowed_vcpus = $cpuinfo->{cpus};
3590
3591     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n"
3592         if ($allowed_vcpus < $maxcpus);
3593
3594     if($hotplug_features->{cpu} && qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 7)) {
3595
3596         push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3597         for (my $i = 2; $i <= $vcpus; $i++)  {
3598             my $cpustr = print_cpu_device($conf,$i);
3599             push @$cmd, '-device', $cpustr;
3600         }
3601
3602     } else {
3603
3604         push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3605     }
3606     push @$cmd, '-nodefaults';
3607
3608     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
3609
3610     my $bootindex_hash = {};
3611     my $i = 1;
3612     foreach my $o (split(//, $bootorder)) {
3613         $bootindex_hash->{$o} = $i*100;
3614         $i++;
3615     }
3616
3617     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
3618
3619     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
3620
3621     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
3622
3623     if ($vga->{type} && $vga->{type} !~ m/^serial\d+$/ && $vga->{type} ne 'none'){
3624         push @$devices, '-device', print_vga_device($conf, $vga, undef, $qxlnum, $bridges);
3625         my $socket = vnc_socket($vmid);
3626         push @$cmd,  '-vnc', "unix:$socket,x509,password";
3627     } else {
3628         push @$cmd, '-vga', 'none' if $vga->{type} eq 'none';
3629         push @$cmd, '-nographic';
3630     }
3631
3632     # time drift fix
3633     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
3634
3635     my $useLocaltime = $conf->{localtime};
3636
3637     if ($winversion >= 5) { # windows
3638         $useLocaltime = 1 if !defined($conf->{localtime});
3639
3640         # use time drift fix when acpi is enabled
3641         if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
3642             $tdf = 1 if !defined($conf->{tdf});
3643         }
3644     }
3645
3646     if ($winversion >= 6) {
3647         push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3648         push @$cmd, '-no-hpet';
3649     }
3650
3651     push @$rtcFlags, 'driftfix=slew' if $tdf;
3652
3653     if (!$kvm) {
3654         push @$machineFlags, 'accel=tcg';
3655     }
3656
3657     if ($machine_type) {
3658         push @$machineFlags, "type=${machine_type}";
3659     }
3660
3661     if ($conf->{startdate}) {
3662         push @$rtcFlags, "base=$conf->{startdate}";
3663     } elsif ($useLocaltime) {
3664         push @$rtcFlags, 'base=localtime';
3665     }
3666
3667     push @$cmd, get_cpu_options($conf, $arch, $kvm, $machine_type, $kvm_off, $kvmver, $winversion, $gpu_passthrough);
3668
3669     PVE::QemuServer::Memory::config($conf, $vmid, $sockets, $cores, $defaults, $hotplug_features, $cmd);
3670
3671     push @$cmd, '-S' if $conf->{freeze};
3672
3673     push @$cmd, '-k', $conf->{keyboard} if defined($conf->{keyboard});
3674
3675     # enable sound
3676     #my $soundhw = $conf->{soundhw} || $defaults->{soundhw};
3677     #push @$cmd, '-soundhw', 'es1370';
3678     #push @$cmd, '-soundhw', $soundhw if $soundhw;
3679
3680     if (parse_guest_agent($conf)->{enabled}) {
3681         my $qgasocket = qmp_socket($vmid, 1);
3682         my $pciaddr = print_pci_addr("qga0", $bridges);
3683         push @$devices, '-chardev', "socket,path=$qgasocket,server,nowait,id=qga0";
3684         push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3685         push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3686     }
3687
3688     my $spice_port;
3689
3690     if ($qxlnum) {
3691         if ($qxlnum > 1) {
3692             if ($winversion){
3693                 for(my $i = 1; $i < $qxlnum; $i++){
3694                     push @$devices, '-device', print_vga_device($conf, $vga, $i, $qxlnum, $bridges);
3695                 }
3696             } else {
3697                 # assume other OS works like Linux
3698                 my ($ram, $vram) = ("134217728", "67108864");
3699                 if ($vga->{memory}) {
3700                     $ram = PVE::Tools::convert_size($qxlnum*4*$vga->{memory}, 'mb' => 'b');
3701                     $vram = PVE::Tools::convert_size($qxlnum*2*$vga->{memory}, 'mb' => 'b');
3702                 }
3703                 push @$cmd, '-global', "qxl-vga.ram_size=$ram";
3704                 push @$cmd, '-global', "qxl-vga.vram_size=$vram";
3705             }
3706         }
3707
3708         my $pciaddr = print_pci_addr("spice", $bridges);
3709
3710         my $nodename = PVE::INotify::nodename();
3711         my $pfamily = PVE::Tools::get_host_address_family($nodename);
3712         my @nodeaddrs = PVE::Tools::getaddrinfo_all('localhost', family => $pfamily);
3713         die "failed to get an ip address of type $pfamily for 'localhost'\n" if !@nodeaddrs;
3714         my $localhost = PVE::Network::addr_to_ip($nodeaddrs[0]->{addr});
3715         $spice_port = PVE::Tools::next_spice_port($pfamily, $localhost);
3716
3717         push @$devices, '-spice', "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";
3718
3719         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3720         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3721         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3722     }
3723
3724     # enable balloon by default, unless explicitly disabled
3725     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3726         $pciaddr = print_pci_addr("balloon0", $bridges);
3727         push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
3728     }
3729
3730     if ($conf->{watchdog}) {
3731         my $wdopts = parse_watchdog($conf->{watchdog});
3732         $pciaddr = print_pci_addr("watchdog", $bridges);
3733         my $watchdog = $wdopts->{model} || 'i6300esb';
3734         push @$devices, '-device', "$watchdog$pciaddr";
3735         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3736     }
3737
3738     my $vollist = [];
3739     my $scsicontroller = {};
3740     my $ahcicontroller = {};
3741     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3742
3743     # Add iscsi initiator name if available
3744     if (my $initiator = get_initiator_name()) {
3745         push @$devices, '-iscsi', "initiator-name=$initiator";
3746     }
3747
3748     foreach_drive($conf, sub {
3749         my ($ds, $drive) = @_;
3750
3751         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3752             push @$vollist, $drive->{file};
3753         }
3754
3755         # ignore efidisk here, already added in bios/fw handling code above
3756         return if $drive->{interface} eq 'efidisk';
3757
3758         $use_virtio = 1 if $ds =~ m/^virtio/;
3759
3760         if (drive_is_cdrom ($drive)) {
3761             if ($bootindex_hash->{d}) {
3762                 $drive->{bootindex} = $bootindex_hash->{d};
3763                 $bootindex_hash->{d} += 1;
3764             }
3765         } else {
3766             if ($bootindex_hash->{c}) {
3767                 $drive->{bootindex} = $bootindex_hash->{c} if $conf->{bootdisk} && ($conf->{bootdisk} eq $ds);
3768                 $bootindex_hash->{c} += 1;
3769             }
3770         }
3771
3772         if($drive->{interface} eq 'virtio'){
3773            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3774         }
3775
3776         if ($drive->{interface} eq 'scsi') {
3777
3778             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
3779
3780             $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges);
3781             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
3782
3783             my $iothread = '';
3784             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
3785                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3786                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3787             } elsif ($drive->{iothread}) {
3788                 warn "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n";
3789             }
3790
3791             my $queues = '';
3792             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
3793                 $queues = ",num_queues=$drive->{queues}";
3794             }
3795
3796             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues" if !$scsicontroller->{$controller};
3797             $scsicontroller->{$controller}=1;
3798         }
3799
3800         if ($drive->{interface} eq 'sata') {
3801            my $controller = int($drive->{index} / $MAX_SATA_DISKS);
3802            $pciaddr = print_pci_addr("ahci$controller", $bridges);
3803            push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr" if !$ahcicontroller->{$controller};
3804            $ahcicontroller->{$controller}=1;
3805         }
3806
3807         my $drive_cmd = print_drive_full($storecfg, $vmid, $drive);
3808         push @$devices, '-drive',$drive_cmd;
3809         push @$devices, '-device', print_drivedevice_full($storecfg, $conf, $vmid, $drive, $bridges);
3810     });
3811
3812     for (my $i = 0; $i < $MAX_NETS; $i++) {
3813          next if !$conf->{"net$i"};
3814          my $d = parse_net($conf->{"net$i"});
3815          next if !$d;
3816
3817          $use_virtio = 1 if $d->{model} eq 'virtio';
3818
3819          if ($bootindex_hash->{n}) {
3820             $d->{bootindex} = $bootindex_hash->{n};
3821             $bootindex_hash->{n} += 1;
3822          }
3823
3824          my $netdevfull = print_netdev_full($vmid,$conf,$d,"net$i");
3825          push @$devices, '-netdev', $netdevfull;
3826
3827          my $netdevicefull = print_netdevice_full($vmid, $conf, $d, "net$i", $bridges, $use_old_bios_files);
3828          push @$devices, '-device', $netdevicefull;
3829     }
3830
3831     if (!$q35) {
3832         # add pci bridges
3833         if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3834            $bridges->{1} = 1;
3835            $bridges->{2} = 1;
3836         }
3837
3838         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
3839
3840         while (my ($k, $v) = each %$bridges) {
3841             $pciaddr = print_pci_addr("pci.$k");
3842             unshift @$devices, '-device', "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr" if $k > 0;
3843         }
3844     }
3845
3846     # add custom args
3847     if ($conf->{args}) {
3848         my $aa = PVE::Tools::split_args($conf->{args});
3849         push @$cmd, @$aa;
3850     }
3851
3852     push @$cmd, @$devices;
3853     push @$cmd, '-rtc', join(',', @$rtcFlags)
3854         if scalar(@$rtcFlags);
3855     push @$cmd, '-machine', join(',', @$machineFlags)
3856         if scalar(@$machineFlags);
3857     push @$cmd, '-global', join(',', @$globalFlags)
3858         if scalar(@$globalFlags);
3859
3860     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
3861 }
3862
3863 sub vnc_socket {
3864     my ($vmid) = @_;
3865     return "${var_run_tmpdir}/$vmid.vnc";
3866 }
3867
3868 sub spice_port {
3869     my ($vmid) = @_;
3870
3871     my $res = vm_mon_cmd($vmid, 'query-spice');
3872
3873     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
3874 }
3875
3876 sub qmp_socket {
3877     my ($vmid, $qga, $name) = @_;
3878     my $sockettype = $qga ? 'qga' : 'qmp';
3879     my $ext = $name ? '-'.$name : '';
3880     return "${var_run_tmpdir}/$vmid$ext.$sockettype";
3881 }
3882
3883 sub pidfile_name {
3884     my ($vmid) = @_;
3885     return "${var_run_tmpdir}/$vmid.pid";
3886 }
3887
3888 sub vm_devices_list {
3889     my ($vmid) = @_;
3890
3891     my $res = vm_mon_cmd($vmid, 'query-pci');
3892     my $devices_to_check = [];
3893     my $devices = {};
3894     foreach my $pcibus (@$res) {
3895         push @$devices_to_check, @{$pcibus->{devices}},
3896     }
3897
3898     while (@$devices_to_check) {
3899         my $to_check = [];
3900         for my $d (@$devices_to_check) {
3901             $devices->{$d->{'qdev_id'}} = 1 if $d->{'qdev_id'};
3902             next if !$d->{'pci_bridge'};
3903
3904             $devices->{$d->{'qdev_id'}} += scalar(@{$d->{'pci_bridge'}->{devices}});
3905             push @$to_check, @{$d->{'pci_bridge'}->{devices}};
3906         }
3907         $devices_to_check = $to_check;
3908     }
3909
3910     my $resblock = vm_mon_cmd($vmid, 'query-block');
3911     foreach my $block (@$resblock) {
3912         if($block->{device} =~ m/^drive-(\S+)/){
3913                 $devices->{$1} = 1;
3914         }
3915     }
3916
3917     my $resmice = vm_mon_cmd($vmid, 'query-mice');
3918     foreach my $mice (@$resmice) {
3919         if ($mice->{name} eq 'QEMU HID Tablet') {
3920             $devices->{tablet} = 1;
3921             last;
3922         }
3923     }
3924
3925     # for usb devices there is no query-usb
3926     # but we can iterate over the entries in
3927     # qom-list path=/machine/peripheral
3928     my $resperipheral = vm_mon_cmd($vmid, 'qom-list', path => '/machine/peripheral');
3929     foreach my $per (@$resperipheral) {
3930         if ($per->{name} =~ m/^usb\d+$/) {
3931             $devices->{$per->{name}} = 1;
3932         }
3933     }
3934
3935     return $devices;
3936 }
3937
3938 sub vm_deviceplug {
3939     my ($storecfg, $conf, $vmid, $deviceid, $device) = @_;
3940
3941     my $q35 = machine_type_is_q35($conf);
3942
3943     my $devices_list = vm_devices_list($vmid);
3944     return 1 if defined($devices_list->{$deviceid});
3945
3946     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid); # add PCI bridge if we need it for the device
3947
3948     if ($deviceid eq 'tablet') {
3949
3950         qemu_deviceadd($vmid, print_tabletdevice_full($conf));
3951
3952     } elsif ($deviceid =~ m/^usb(\d+)$/) {
3953
3954         die "usb hotplug currently not reliable\n";
3955         # since we can't reliably hot unplug all added usb devices
3956         # and usb passthrough disables live migration
3957         # we disable usb hotplugging for now
3958         qemu_deviceadd($vmid, PVE::QemuServer::USB::print_usbdevice_full($conf, $deviceid, $device));
3959
3960     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3961
3962         qemu_iothread_add($vmid, $deviceid, $device);
3963
3964         qemu_driveadd($storecfg, $vmid, $device);
3965         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
3966
3967         qemu_deviceadd($vmid, $devicefull);
3968         eval { qemu_deviceaddverify($vmid, $deviceid); };
3969         if (my $err = $@) {
3970             eval { qemu_drivedel($vmid, $deviceid); };
3971             warn $@ if $@;
3972             die $err;
3973         }
3974
3975     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3976
3977
3978         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
3979         my $pciaddr = print_pci_addr($deviceid);
3980         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
3981
3982         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
3983
3984         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
3985             qemu_iothread_add($vmid, $deviceid, $device);
3986             $devicefull .= ",iothread=iothread-$deviceid";
3987         }
3988
3989         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
3990             $devicefull .= ",num_queues=$device->{queues}";
3991         }
3992
3993         qemu_deviceadd($vmid, $devicefull);
3994         qemu_deviceaddverify($vmid, $deviceid);
3995
3996     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3997
3998         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device);
3999         qemu_driveadd($storecfg, $vmid, $device);
4000
4001         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
4002         eval { qemu_deviceadd($vmid, $devicefull); };
4003         if (my $err = $@) {
4004             eval { qemu_drivedel($vmid, $deviceid); };
4005             warn $@ if $@;
4006             die $err;
4007         }
4008
4009     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4010
4011         return undef if !qemu_netdevadd($vmid, $conf, $device, $deviceid);
4012
4013         my $machine_type = PVE::QemuServer::qemu_machine_pxe($vmid, $conf);
4014         my $use_old_bios_files = undef;
4015         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
4016
4017         my $netdevicefull = print_netdevice_full($vmid, $conf, $device, $deviceid, undef, $use_old_bios_files);
4018         qemu_deviceadd($vmid, $netdevicefull);
4019         eval { qemu_deviceaddverify($vmid, $deviceid); };
4020         if (my $err = $@) {
4021             eval { qemu_netdevdel($vmid, $deviceid); };
4022             warn $@ if $@;
4023             die $err;
4024         }
4025
4026     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
4027
4028         my $bridgeid = $2;
4029         my $pciaddr = print_pci_addr($deviceid);
4030         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
4031
4032         qemu_deviceadd($vmid, $devicefull);
4033         qemu_deviceaddverify($vmid, $deviceid);
4034
4035     } else {
4036         die "can't hotplug device '$deviceid'\n";
4037     }
4038
4039     return 1;
4040 }
4041
4042 # fixme: this should raise exceptions on error!
4043 sub vm_deviceunplug {
4044     my ($vmid, $conf, $deviceid) = @_;
4045
4046     my $devices_list = vm_devices_list($vmid);
4047     return 1 if !defined($devices_list->{$deviceid});
4048
4049     die "can't unplug bootdisk" if $conf->{bootdisk} && $conf->{bootdisk} eq $deviceid;
4050
4051     if ($deviceid eq 'tablet') {
4052
4053         qemu_devicedel($vmid, $deviceid);
4054
4055     } elsif ($deviceid =~ m/^usb\d+$/) {
4056
4057         die "usb hotplug currently not reliable\n";
4058         # when unplugging usb devices this way,
4059         # there may be remaining usb controllers/hubs
4060         # so we disable it for now
4061         qemu_devicedel($vmid, $deviceid);
4062         qemu_devicedelverify($vmid, $deviceid);
4063
4064     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4065
4066         qemu_devicedel($vmid, $deviceid);
4067         qemu_devicedelverify($vmid, $deviceid);
4068         qemu_drivedel($vmid, $deviceid);
4069         qemu_iothread_del($conf, $vmid, $deviceid);
4070
4071     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4072
4073         qemu_devicedel($vmid, $deviceid);
4074         qemu_devicedelverify($vmid, $deviceid);
4075         qemu_iothread_del($conf, $vmid, $deviceid);
4076
4077     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4078
4079         qemu_devicedel($vmid, $deviceid);
4080         qemu_drivedel($vmid, $deviceid);
4081         qemu_deletescsihw($conf, $vmid, $deviceid);
4082
4083     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4084
4085         qemu_devicedel($vmid, $deviceid);
4086         qemu_devicedelverify($vmid, $deviceid);
4087         qemu_netdevdel($vmid, $deviceid);
4088
4089     } else {
4090         die "can't unplug device '$deviceid'\n";
4091     }
4092
4093     return 1;
4094 }
4095
4096 sub qemu_deviceadd {
4097     my ($vmid, $devicefull) = @_;
4098
4099     $devicefull = "driver=".$devicefull;
4100     my %options =  split(/[=,]/, $devicefull);
4101
4102     vm_mon_cmd($vmid, "device_add" , %options);
4103 }
4104
4105 sub qemu_devicedel {
4106     my ($vmid, $deviceid) = @_;
4107
4108     my $ret = vm_mon_cmd($vmid, "device_del", id => $deviceid);
4109 }
4110
4111 sub qemu_iothread_add {
4112     my($vmid, $deviceid, $device) = @_;
4113
4114     if ($device->{iothread}) {
4115         my $iothreads = vm_iothreads_list($vmid);
4116         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
4117     }
4118 }
4119
4120 sub qemu_iothread_del {
4121     my($conf, $vmid, $deviceid) = @_;
4122
4123     my $device = parse_drive($deviceid, $conf->{$deviceid});
4124     if ($device->{iothread}) {
4125         my $iothreads = vm_iothreads_list($vmid);
4126         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
4127     }
4128 }
4129
4130 sub qemu_objectadd {
4131     my($vmid, $objectid, $qomtype) = @_;
4132
4133     vm_mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
4134
4135     return 1;
4136 }
4137
4138 sub qemu_objectdel {
4139     my($vmid, $objectid) = @_;
4140
4141     vm_mon_cmd($vmid, "object-del", id => $objectid);
4142
4143     return 1;
4144 }
4145
4146 sub qemu_driveadd {
4147     my ($storecfg, $vmid, $device) = @_;
4148
4149     my $drive = print_drive_full($storecfg, $vmid, $device);
4150     $drive =~ s/\\/\\\\/g;
4151     my $ret = vm_human_monitor_command($vmid, "drive_add auto \"$drive\"");
4152
4153     # If the command succeeds qemu prints: "OK"
4154     return 1 if $ret =~ m/OK/s;
4155
4156     die "adding drive failed: $ret\n";
4157 }
4158
4159 sub qemu_drivedel {
4160     my($vmid, $deviceid) = @_;
4161
4162     my $ret = vm_human_monitor_command($vmid, "drive_del drive-$deviceid");
4163     $ret =~ s/^\s+//;
4164
4165     return 1 if $ret eq "";
4166
4167     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
4168     return 1 if $ret =~ m/Device \'.*?\' not found/s;
4169
4170     die "deleting drive $deviceid failed : $ret\n";
4171 }
4172
4173 sub qemu_deviceaddverify {
4174     my ($vmid, $deviceid) = @_;
4175
4176     for (my $i = 0; $i <= 5; $i++) {
4177          my $devices_list = vm_devices_list($vmid);
4178          return 1 if defined($devices_list->{$deviceid});
4179          sleep 1;
4180     }
4181
4182     die "error on hotplug device '$deviceid'\n";
4183 }
4184
4185
4186 sub qemu_devicedelverify {
4187     my ($vmid, $deviceid) = @_;
4188
4189     # need to verify that the device is correctly removed as device_del
4190     # is async and empty return is not reliable
4191
4192     for (my $i = 0; $i <= 5; $i++) {
4193          my $devices_list = vm_devices_list($vmid);
4194          return 1 if !defined($devices_list->{$deviceid});
4195          sleep 1;
4196     }
4197
4198     die "error on hot-unplugging device '$deviceid'\n";
4199 }
4200
4201 sub qemu_findorcreatescsihw {
4202     my ($storecfg, $conf, $vmid, $device) = @_;
4203
4204     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4205
4206     my $scsihwid="$controller_prefix$controller";
4207     my $devices_list = vm_devices_list($vmid);
4208
4209     if(!defined($devices_list->{$scsihwid})) {
4210         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device);
4211     }
4212
4213     return 1;
4214 }
4215
4216 sub qemu_deletescsihw {
4217     my ($conf, $vmid, $opt) = @_;
4218
4219     my $device = parse_drive($opt, $conf->{$opt});
4220
4221     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
4222         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
4223         return 1;
4224     }
4225
4226     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4227
4228     my $devices_list = vm_devices_list($vmid);
4229     foreach my $opt (keys %{$devices_list}) {
4230         if (PVE::QemuServer::is_valid_drivename($opt)) {
4231             my $drive = PVE::QemuServer::parse_drive($opt, $conf->{$opt});
4232             if($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
4233                 return 1;
4234             }
4235         }
4236     }
4237
4238     my $scsihwid="scsihw$controller";
4239
4240     vm_deviceunplug($vmid, $conf, $scsihwid);
4241
4242     return 1;
4243 }
4244
4245 sub qemu_add_pci_bridge {
4246     my ($storecfg, $conf, $vmid, $device) = @_;
4247
4248     my $bridges = {};
4249
4250     my $bridgeid;
4251
4252     print_pci_addr($device, $bridges);
4253
4254     while (my ($k, $v) = each %$bridges) {
4255         $bridgeid = $k;
4256     }
4257     return 1 if !defined($bridgeid) || $bridgeid < 1;
4258
4259     my $bridge = "pci.$bridgeid";
4260     my $devices_list = vm_devices_list($vmid);
4261
4262     if (!defined($devices_list->{$bridge})) {
4263         vm_deviceplug($storecfg, $conf, $vmid, $bridge);
4264     }
4265
4266     return 1;
4267 }
4268
4269 sub qemu_set_link_status {
4270     my ($vmid, $device, $up) = @_;
4271
4272     vm_mon_cmd($vmid, "set_link", name => $device,
4273                up => $up ? JSON::true : JSON::false);
4274 }
4275
4276 sub qemu_netdevadd {
4277     my ($vmid, $conf, $device, $deviceid) = @_;
4278
4279     my $netdev = print_netdev_full($vmid, $conf, $device, $deviceid, 1);
4280     my %options =  split(/[=,]/, $netdev);
4281
4282     vm_mon_cmd($vmid, "netdev_add",  %options);
4283     return 1;
4284 }
4285
4286 sub qemu_netdevdel {
4287     my ($vmid, $deviceid) = @_;
4288
4289     vm_mon_cmd($vmid, "netdev_del", id => $deviceid);
4290 }
4291
4292 sub qemu_usb_hotplug {
4293     my ($storecfg, $conf, $vmid, $deviceid, $device) = @_;
4294
4295     return if !$device;
4296
4297     # remove the old one first
4298     vm_deviceunplug($vmid, $conf, $deviceid);
4299
4300     # check if xhci controller is necessary and available
4301     if ($device->{usb3}) {
4302
4303         my $devicelist = vm_devices_list($vmid);
4304
4305         if (!$devicelist->{xhci}) {
4306             my $pciaddr = print_pci_addr("xhci");
4307             qemu_deviceadd($vmid, "nec-usb-xhci,id=xhci$pciaddr");
4308         }
4309     }
4310     my $d = parse_usb_device($device->{host});
4311     $d->{usb3} = $device->{usb3};
4312
4313     # add the new one
4314     vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $d);
4315 }
4316
4317 sub qemu_cpu_hotplug {
4318     my ($vmid, $conf, $vcpus) = @_;
4319
4320     my $machine_type = PVE::QemuServer::get_current_qemu_machine($vmid);
4321
4322     my $sockets = 1;
4323     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
4324     $sockets = $conf->{sockets} if  $conf->{sockets};
4325     my $cores = $conf->{cores} || 1;
4326     my $maxcpus = $sockets * $cores;
4327
4328     $vcpus = $maxcpus if !$vcpus;
4329
4330     die "you can't add more vcpus than maxcpus\n"
4331         if $vcpus > $maxcpus;
4332
4333     my $currentvcpus = $conf->{vcpus} || $maxcpus;
4334
4335     if ($vcpus < $currentvcpus) {
4336
4337         if (qemu_machine_feature_enabled ($machine_type, undef, 2, 7)) {
4338
4339             for (my $i = $currentvcpus; $i > $vcpus; $i--) {
4340                 qemu_devicedel($vmid, "cpu$i");
4341                 my $retry = 0;
4342                 my $currentrunningvcpus = undef;
4343                 while (1) {
4344                     $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
4345                     last if scalar(@{$currentrunningvcpus}) == $i-1;
4346                     raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
4347                     $retry++;
4348                     sleep 1;
4349                 }
4350                 #update conf after each succesfull cpu unplug
4351                 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4352                 PVE::QemuConfig->write_config($vmid, $conf);
4353             }
4354         } else {
4355             die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
4356         }
4357
4358         return;
4359     }
4360
4361     my $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
4362     die "vcpus in running vm does not match its configuration\n"
4363         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
4364
4365     if (qemu_machine_feature_enabled ($machine_type, undef, 2, 7)) {
4366
4367         for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
4368             my $cpustr = print_cpu_device($conf, $i);
4369             qemu_deviceadd($vmid, $cpustr);
4370
4371             my $retry = 0;
4372             my $currentrunningvcpus = undef;
4373             while (1) {
4374                 $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
4375                 last if scalar(@{$currentrunningvcpus}) == $i;
4376                 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
4377                 sleep 1;
4378                 $retry++;
4379             }
4380             #update conf after each succesfull cpu hotplug
4381             $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4382             PVE::QemuConfig->write_config($vmid, $conf);
4383         }
4384     } else {
4385
4386         for (my $i = $currentvcpus; $i < $vcpus; $i++) {
4387             vm_mon_cmd($vmid, "cpu-add", id => int($i));
4388         }
4389     }
4390 }
4391
4392 sub qemu_block_set_io_throttle {
4393     my ($vmid, $deviceid,
4394         $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
4395         $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max,
4396         $bps_max_length, $bps_rd_max_length, $bps_wr_max_length,
4397         $iops_max_length, $iops_rd_max_length, $iops_wr_max_length) = @_;
4398
4399     return if !check_running($vmid) ;
4400
4401     vm_mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
4402         bps => int($bps),
4403         bps_rd => int($bps_rd),
4404         bps_wr => int($bps_wr),
4405         iops => int($iops),
4406         iops_rd => int($iops_rd),
4407         iops_wr => int($iops_wr),
4408         bps_max => int($bps_max),
4409         bps_rd_max => int($bps_rd_max),
4410         bps_wr_max => int($bps_wr_max),
4411         iops_max => int($iops_max),
4412         iops_rd_max => int($iops_rd_max),
4413         iops_wr_max => int($iops_wr_max),
4414         bps_max_length => int($bps_max_length),
4415         bps_rd_max_length => int($bps_rd_max_length),
4416         bps_wr_max_length => int($bps_wr_max_length),
4417         iops_max_length => int($iops_max_length),
4418         iops_rd_max_length => int($iops_rd_max_length),
4419         iops_wr_max_length => int($iops_wr_max_length),
4420     );
4421
4422 }
4423
4424 # old code, only used to shutdown old VM after update
4425 sub __read_avail {
4426     my ($fh, $timeout) = @_;
4427
4428     my $sel = new IO::Select;
4429     $sel->add($fh);
4430
4431     my $res = '';
4432     my $buf;
4433
4434     my @ready;
4435     while (scalar (@ready = $sel->can_read($timeout))) {
4436         my $count;
4437         if ($count = $fh->sysread($buf, 8192)) {
4438             if ($buf =~ /^(.*)\(qemu\) $/s) {
4439                 $res .= $1;
4440                 last;
4441             } else {
4442                 $res .= $buf;
4443             }
4444         } else {
4445             if (!defined($count)) {
4446                 die "$!\n";
4447             }
4448             last;
4449         }
4450     }
4451
4452     die "monitor read timeout\n" if !scalar(@ready);
4453
4454     return $res;
4455 }
4456
4457 sub qemu_block_resize {
4458     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4459
4460     my $running = check_running($vmid);
4461
4462     $size = 0 if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4463
4464     return if !$running;
4465
4466     vm_mon_cmd($vmid, "block_resize", device => $deviceid, size => int($size));
4467
4468 }
4469
4470 sub qemu_volume_snapshot {
4471     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4472
4473     my $running = check_running($vmid);
4474
4475     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4476         vm_mon_cmd($vmid, 'blockdev-snapshot-internal-sync', device => $deviceid, name => $snap);
4477     } else {
4478         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4479     }
4480 }
4481
4482 sub qemu_volume_snapshot_delete {
4483     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4484
4485     my $running = check_running($vmid);
4486
4487     if($running) {
4488
4489         $running = undef;
4490         my $conf = PVE::QemuConfig->load_config($vmid);
4491         foreach_drive($conf, sub {
4492             my ($ds, $drive) = @_;
4493             $running = 1 if $drive->{file} eq $volid;
4494         });
4495     }
4496
4497     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4498         vm_mon_cmd($vmid, 'blockdev-snapshot-delete-internal-sync', device => $deviceid, name => $snap);
4499     } else {
4500         PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4501     }
4502 }
4503
4504 sub set_migration_caps {
4505     my ($vmid) = @_;
4506
4507     my $cap_ref = [];
4508
4509     my $enabled_cap = {
4510         "auto-converge" => 1,
4511         "xbzrle" => 1,
4512         "x-rdma-pin-all" => 0,
4513         "zero-blocks" => 0,
4514         "compress" => 0
4515     };
4516
4517     my $supported_capabilities = vm_mon_cmd_nocheck($vmid, "query-migrate-capabilities");
4518
4519     for my $supported_capability (@$supported_capabilities) {
4520         push @$cap_ref, {
4521             capability => $supported_capability->{capability},
4522             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4523         };
4524     }
4525
4526     vm_mon_cmd_nocheck($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4527 }
4528
4529 my $fast_plug_option = {
4530     'lock' => 1,
4531     'name' => 1,
4532     'onboot' => 1,
4533     'shares' => 1,
4534     'startup' => 1,
4535     'description' => 1,
4536     'protection' => 1,
4537     'vmstatestorage' => 1,
4538 };
4539
4540 # hotplug changes in [PENDING]
4541 # $selection hash can be used to only apply specified options, for
4542 # example: { cores => 1 } (only apply changed 'cores')
4543 # $errors ref is used to return error messages
4544 sub vmconfig_hotplug_pending {
4545     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4546
4547     my $defaults = load_defaults();
4548
4549     # commit values which do not have any impact on running VM first
4550     # Note: those option cannot raise errors, we we do not care about
4551     # $selection and always apply them.
4552
4553     my $add_error = sub {
4554         my ($opt, $msg) = @_;
4555         $errors->{$opt} = "hotplug problem - $msg";
4556     };
4557
4558     my $changes = 0;
4559     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4560         if ($fast_plug_option->{$opt}) {
4561             $conf->{$opt} = $conf->{pending}->{$opt};
4562             delete $conf->{pending}->{$opt};
4563             $changes = 1;
4564         }
4565     }
4566
4567     if ($changes) {
4568         PVE::QemuConfig->write_config($vmid, $conf);
4569         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4570     }
4571
4572     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4573
4574     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4575     while (my ($opt, $force) = each %$pending_delete_hash) {
4576         next if $selection && !$selection->{$opt};
4577         eval {
4578             if ($opt eq 'hotplug') {
4579                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
4580             } elsif ($opt eq 'tablet') {
4581                 die "skip\n" if !$hotplug_features->{usb};
4582                 if ($defaults->{tablet}) {
4583                     vm_deviceplug($storecfg, $conf, $vmid, $opt);
4584                 } else {
4585                     vm_deviceunplug($vmid, $conf, $opt);
4586                 }
4587             } elsif ($opt =~ m/^usb\d+/) {
4588                 die "skip\n";
4589                 # since we cannot reliably hot unplug usb devices
4590                 # we are disabling it
4591                 die "skip\n" if !$hotplug_features->{usb} || $conf->{$opt} =~ m/spice/i;
4592                 vm_deviceunplug($vmid, $conf, $opt);
4593             } elsif ($opt eq 'vcpus') {
4594                 die "skip\n" if !$hotplug_features->{cpu};
4595                 qemu_cpu_hotplug($vmid, $conf, undef);
4596             } elsif ($opt eq 'balloon') {
4597                 # enable balloon device is not hotpluggable
4598                 die "skip\n" if defined($conf->{balloon}) && $conf->{balloon} == 0;
4599                 # here we reset the ballooning value to memory
4600                 my $balloon = $conf->{memory} || $defaults->{memory};
4601                 vm_mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4602             } elsif ($fast_plug_option->{$opt}) {
4603                 # do nothing
4604             } elsif ($opt =~ m/^net(\d+)$/) {
4605                 die "skip\n" if !$hotplug_features->{network};
4606                 vm_deviceunplug($vmid, $conf, $opt);
4607             } elsif (is_valid_drivename($opt)) {
4608                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4609                 vm_deviceunplug($vmid, $conf, $opt);
4610                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4611             } elsif ($opt =~ m/^memory$/) {
4612                 die "skip\n" if !$hotplug_features->{memory};
4613                 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
4614             } elsif ($opt eq 'cpuunits') {
4615                 cgroups_write("cpu", $vmid, "cpu.shares", $defaults->{cpuunits});
4616             } elsif ($opt eq 'cpulimit') {
4617                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", -1);
4618             } else {
4619                 die "skip\n";
4620             }
4621         };
4622         if (my $err = $@) {
4623             &$add_error($opt, $err) if $err ne "skip\n";
4624         } else {
4625             # save new config if hotplug was successful
4626             delete $conf->{$opt};
4627             vmconfig_undelete_pending_option($conf, $opt);
4628             PVE::QemuConfig->write_config($vmid, $conf);
4629             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4630         }
4631     }
4632
4633     my $apply_pending_cloudinit;
4634     $apply_pending_cloudinit = sub {
4635         my ($key, $value) = @_;
4636         $apply_pending_cloudinit = sub {}; # once is enough
4637
4638         my @cloudinit_opts = keys %$confdesc_cloudinit;
4639         foreach my $opt (keys %{$conf->{pending}}) {
4640             next if !grep { $_ eq $opt } @cloudinit_opts;
4641             $conf->{$opt} = delete $conf->{pending}->{$opt};
4642         }
4643
4644         my $new_conf = { %$conf };
4645         $new_conf->{$key} = $value;
4646         PVE::QemuServer::Cloudinit::generate_cloudinitconfig($new_conf, $vmid);
4647     };
4648
4649     foreach my $opt (keys %{$conf->{pending}}) {
4650         next if $selection && !$selection->{$opt};
4651         my $value = $conf->{pending}->{$opt};
4652         eval {
4653             if ($opt eq 'hotplug') {
4654                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
4655             } elsif ($opt eq 'tablet') {
4656                 die "skip\n" if !$hotplug_features->{usb};
4657                 if ($value == 1) {
4658                     vm_deviceplug($storecfg, $conf, $vmid, $opt);
4659                 } elsif ($value == 0) {
4660                     vm_deviceunplug($vmid, $conf, $opt);
4661                 }
4662             } elsif ($opt =~ m/^usb\d+$/) {
4663                 die "skip\n";
4664                 # since we cannot reliably hot unplug usb devices
4665                 # we are disabling it
4666                 die "skip\n" if !$hotplug_features->{usb} || $value =~ m/spice/i;
4667                 my $d = eval { PVE::JSONSchema::parse_property_string($usbdesc->{format}, $value) };
4668                 die "skip\n" if !$d;
4669                 qemu_usb_hotplug($storecfg, $conf, $vmid, $opt, $d);
4670             } elsif ($opt eq 'vcpus') {
4671                 die "skip\n" if !$hotplug_features->{cpu};
4672                 qemu_cpu_hotplug($vmid, $conf, $value);
4673             } elsif ($opt eq 'balloon') {
4674                 # enable/disable balloning device is not hotpluggable
4675                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
4676                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
4677                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
4678
4679                 # allow manual ballooning if shares is set to zero
4680                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
4681                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
4682                     vm_mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4683                 }
4684             } elsif ($opt =~ m/^net(\d+)$/) {
4685                 # some changes can be done without hotplug
4686                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
4687                                     $vmid, $opt, $value);
4688             } elsif (is_valid_drivename($opt)) {
4689                 # some changes can be done without hotplug
4690                 my $drive = parse_drive($opt, $value);
4691                 if (drive_is_cloudinit($drive)) {
4692                     &$apply_pending_cloudinit($opt, $value);
4693                 }
4694                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
4695                                      $vmid, $opt, $value, 1);
4696             } elsif ($opt =~ m/^memory$/) { #dimms
4697                 die "skip\n" if !$hotplug_features->{memory};
4698                 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
4699             } elsif ($opt eq 'cpuunits') {
4700                 cgroups_write("cpu", $vmid, "cpu.shares", $conf->{pending}->{$opt});
4701             } elsif ($opt eq 'cpulimit') {
4702                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
4703                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", $cpulimit);
4704             } else {
4705                 die "skip\n";  # skip non-hot-pluggable options
4706             }
4707         };
4708         if (my $err = $@) {
4709             &$add_error($opt, $err) if $err ne "skip\n";
4710         } else {
4711             # save new config if hotplug was successful
4712             $conf->{$opt} = $value;
4713             delete $conf->{pending}->{$opt};
4714             PVE::QemuConfig->write_config($vmid, $conf);
4715             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4716         }
4717     }
4718 }
4719
4720 sub try_deallocate_drive {
4721     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
4722
4723     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
4724         my $volid = $drive->{file};
4725         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
4726             my $sid = PVE::Storage::parse_volume_id($volid);
4727             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
4728
4729             # check if the disk is really unused
4730             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
4731                 if is_volume_in_use($storecfg, $conf, $key, $volid);
4732             PVE::Storage::vdisk_free($storecfg, $volid);
4733             return 1;
4734         } else {
4735             # If vm is not owner of this disk remove from config
4736             return 1;
4737         }
4738     }
4739
4740     return undef;
4741 }
4742
4743 sub vmconfig_delete_or_detach_drive {
4744     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
4745
4746     my $drive = parse_drive($opt, $conf->{$opt});
4747
4748     my $rpcenv = PVE::RPCEnvironment::get();
4749     my $authuser = $rpcenv->get_user();
4750
4751     if ($force) {
4752         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
4753         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
4754     } else {
4755         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
4756     }
4757 }
4758
4759 sub vmconfig_apply_pending {
4760     my ($vmid, $conf, $storecfg) = @_;
4761
4762     # cold plug
4763
4764     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4765     while (my ($opt, $force) = each %$pending_delete_hash) {
4766         die "internal error" if $opt =~ m/^unused/;
4767         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4768         if (!defined($conf->{$opt})) {
4769             vmconfig_undelete_pending_option($conf, $opt);
4770             PVE::QemuConfig->write_config($vmid, $conf);
4771         } elsif (is_valid_drivename($opt)) {
4772             vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4773             vmconfig_undelete_pending_option($conf, $opt);
4774             delete $conf->{$opt};
4775             PVE::QemuConfig->write_config($vmid, $conf);
4776         } else {
4777             vmconfig_undelete_pending_option($conf, $opt);
4778             delete $conf->{$opt};
4779             PVE::QemuConfig->write_config($vmid, $conf);
4780         }
4781     }
4782
4783     $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4784
4785     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4786         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4787
4788         if (defined($conf->{$opt}) && ($conf->{$opt} eq $conf->{pending}->{$opt})) {
4789             # skip if nothing changed
4790         } elsif (is_valid_drivename($opt)) {
4791             vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
4792                 if defined($conf->{$opt});
4793             $conf->{$opt} = $conf->{pending}->{$opt};
4794         } else {
4795             $conf->{$opt} = $conf->{pending}->{$opt};
4796         }
4797
4798         delete $conf->{pending}->{$opt};
4799         PVE::QemuConfig->write_config($vmid, $conf);
4800     }
4801 }
4802
4803 my $safe_num_ne = sub {
4804     my ($a, $b) = @_;
4805
4806     return 0 if !defined($a) && !defined($b);
4807     return 1 if !defined($a);
4808     return 1 if !defined($b);
4809
4810     return $a != $b;
4811 };
4812
4813 my $safe_string_ne = sub {
4814     my ($a, $b) = @_;
4815
4816     return 0 if !defined($a) && !defined($b);
4817     return 1 if !defined($a);
4818     return 1 if !defined($b);
4819
4820     return $a ne $b;
4821 };
4822
4823 sub vmconfig_update_net {
4824     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value) = @_;
4825
4826     my $newnet = parse_net($value);
4827
4828     if ($conf->{$opt}) {
4829         my $oldnet = parse_net($conf->{$opt});
4830
4831         if (&$safe_string_ne($oldnet->{model}, $newnet->{model}) ||
4832             &$safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
4833             &$safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
4834             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
4835
4836             # for non online change, we try to hot-unplug
4837             die "skip\n" if !$hotplug;
4838             vm_deviceunplug($vmid, $conf, $opt);
4839         } else {
4840
4841             die "internal error" if $opt !~ m/net(\d+)/;
4842             my $iface = "tap${vmid}i$1";
4843
4844             if (&$safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
4845                 &$safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
4846                 &$safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
4847                 &$safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
4848                 PVE::Network::tap_unplug($iface);
4849                 PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
4850             } elsif (&$safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
4851                 # Rate can be applied on its own but any change above needs to
4852                 # include the rate in tap_plug since OVS resets everything.
4853                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
4854             }
4855
4856             if (&$safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
4857                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
4858             }
4859
4860             return 1;
4861         }
4862     }
4863
4864     if ($hotplug) {
4865         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet);
4866     } else {
4867         die "skip\n";
4868     }
4869 }
4870
4871 sub vmconfig_update_disk {
4872     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $force) = @_;
4873
4874     # fixme: do we need force?
4875
4876     my $drive = parse_drive($opt, $value);
4877
4878     if ($conf->{$opt}) {
4879
4880         if (my $old_drive = parse_drive($opt, $conf->{$opt}))  {
4881
4882             my $media = $drive->{media} || 'disk';
4883             my $oldmedia = $old_drive->{media} || 'disk';
4884             die "unable to change media type\n" if $media ne $oldmedia;
4885
4886             if (!drive_is_cdrom($old_drive)) {
4887
4888                 if ($drive->{file} ne $old_drive->{file}) {
4889
4890                     die "skip\n" if !$hotplug;
4891
4892                     # unplug and register as unused
4893                     vm_deviceunplug($vmid, $conf, $opt);
4894                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
4895
4896                 } else {
4897                     # update existing disk
4898
4899                     # skip non hotpluggable value
4900                     if (&$safe_string_ne($drive->{discard}, $old_drive->{discard}) ||
4901                         &$safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
4902                         &$safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
4903                         &$safe_string_ne($drive->{cache}, $old_drive->{cache})) {
4904                         die "skip\n";
4905                     }
4906
4907                     # apply throttle
4908                     if (&$safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
4909                         &$safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
4910                         &$safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
4911                         &$safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
4912                         &$safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
4913                         &$safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
4914                         &$safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
4915                         &$safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
4916                         &$safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
4917                         &$safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
4918                         &$safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
4919                         &$safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max}) ||
4920                         &$safe_num_ne($drive->{bps_max_length}, $old_drive->{bps_max_length}) ||
4921                         &$safe_num_ne($drive->{bps_rd_max_length}, $old_drive->{bps_rd_max_length}) ||
4922                         &$safe_num_ne($drive->{bps_wr_max_length}, $old_drive->{bps_wr_max_length}) ||
4923                         &$safe_num_ne($drive->{iops_max_length}, $old_drive->{iops_max_length}) ||
4924                         &$safe_num_ne($drive->{iops_rd_max_length}, $old_drive->{iops_rd_max_length}) ||
4925                         &$safe_num_ne($drive->{iops_wr_max_length}, $old_drive->{iops_wr_max_length})) {
4926
4927                         qemu_block_set_io_throttle($vmid,"drive-$opt",
4928                                                    ($drive->{mbps} || 0)*1024*1024,
4929                                                    ($drive->{mbps_rd} || 0)*1024*1024,
4930                                                    ($drive->{mbps_wr} || 0)*1024*1024,
4931                                                    $drive->{iops} || 0,
4932                                                    $drive->{iops_rd} || 0,
4933                                                    $drive->{iops_wr} || 0,
4934                                                    ($drive->{mbps_max} || 0)*1024*1024,
4935                                                    ($drive->{mbps_rd_max} || 0)*1024*1024,
4936                                                    ($drive->{mbps_wr_max} || 0)*1024*1024,
4937                                                    $drive->{iops_max} || 0,
4938                                                    $drive->{iops_rd_max} || 0,
4939                                                    $drive->{iops_wr_max} || 0,
4940                                                    $drive->{bps_max_length} || 1,
4941                                                    $drive->{bps_rd_max_length} || 1,
4942                                                    $drive->{bps_wr_max_length} || 1,
4943                                                    $drive->{iops_max_length} || 1,
4944                                                    $drive->{iops_rd_max_length} || 1,
4945                                                    $drive->{iops_wr_max_length} || 1);
4946
4947                     }
4948
4949                     return 1;
4950                 }
4951
4952             } else { # cdrom
4953
4954                 if ($drive->{file} eq 'none') {
4955                     vm_mon_cmd($vmid, "eject",force => JSON::true,device => "drive-$opt");
4956                     if (drive_is_cloudinit($old_drive)) {
4957                         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive);
4958                     }
4959                 } else {
4960                     my $path = get_iso_path($storecfg, $vmid, $drive->{file});
4961                     vm_mon_cmd($vmid, "eject", force => JSON::true,device => "drive-$opt"); # force eject if locked
4962                     vm_mon_cmd($vmid, "change", device => "drive-$opt",target => "$path") if $path;
4963                 }
4964
4965                 return 1;
4966             }
4967         }
4968     }
4969
4970     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
4971     # hotplug new disks
4972     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
4973     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive);
4974 }
4975
4976 sub vm_start {
4977     my ($storecfg, $vmid, $statefile, $skiplock, $migratedfrom, $paused,
4978         $forcemachine, $spice_ticket, $migration_network, $migration_type, $targetstorage) = @_;
4979
4980     PVE::QemuConfig->lock_config($vmid, sub {
4981         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
4982
4983         die "you can't start a vm if it's a template\n" if PVE::QemuConfig->is_template($conf);
4984
4985         PVE::QemuConfig->check_lock($conf) if !$skiplock;
4986
4987         die "VM $vmid already running\n" if check_running($vmid, undef, $migratedfrom);
4988
4989         if (!$statefile && scalar(keys %{$conf->{pending}})) {
4990             vmconfig_apply_pending($vmid, $conf, $storecfg);
4991             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4992         }
4993
4994         PVE::QemuServer::Cloudinit::generate_cloudinitconfig($conf, $vmid);
4995
4996         my $defaults = load_defaults();
4997
4998         # set environment variable useful inside network script
4999         $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
5000
5001         my $local_volumes = {};
5002
5003         if ($targetstorage) {
5004             foreach_drive($conf, sub {
5005                 my ($ds, $drive) = @_;
5006
5007                 return if drive_is_cdrom($drive);
5008
5009                 my $volid = $drive->{file};
5010
5011                 return if !$volid;
5012
5013                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
5014
5015                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5016                 return if $scfg->{shared};
5017                 $local_volumes->{$ds} = [$volid, $storeid, $volname];
5018             });
5019
5020             my $format = undef;
5021
5022             foreach my $opt (sort keys %$local_volumes) {
5023
5024                 my ($volid, $storeid, $volname) = @{$local_volumes->{$opt}};
5025                 my $drive = parse_drive($opt, $conf->{$opt});
5026
5027                 #if remote storage is specified, use default format
5028                 if ($targetstorage && $targetstorage ne "1") {
5029                     $storeid = $targetstorage;
5030                     my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
5031                     $format = $defFormat;
5032                 } else {
5033                     #else we use same format than original
5034                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5035                     $format = qemu_img_format($scfg, $volid);
5036                 }
5037
5038                 my $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $format, undef, ($drive->{size}/1024));
5039                 my $newdrive = $drive;
5040                 $newdrive->{format} = $format;
5041                 $newdrive->{file} = $newvolid;
5042                 my $drivestr = PVE::QemuServer::print_drive($vmid, $newdrive);
5043                 $local_volumes->{$opt} = $drivestr;
5044                 #pass drive to conf for command line
5045                 $conf->{$opt} = $drivestr;
5046             }
5047         }
5048
5049         my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine);
5050
5051         my $migrate_port = 0;
5052         my $migrate_uri;
5053         if ($statefile) {
5054             if ($statefile eq 'tcp') {
5055                 my $localip = "localhost";
5056                 my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
5057                 my $nodename = PVE::INotify::nodename();
5058
5059                 if (!defined($migration_type)) {
5060                     if (defined($datacenterconf->{migration}->{type})) {
5061                         $migration_type = $datacenterconf->{migration}->{type};
5062                     } else {
5063                         $migration_type = 'secure';
5064                     }
5065                 }
5066
5067                 if ($migration_type eq 'insecure') {
5068                     my $migrate_network_addr = PVE::Cluster::get_local_migration_ip($migration_network);
5069                     if ($migrate_network_addr) {
5070                         $localip = $migrate_network_addr;
5071                     } else {
5072                         $localip = PVE::Cluster::remote_node_ip($nodename, 1);
5073                     }
5074
5075                     $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5076                 }
5077
5078                 my $pfamily = PVE::Tools::get_host_address_family($nodename);
5079                 $migrate_port = PVE::Tools::next_migrate_port($pfamily);
5080                 $migrate_uri = "tcp:${localip}:${migrate_port}";
5081                 push @$cmd, '-incoming', $migrate_uri;
5082                 push @$cmd, '-S';
5083
5084             } elsif ($statefile eq 'unix') {
5085                 # should be default for secure migrations as a ssh TCP forward
5086                 # tunnel is not deterministic reliable ready and fails regurarly
5087                 # to set up in time, so use UNIX socket forwards
5088                 my $socket_addr = "/run/qemu-server/$vmid.migrate";
5089                 unlink $socket_addr;
5090
5091                 $migrate_uri = "unix:$socket_addr";
5092
5093                 push @$cmd, '-incoming', $migrate_uri;
5094                 push @$cmd, '-S';
5095
5096             } else {
5097                 push @$cmd, '-loadstate', $statefile;
5098             }
5099         } elsif ($paused) {
5100             push @$cmd, '-S';
5101         }
5102
5103         # host pci devices
5104         for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
5105           my $d = parse_hostpci($conf->{"hostpci$i"});
5106           next if !$d;
5107           my $pcidevices = $d->{pciid};
5108           foreach my $pcidevice (@$pcidevices) {
5109                 my $pciid = $pcidevice->{id}.".".$pcidevice->{function};
5110
5111                 my $info = pci_device_info("0000:$pciid");
5112                 die "IOMMU not present\n" if !check_iommu_support();
5113                 die "no pci device info for device '$pciid'\n" if !$info;
5114                 die "can't unbind/bind pci group to vfio '$pciid'\n" if !pci_dev_group_bind_to_vfio($pciid);
5115                 die "can't reset pci device '$pciid'\n" if $info->{has_fl_reset} and !pci_dev_reset($info);
5116           }
5117         }
5118
5119         PVE::Storage::activate_volumes($storecfg, $vollist);
5120
5121         if (!check_running($vmid, 1)) {
5122             eval {
5123                 run_command(['/bin/systemctl', 'stop', "$vmid.scope"],
5124                     outfunc => sub {}, errfunc => sub {});
5125             };
5126         }
5127
5128         my $cpuunits = defined($conf->{cpuunits}) ? $conf->{cpuunits}
5129                                                   : $defaults->{cpuunits};
5130
5131         my $start_timeout = $conf->{hugepages} ? 300 : 30;
5132         my %run_params = (timeout => $statefile ? undef : $start_timeout, umask => 0077);
5133
5134         my %properties = (
5135             Slice => 'qemu.slice',
5136             KillMode => 'none',
5137             CPUShares => $cpuunits
5138         );
5139
5140         if (my $cpulimit = $conf->{cpulimit}) {
5141             $properties{CPUQuota} = int($cpulimit * 100);
5142         }
5143         $properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
5144
5145         my $run_qemu = sub {
5146             PVE::Tools::run_fork sub {
5147                 PVE::Systemd::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
5148                 run_command($cmd, %run_params);
5149             };
5150         };
5151
5152         if ($conf->{hugepages}) {
5153
5154             my $code = sub {
5155                 my $hugepages_topology = PVE::QemuServer::Memory::hugepages_topology($conf);
5156                 my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
5157
5158                 PVE::QemuServer::Memory::hugepages_mount();
5159                 PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
5160
5161                 eval { $run_qemu->() };
5162                 if (my $err = $@) {
5163                     PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology);
5164                     die $err;
5165                 }
5166
5167                 PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology);
5168             };
5169             eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
5170
5171         } else {
5172             eval { $run_qemu->() };
5173         }
5174
5175         if (my $err = $@) {
5176             # deactivate volumes if start fails
5177             eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
5178             die "start failed: $err";
5179         }
5180
5181         print "migration listens on $migrate_uri\n" if $migrate_uri;
5182
5183         if ($statefile && $statefile ne 'tcp' && $statefile ne 'unix')  {
5184             eval { vm_mon_cmd_nocheck($vmid, "cont"); };
5185             warn $@ if $@;
5186         }
5187
5188         #start nbd server for storage migration
5189         if ($targetstorage) {
5190             my $nodename = PVE::INotify::nodename();
5191             my $migrate_network_addr = PVE::Cluster::get_local_migration_ip($migration_network);
5192             my $localip = $migrate_network_addr ? $migrate_network_addr : PVE::Cluster::remote_node_ip($nodename, 1);
5193             my $pfamily = PVE::Tools::get_host_address_family($nodename);
5194             $migrate_port = PVE::Tools::next_migrate_port($pfamily);
5195
5196             vm_mon_cmd_nocheck($vmid, "nbd-server-start", addr => { type => 'inet', data => { host => "${localip}", port => "${migrate_port}" } } );
5197
5198             $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5199
5200             foreach my $opt (sort keys %$local_volumes) {
5201                 my $volid = $local_volumes->{$opt};
5202                 vm_mon_cmd_nocheck($vmid, "nbd-server-add", device => "drive-$opt", writable => JSON::true );
5203                 my $migrate_storage_uri = "nbd:${localip}:${migrate_port}:exportname=drive-$opt";
5204                 print "storage migration listens on $migrate_storage_uri volume:$volid\n";
5205             }
5206         }
5207
5208         if ($migratedfrom) {
5209             eval {
5210                 set_migration_caps($vmid);
5211             };
5212             warn $@ if $@;
5213
5214             if ($spice_port) {
5215                 print "spice listens on port $spice_port\n";
5216                 if ($spice_ticket) {
5217                     vm_mon_cmd_nocheck($vmid, "set_password", protocol => 'spice', password => $spice_ticket);
5218                     vm_mon_cmd_nocheck($vmid, "expire_password", protocol => 'spice', time => "+30");
5219                 }
5220             }
5221
5222         } else {
5223             vm_mon_cmd_nocheck($vmid, "balloon", value => $conf->{balloon}*1024*1024)
5224                 if !$statefile && $conf->{balloon};
5225
5226             foreach my $opt (keys %$conf) {
5227                 next if $opt !~  m/^net\d+$/;
5228                 my $nicconf = parse_net($conf->{$opt});
5229                 qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
5230             }
5231         }
5232
5233         vm_mon_cmd_nocheck($vmid, 'qom-set',
5234                     path => "machine/peripheral/balloon0",
5235                     property => "guest-stats-polling-interval",
5236                     value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
5237
5238     });
5239 }
5240
5241 sub vm_mon_cmd {
5242     my ($vmid, $execute, %params) = @_;
5243
5244     my $cmd = { execute => $execute, arguments => \%params };
5245     vm_qmp_command($vmid, $cmd);
5246 }
5247
5248 sub vm_mon_cmd_nocheck {
5249     my ($vmid, $execute, %params) = @_;
5250
5251     my $cmd = { execute => $execute, arguments => \%params };
5252     vm_qmp_command($vmid, $cmd, 1);
5253 }
5254
5255 sub vm_qmp_command {
5256     my ($vmid, $cmd, $nocheck) = @_;
5257
5258     my $res;
5259
5260     my $timeout;
5261     if ($cmd->{arguments} && $cmd->{arguments}->{timeout}) {
5262         $timeout = $cmd->{arguments}->{timeout};
5263         delete $cmd->{arguments}->{timeout};
5264     }
5265
5266     eval {
5267         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
5268         my $sname = qmp_socket($vmid);
5269         if (-e $sname) { # test if VM is reasonambe new and supports qmp/qga
5270             my $qmpclient = PVE::QMPClient->new();
5271
5272             $res = $qmpclient->cmd($vmid, $cmd, $timeout);
5273         } else {
5274             die "unable to open monitor socket\n";
5275         }
5276     };
5277     if (my $err = $@) {
5278         syslog("err", "VM $vmid qmp command failed - $err");
5279         die $err;
5280     }
5281
5282     return $res;
5283 }
5284
5285 sub vm_human_monitor_command {
5286     my ($vmid, $cmdline) = @_;
5287
5288     my $res;
5289
5290     my $cmd = {
5291         execute => 'human-monitor-command',
5292         arguments => { 'command-line' => $cmdline},
5293     };
5294
5295     return vm_qmp_command($vmid, $cmd);
5296 }
5297
5298 sub vm_commandline {
5299     my ($storecfg, $vmid) = @_;
5300
5301     my $conf = PVE::QemuConfig->load_config($vmid);
5302
5303     my $defaults = load_defaults();
5304
5305     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults);
5306
5307     return PVE::Tools::cmd2string($cmd);
5308 }
5309
5310 sub vm_reset {
5311     my ($vmid, $skiplock) = @_;
5312
5313     PVE::QemuConfig->lock_config($vmid, sub {
5314
5315         my $conf = PVE::QemuConfig->load_config($vmid);
5316
5317         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5318
5319         vm_mon_cmd($vmid, "system_reset");
5320     });
5321 }
5322
5323 sub get_vm_volumes {
5324     my ($conf) = @_;
5325
5326     my $vollist = [];
5327     foreach_volid($conf, sub {
5328         my ($volid, $attr) = @_;
5329
5330         return if $volid =~ m|^/|;
5331
5332         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5333         return if !$sid;
5334
5335         push @$vollist, $volid;
5336     });
5337
5338     return $vollist;
5339 }
5340
5341 sub vm_stop_cleanup {
5342     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
5343
5344     eval {
5345
5346         if (!$keepActive) {
5347             my $vollist = get_vm_volumes($conf);
5348             PVE::Storage::deactivate_volumes($storecfg, $vollist);
5349         }
5350
5351         foreach my $ext (qw(mon qmp pid vnc qga)) {
5352             unlink "/var/run/qemu-server/${vmid}.$ext";
5353         }
5354
5355         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
5356     };
5357     warn $@ if $@; # avoid errors - just warn
5358 }
5359
5360 # Note: use $nockeck to skip tests if VM configuration file exists.
5361 # We need that when migration VMs to other nodes (files already moved)
5362 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
5363 sub vm_stop {
5364     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
5365
5366     $force = 1 if !defined($force) && !$shutdown;
5367
5368     if ($migratedfrom){
5369         my $pid = check_running($vmid, $nocheck, $migratedfrom);
5370         kill 15, $pid if $pid;
5371         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
5372         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
5373         return;
5374     }
5375
5376     PVE::QemuConfig->lock_config($vmid, sub {
5377
5378         my $pid = check_running($vmid, $nocheck);
5379         return if !$pid;
5380
5381         my $conf;
5382         if (!$nocheck) {
5383             $conf = PVE::QemuConfig->load_config($vmid);
5384             PVE::QemuConfig->check_lock($conf) if !$skiplock;
5385             if (!defined($timeout) && $shutdown && $conf->{startup}) {
5386                 my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
5387                 $timeout = $opts->{down} if $opts->{down};
5388             }
5389         }
5390
5391         $timeout = 60 if !defined($timeout);
5392
5393         eval {
5394             if ($shutdown) {
5395                 if (defined($conf) && parse_guest_agent($conf)->{enabled}) {
5396                     vm_qmp_command($vmid, { execute => "guest-shutdown" }, $nocheck);
5397                 } else {
5398                     vm_qmp_command($vmid, { execute => "system_powerdown" }, $nocheck);
5399                 }
5400             } else {
5401                 vm_qmp_command($vmid, { execute => "quit" }, $nocheck);
5402             }
5403         };
5404         my $err = $@;
5405
5406         if (!$err) {
5407             my $count = 0;
5408             while (($count < $timeout) && check_running($vmid, $nocheck)) {
5409                 $count++;
5410                 sleep 1;
5411             }
5412
5413             if ($count >= $timeout) {
5414                 if ($force) {
5415                     warn "VM still running - terminating now with SIGTERM\n";
5416                     kill 15, $pid;
5417                 } else {
5418                     die "VM quit/powerdown failed - got timeout\n";
5419                 }
5420             } else {
5421                 vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5422                 return;
5423             }
5424         } else {
5425             if ($force) {
5426                 warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
5427                 kill 15, $pid;
5428             } else {
5429                 die "VM quit/powerdown failed\n";
5430             }
5431         }
5432
5433         # wait again
5434         $timeout = 10;
5435
5436         my $count = 0;
5437         while (($count < $timeout) && check_running($vmid, $nocheck)) {
5438             $count++;
5439             sleep 1;
5440         }
5441
5442         if ($count >= $timeout) {
5443             warn "VM still running - terminating now with SIGKILL\n";
5444             kill 9, $pid;
5445             sleep 1;
5446         }
5447
5448         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5449    });
5450 }
5451
5452 sub vm_suspend {
5453     my ($vmid, $skiplock) = @_;
5454
5455     PVE::QemuConfig->lock_config($vmid, sub {
5456
5457         my $conf = PVE::QemuConfig->load_config($vmid);
5458
5459         PVE::QemuConfig->check_lock($conf)
5460             if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
5461
5462         vm_mon_cmd($vmid, "stop");
5463     });
5464 }
5465
5466 sub vm_resume {
5467     my ($vmid, $skiplock, $nocheck) = @_;
5468
5469     PVE::QemuConfig->lock_config($vmid, sub {
5470
5471         my $res = vm_mon_cmd($vmid, 'query-status');
5472         my $resume_cmd = 'cont';
5473
5474         if ($res->{status} && $res->{status} eq 'suspended') {
5475             $resume_cmd = 'system_wakeup';
5476         }
5477
5478         if (!$nocheck) {
5479
5480             my $conf = PVE::QemuConfig->load_config($vmid);
5481
5482             PVE::QemuConfig->check_lock($conf)
5483                 if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
5484
5485             vm_mon_cmd($vmid, $resume_cmd);
5486
5487         } else {
5488             vm_mon_cmd_nocheck($vmid, $resume_cmd);
5489         }
5490     });
5491 }
5492
5493 sub vm_sendkey {
5494     my ($vmid, $skiplock, $key) = @_;
5495
5496     PVE::QemuConfig->lock_config($vmid, sub {
5497
5498         my $conf = PVE::QemuConfig->load_config($vmid);
5499
5500         # there is no qmp command, so we use the human monitor command
5501         vm_human_monitor_command($vmid, "sendkey $key");
5502     });
5503 }
5504
5505 sub vm_destroy {
5506     my ($storecfg, $vmid, $skiplock) = @_;
5507
5508     PVE::QemuConfig->lock_config($vmid, sub {
5509
5510         my $conf = PVE::QemuConfig->load_config($vmid);
5511
5512         if (!check_running($vmid)) {
5513             destroy_vm($storecfg, $vmid, undef, $skiplock);
5514         } else {
5515             die "VM $vmid is running - destroy failed\n";
5516         }
5517     });
5518 }
5519
5520 # pci helpers
5521
5522 sub file_write {
5523     my ($filename, $buf) = @_;
5524
5525     my $fh = IO::File->new($filename, "w");
5526     return undef if !$fh;
5527
5528     my $res = print $fh $buf;
5529
5530     $fh->close();
5531
5532     return $res;
5533 }
5534
5535 sub pci_device_info {
5536     my ($name) = @_;
5537
5538     my $res;
5539
5540     return undef if $name !~ m/^([a-f0-9]{4}):([a-f0-9]{2}):([a-f0-9]{2})\.([a-f0-9])$/;
5541     my ($domain, $bus, $slot, $func) = ($1, $2, $3, $4);
5542
5543     my $irq = file_read_firstline("$pcisysfs/devices/$name/irq");
5544     return undef if !defined($irq) || $irq !~ m/^\d+$/;
5545
5546     my $vendor = file_read_firstline("$pcisysfs/devices/$name/vendor");
5547     return undef if !defined($vendor) || $vendor !~ s/^0x//;
5548
5549     my $product = file_read_firstline("$pcisysfs/devices/$name/device");
5550     return undef if !defined($product) || $product !~ s/^0x//;
5551
5552     $res = {
5553         name => $name,
5554         vendor => $vendor,
5555         product => $product,
5556         domain => $domain,
5557         bus => $bus,
5558         slot => $slot,
5559         func => $func,
5560         irq => $irq,
5561         has_fl_reset => -f "$pcisysfs/devices/$name/reset" || 0,
5562     };
5563
5564     return $res;
5565 }
5566
5567 sub pci_dev_reset {
5568     my ($dev) = @_;
5569
5570     my $name = $dev->{name};
5571
5572     my $fn = "$pcisysfs/devices/$name/reset";
5573
5574     return file_write($fn, "1");
5575 }
5576
5577 sub pci_dev_bind_to_vfio {
5578     my ($dev) = @_;
5579
5580     my $name = $dev->{name};
5581
5582     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
5583
5584     if (!-d $vfio_basedir) {
5585         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
5586     }
5587     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
5588
5589     my $testdir = "$vfio_basedir/$name";
5590     return 1 if -d $testdir;
5591
5592     my $data = "$dev->{vendor} $dev->{product}";
5593     return undef if !file_write("$vfio_basedir/new_id", $data);
5594
5595     my $fn = "$pcisysfs/devices/$name/driver/unbind";
5596     if (!file_write($fn, $name)) {
5597         return undef if -f $fn;
5598     }
5599
5600     $fn = "$vfio_basedir/bind";
5601     if (! -d $testdir) {
5602         return undef if !file_write($fn, $name);
5603     }
5604
5605     return -d $testdir;
5606 }
5607
5608 sub pci_dev_group_bind_to_vfio {
5609     my ($pciid) = @_;
5610
5611     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
5612
5613     if (!-d $vfio_basedir) {
5614         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
5615     }
5616     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
5617
5618     # get IOMMU group devices
5619     opendir(my $D, "$pcisysfs/devices/0000:$pciid/iommu_group/devices/") || die "Cannot open iommu_group: $!\n";
5620       my @devs = grep /^0000:/, readdir($D);
5621     closedir($D);
5622
5623     foreach my $pciid (@devs) {
5624         $pciid =~ m/^([:\.\da-f]+)$/ or die "PCI ID $pciid not valid!\n";
5625
5626         # pci bridges, switches or root ports are not supported
5627         # they have a pci_bus subdirectory so skip them
5628         next if (-e "$pcisysfs/devices/$pciid/pci_bus");
5629
5630         my $info = pci_device_info($1);
5631         pci_dev_bind_to_vfio($info) || die "Cannot bind $pciid to vfio\n";
5632     }
5633
5634     return 1;
5635 }
5636
5637 # vzdump restore implementaion
5638
5639 sub tar_archive_read_firstfile {
5640     my $archive = shift;
5641
5642     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
5643
5644     # try to detect archive type first
5645     my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
5646         die "unable to open file '$archive'\n";
5647     my $firstfile = <$fh>;
5648     kill 15, $pid;
5649     close $fh;
5650
5651     die "ERROR: archive contaions no data\n" if !$firstfile;
5652     chomp $firstfile;
5653
5654     return $firstfile;
5655 }
5656
5657 sub tar_restore_cleanup {
5658     my ($storecfg, $statfile) = @_;
5659
5660     print STDERR "starting cleanup\n";
5661
5662     if (my $fd = IO::File->new($statfile, "r")) {
5663         while (defined(my $line = <$fd>)) {
5664             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5665                 my $volid = $2;
5666                 eval {
5667                     if ($volid =~ m|^/|) {
5668                         unlink $volid || die 'unlink failed\n';
5669                     } else {
5670                         PVE::Storage::vdisk_free($storecfg, $volid);
5671                     }
5672                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
5673                 };
5674                 print STDERR "unable to cleanup '$volid' - $@" if $@;
5675             } else {
5676                 print STDERR "unable to parse line in statfile - $line";
5677             }
5678         }
5679         $fd->close();
5680     }
5681 }
5682
5683 sub restore_archive {
5684     my ($archive, $vmid, $user, $opts) = @_;
5685
5686     my $format = $opts->{format};
5687     my $comp;
5688
5689     if ($archive =~ m/\.tgz$/ || $archive =~ m/\.tar\.gz$/) {
5690         $format = 'tar' if !$format;
5691         $comp = 'gzip';
5692     } elsif ($archive =~ m/\.tar$/) {
5693         $format = 'tar' if !$format;
5694     } elsif ($archive =~ m/.tar.lzo$/) {
5695         $format = 'tar' if !$format;
5696         $comp = 'lzop';
5697     } elsif ($archive =~ m/\.vma$/) {
5698         $format = 'vma' if !$format;
5699     } elsif ($archive =~ m/\.vma\.gz$/) {
5700         $format = 'vma' if !$format;
5701         $comp = 'gzip';
5702     } elsif ($archive =~ m/\.vma\.lzo$/) {
5703         $format = 'vma' if !$format;
5704         $comp = 'lzop';
5705     } else {
5706         $format = 'vma' if !$format; # default
5707     }
5708
5709     # try to detect archive format
5710     if ($format eq 'tar') {
5711         return restore_tar_archive($archive, $vmid, $user, $opts);
5712     } else {
5713         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
5714     }
5715 }
5716
5717 sub restore_update_config_line {
5718     my ($outfd, $cookie, $vmid, $map, $line, $unique) = @_;
5719
5720     return if $line =~ m/^\#qmdump\#/;
5721     return if $line =~ m/^\#vzdump\#/;
5722     return if $line =~ m/^lock:/;
5723     return if $line =~ m/^unused\d+:/;
5724     return if $line =~ m/^parent:/;
5725     return if $line =~ m/^template:/; # restored VM is never a template
5726
5727     my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
5728     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
5729         # try to convert old 1.X settings
5730         my ($id, $ind, $ethcfg) = ($1, $2, $3);
5731         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
5732             my ($model, $macaddr) = split(/\=/, $devconfig);
5733             $macaddr = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if !$macaddr || $unique;
5734             my $net = {
5735                 model => $model,
5736                 bridge => "vmbr$ind",
5737                 macaddr => $macaddr,
5738             };
5739             my $netstr = print_net($net);
5740
5741             print $outfd "net$cookie->{netcount}: $netstr\n";
5742             $cookie->{netcount}++;
5743         }
5744     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
5745         my ($id, $netstr) = ($1, $2);
5746         my $net = parse_net($netstr);
5747         $net->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if $net->{macaddr};
5748         $netstr = print_net($net);
5749         print $outfd "$id: $netstr\n";
5750     } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk)\d+):\s*(\S+)\s*$/) {
5751         my $virtdev = $1;
5752         my $value = $3;
5753         my $di = parse_drive($virtdev, $value);
5754         if (defined($di->{backup}) && !$di->{backup}) {
5755             print $outfd "#$line";
5756         } elsif ($map->{$virtdev}) {
5757             delete $di->{format}; # format can change on restore
5758             $di->{file} = $map->{$virtdev};
5759             $value = print_drive($vmid, $di);
5760             print $outfd "$virtdev: $value\n";
5761         } else {
5762             print $outfd $line;
5763         }
5764     } elsif (($line =~ m/^vmgenid: (.*)/)) {
5765         my $vmgenid = $1;
5766         if ($vmgenid ne '0') {
5767             # always generate a new vmgenid if there was a valid one setup
5768             $vmgenid = generate_uuid();
5769         }
5770         print $outfd "vmgenid: $vmgenid\n";
5771     } elsif (($line =~ m/^(smbios1: )(.*)/) && $unique) {
5772         my ($uuid, $uuid_str);
5773         UUID::generate($uuid);
5774         UUID::unparse($uuid, $uuid_str);
5775         my $smbios1 = parse_smbios1($2);
5776         $smbios1->{uuid} = $uuid_str;
5777         print $outfd $1.print_smbios1($smbios1)."\n";
5778     } else {
5779         print $outfd $line;
5780     }
5781 }
5782
5783 sub scan_volids {
5784     my ($cfg, $vmid) = @_;
5785
5786     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid);
5787
5788     my $volid_hash = {};
5789     foreach my $storeid (keys %$info) {
5790         foreach my $item (@{$info->{$storeid}}) {
5791             next if !($item->{volid} && $item->{size});
5792             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
5793             $volid_hash->{$item->{volid}} = $item;
5794         }
5795     }
5796
5797     return $volid_hash;
5798 }
5799
5800 sub is_volume_in_use {
5801     my ($storecfg, $conf, $skip_drive, $volid) = @_;
5802
5803     my $path = PVE::Storage::path($storecfg, $volid);
5804
5805     my $scan_config = sub {
5806         my ($cref, $snapname) = @_;
5807
5808         foreach my $key (keys %$cref) {
5809             my $value = $cref->{$key};
5810             if (is_valid_drivename($key)) {
5811                 next if $skip_drive && $key eq $skip_drive;
5812                 my $drive = parse_drive($key, $value);
5813                 next if !$drive || !$drive->{file} || drive_is_cdrom($drive);
5814                 return 1 if $volid eq $drive->{file};
5815                 if ($drive->{file} =~ m!^/!) {
5816                     return 1 if $drive->{file} eq $path;
5817                 } else {
5818                     my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file}, 1);
5819                     next if !$storeid;
5820                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid, 1);
5821                     next if !$scfg;
5822                     return 1 if $path eq PVE::Storage::path($storecfg, $drive->{file}, $snapname);
5823                 }
5824             }
5825         }
5826
5827         return 0;
5828     };
5829
5830     return 1 if &$scan_config($conf);
5831
5832     undef $skip_drive;
5833
5834     foreach my $snapname (keys %{$conf->{snapshots}}) {
5835         return 1 if &$scan_config($conf->{snapshots}->{$snapname}, $snapname);
5836     }
5837
5838     return 0;
5839 }
5840
5841 sub update_disksize {
5842     my ($vmid, $conf, $volid_hash) = @_;
5843
5844     my $changes;
5845     my $prefix = "VM $vmid:";
5846
5847     # used and unused disks
5848     my $referenced = {};
5849
5850     # Note: it is allowed to define multiple storages with same path (alias), so
5851     # we need to check both 'volid' and real 'path' (two different volid can point
5852     # to the same path).
5853
5854     my $referencedpath = {};
5855
5856     # update size info
5857     foreach my $opt (keys %$conf) {
5858         if (is_valid_drivename($opt)) {
5859             my $drive = parse_drive($opt, $conf->{$opt});
5860             my $volid = $drive->{file};
5861             next if !$volid;
5862
5863             $referenced->{$volid} = 1;
5864             if ($volid_hash->{$volid} &&
5865                 (my $path = $volid_hash->{$volid}->{path})) {
5866                 $referencedpath->{$path} = 1;
5867             }
5868
5869             next if drive_is_cdrom($drive);
5870             next if !$volid_hash->{$volid};
5871
5872             $drive->{size} = $volid_hash->{$volid}->{size};
5873             my $new = print_drive($vmid, $drive);
5874             if ($new ne $conf->{$opt}) {
5875                 $changes = 1;
5876                 $conf->{$opt} = $new;
5877                 print "$prefix update disk '$opt' information.\n";
5878             }
5879         }
5880     }
5881
5882     # remove 'unusedX' entry if volume is used
5883     foreach my $opt (keys %$conf) {
5884         next if $opt !~ m/^unused\d+$/;
5885         my $volid = $conf->{$opt};
5886         my $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
5887         if ($referenced->{$volid} || ($path && $referencedpath->{$path})) {
5888             print "$prefix remove entry '$opt', its volume '$volid' is in use.\n";
5889             $changes = 1;
5890             delete $conf->{$opt};
5891         }
5892
5893         $referenced->{$volid} = 1;
5894         $referencedpath->{$path} = 1 if $path;
5895     }
5896
5897     foreach my $volid (sort keys %$volid_hash) {
5898         next if $volid =~ m/vm-$vmid-state-/;
5899         next if $referenced->{$volid};
5900         my $path = $volid_hash->{$volid}->{path};
5901         next if !$path; # just to be sure
5902         next if $referencedpath->{$path};
5903         $changes = 1;
5904         my $key = PVE::QemuConfig->add_unused_volume($conf, $volid);
5905         print "$prefix add unreferenced volume '$volid' as '$key' to config.\n";
5906         $referencedpath->{$path} = 1; # avoid to add more than once (aliases)
5907     }
5908
5909     return $changes;
5910 }
5911
5912 sub rescan {
5913     my ($vmid, $nolock, $dryrun) = @_;
5914
5915     my $cfg = PVE::Storage::config();
5916
5917     # FIXME: Remove once our RBD plugin can handle CT and VM on a single storage
5918     # see: https://pve.proxmox.com/pipermail/pve-devel/2018-July/032900.html
5919     foreach my $stor (keys %{$cfg->{ids}}) {
5920         delete($cfg->{ids}->{$stor}) if ! $cfg->{ids}->{$stor}->{content}->{images};
5921     }
5922
5923     print "rescan volumes...\n";
5924     my $volid_hash = scan_volids($cfg, $vmid);
5925
5926     my $updatefn =  sub {
5927         my ($vmid) = @_;
5928
5929         my $conf = PVE::QemuConfig->load_config($vmid);
5930
5931         PVE::QemuConfig->check_lock($conf);
5932
5933         my $vm_volids = {};
5934         foreach my $volid (keys %$volid_hash) {
5935             my $info = $volid_hash->{$volid};
5936             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
5937         }
5938
5939         my $changes = update_disksize($vmid, $conf, $vm_volids);
5940
5941         PVE::QemuConfig->write_config($vmid, $conf) if $changes && !$dryrun;
5942     };
5943
5944     if (defined($vmid)) {
5945         if ($nolock) {
5946             &$updatefn($vmid);
5947         } else {
5948             PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
5949         }
5950     } else {
5951         my $vmlist = config_list();
5952         foreach my $vmid (keys %$vmlist) {
5953             if ($nolock) {
5954                 &$updatefn($vmid);
5955             } else {
5956                 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
5957             }
5958         }
5959     }
5960 }
5961
5962 sub restore_vma_archive {
5963     my ($archive, $vmid, $user, $opts, $comp) = @_;
5964
5965     my $readfrom = $archive;
5966
5967     my $cfg = PVE::Storage::config();
5968     my $commands = [];
5969     my $bwlimit = $opts->{bwlimit};
5970
5971     my $dbg_cmdstring = '';
5972     my $add_pipe = sub {
5973         my ($cmd) = @_;
5974         push @$commands, $cmd;
5975         $dbg_cmdstring .= ' | ' if length($dbg_cmdstring);
5976         $dbg_cmdstring .= PVE::Tools::cmd2string($cmd);
5977         $readfrom = '-';
5978     };
5979
5980     my $input = undef;
5981     if ($archive eq '-') {
5982         $input = '<&STDIN';
5983     } else {
5984         # If we use a backup from a PVE defined storage we also consider that
5985         # storage's rate limit:
5986         my (undef, $volid) = PVE::Storage::path_to_volume_id($cfg, $archive);
5987         if (defined($volid)) {
5988             my ($sid, undef) = PVE::Storage::parse_volume_id($volid);
5989             my $readlimit = PVE::Storage::get_bandwidth_limit('restore', [$sid], $bwlimit);
5990             if ($readlimit) {
5991                 print STDERR "applying read rate limit: $readlimit\n";
5992                 my $cstream = ['cstream', '-t', $readlimit*1024, '--', $readfrom];
5993                 $add_pipe->($cstream);
5994             }
5995         }
5996     }
5997
5998     if ($comp) {
5999         my $cmd;
6000         if ($comp eq 'gzip') {
6001             $cmd = ['zcat', $readfrom];
6002         } elsif ($comp eq 'lzop') {
6003             $cmd = ['lzop', '-d', '-c', $readfrom];
6004         } else {
6005             die "unknown compression method '$comp'\n";
6006         }
6007         $add_pipe->($cmd);
6008     }
6009
6010     my $tmpdir = "/var/tmp/vzdumptmp$$";
6011     rmtree $tmpdir;
6012
6013     # disable interrupts (always do cleanups)
6014     local $SIG{INT} =
6015         local $SIG{TERM} =
6016         local $SIG{QUIT} =
6017         local $SIG{HUP} = sub { warn "got interrupt - ignored\n"; };
6018
6019     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
6020     POSIX::mkfifo($mapfifo, 0600);
6021     my $fifofh;
6022
6023     my $openfifo = sub {
6024         open($fifofh, '>', $mapfifo) || die $!;
6025     };
6026
6027     $add_pipe->(['vma', 'extract', '-v', '-r', $mapfifo, $readfrom, $tmpdir]);
6028
6029     my $oldtimeout;
6030     my $timeout = 5;
6031
6032     my $devinfo = {};
6033
6034     my $rpcenv = PVE::RPCEnvironment::get();
6035
6036     my $conffile = PVE::QemuConfig->config_file($vmid);
6037     my $tmpfn = "$conffile.$$.tmp";
6038
6039     # Note: $oldconf is undef if VM does not exists
6040     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
6041     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
6042
6043     my %storage_limits;
6044
6045     my $print_devmap = sub {
6046         my $virtdev_hash = {};
6047
6048         my $cfgfn = "$tmpdir/qemu-server.conf";
6049
6050         # we can read the config - that is already extracted
6051         my $fh = IO::File->new($cfgfn, "r") ||
6052             "unable to read qemu-server.conf - $!\n";
6053
6054         my $fwcfgfn = "$tmpdir/qemu-server.fw";
6055         if (-f $fwcfgfn) {
6056             my $pve_firewall_dir = '/etc/pve/firewall';
6057             mkdir $pve_firewall_dir; # make sure the dir exists
6058             PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
6059         }
6060
6061         while (defined(my $line = <$fh>)) {
6062             if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
6063                 my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
6064                 die "archive does not contain data for drive '$virtdev'\n"
6065                     if !$devinfo->{$devname};
6066                 if (defined($opts->{storage})) {
6067                     $storeid = $opts->{storage} || 'local';
6068                 } elsif (!$storeid) {
6069                     $storeid = 'local';
6070                 }
6071                 $format = 'raw' if !$format;
6072                 $devinfo->{$devname}->{devname} = $devname;
6073                 $devinfo->{$devname}->{virtdev} = $virtdev;
6074                 $devinfo->{$devname}->{format} = $format;
6075                 $devinfo->{$devname}->{storeid} = $storeid;
6076
6077                 # check permission on storage
6078                 my $pool = $opts->{pool}; # todo: do we need that?
6079                 if ($user ne 'root@pam') {
6080                     $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
6081                 }
6082
6083                 $storage_limits{$storeid} = $bwlimit;
6084
6085                 $virtdev_hash->{$virtdev} = $devinfo->{$devname};
6086             }
6087         }
6088
6089         foreach my $key (keys %storage_limits) {
6090             my $limit = PVE::Storage::get_bandwidth_limit('restore', [$key], $bwlimit);
6091             next if !$limit;
6092             print STDERR "rate limit for storage $key: $limit KiB/s\n";
6093             $storage_limits{$key} = $limit * 1024;
6094         }
6095
6096         foreach my $devname (keys %$devinfo) {
6097             die "found no device mapping information for device '$devname'\n"
6098                 if !$devinfo->{$devname}->{virtdev};
6099         }
6100
6101         # create empty/temp config
6102         if ($oldconf) {
6103             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
6104             foreach_drive($oldconf, sub {
6105                 my ($ds, $drive) = @_;
6106
6107                 return if drive_is_cdrom($drive);
6108
6109                 my $volid = $drive->{file};
6110
6111                 return if !$volid || $volid =~ m|^/|;
6112
6113                 my ($path, $owner) = PVE::Storage::path($cfg, $volid);
6114                 return if !$path || !$owner || ($owner != $vmid);
6115
6116                 # Note: only delete disk we want to restore
6117                 # other volumes will become unused
6118                 if ($virtdev_hash->{$ds}) {
6119                     eval { PVE::Storage::vdisk_free($cfg, $volid); };
6120                     if (my $err = $@) {
6121                         warn $err;
6122                     }
6123                 }
6124             });
6125
6126             # delete vmstate files
6127             # since after the restore we have no snapshots anymore
6128             foreach my $snapname (keys %{$oldconf->{snapshots}}) {
6129                 my $snap = $oldconf->{snapshots}->{$snapname};
6130                 if ($snap->{vmstate}) {
6131                     eval { PVE::Storage::vdisk_free($cfg, $snap->{vmstate}); };
6132                     if (my $err = $@) {
6133                         warn $err;
6134                     }
6135                 }
6136             }
6137         }
6138
6139         my $map = {};
6140         foreach my $virtdev (sort keys %$virtdev_hash) {
6141             my $d = $virtdev_hash->{$virtdev};
6142             my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
6143             my $storeid = $d->{storeid};
6144             my $scfg = PVE::Storage::storage_config($cfg, $storeid);
6145
6146             my $map_opts = '';
6147             if (my $limit = $storage_limits{$storeid}) {
6148                 $map_opts .= "throttling.bps=$limit:throttling.group=$storeid:";
6149             }
6150
6151             # test if requested format is supported
6152             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($cfg, $storeid);
6153             my $supported = grep { $_ eq $d->{format} } @$validFormats;
6154             $d->{format} = $defFormat if !$supported;
6155
6156             my $volid = PVE::Storage::vdisk_alloc($cfg, $storeid, $vmid,
6157                                                   $d->{format}, undef, $alloc_size);
6158             print STDERR "new volume ID is '$volid'\n";
6159             $d->{volid} = $volid;
6160             my $path = PVE::Storage::path($cfg, $volid);
6161
6162             PVE::Storage::activate_volumes($cfg,[$volid]);
6163
6164             my $write_zeros = 1;
6165             if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
6166                 $write_zeros = 0;
6167             }
6168
6169             print $fifofh "${map_opts}format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
6170
6171             print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
6172             $map->{$virtdev} = $volid;
6173         }
6174
6175         $fh->seek(0, 0) || die "seek failed - $!\n";
6176
6177         my $outfd = new IO::File ($tmpfn, "w") ||
6178             die "unable to write config for VM $vmid\n";
6179
6180         my $cookie = { netcount => 0 };
6181         while (defined(my $line = <$fh>)) {
6182             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
6183         }
6184
6185         $fh->close();
6186         $outfd->close();
6187     };
6188
6189     eval {
6190         # enable interrupts
6191         local $SIG{INT} =
6192             local $SIG{TERM} =
6193             local $SIG{QUIT} =
6194             local $SIG{HUP} =
6195             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6196         local $SIG{ALRM} = sub { die "got timeout\n"; };
6197
6198         $oldtimeout = alarm($timeout);
6199
6200         my $parser = sub {
6201             my $line = shift;
6202
6203             print "$line\n";
6204
6205             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
6206                 my ($dev_id, $size, $devname) = ($1, $2, $3);
6207                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
6208             } elsif ($line =~ m/^CTIME: /) {
6209                 # we correctly received the vma config, so we can disable
6210                 # the timeout now for disk allocation (set to 10 minutes, so
6211                 # that we always timeout if something goes wrong)
6212                 alarm(600);
6213                 &$print_devmap();
6214                 print $fifofh "done\n";
6215                 my $tmp = $oldtimeout || 0;
6216                 $oldtimeout = undef;
6217                 alarm($tmp);
6218                 close($fifofh);
6219             }
6220         };
6221
6222         print "restore vma archive: $dbg_cmdstring\n";
6223         run_command($commands, input => $input, outfunc => $parser, afterfork => $openfifo);
6224     };
6225     my $err = $@;
6226
6227     alarm($oldtimeout) if $oldtimeout;
6228
6229     my $vollist = [];
6230     foreach my $devname (keys %$devinfo) {
6231         my $volid = $devinfo->{$devname}->{volid};
6232         push @$vollist, $volid if $volid;
6233     }
6234
6235     PVE::Storage::deactivate_volumes($cfg, $vollist);
6236
6237     unlink $mapfifo;
6238
6239     if ($err) {
6240         rmtree $tmpdir;
6241         unlink $tmpfn;
6242
6243         foreach my $devname (keys %$devinfo) {
6244             my $volid = $devinfo->{$devname}->{volid};
6245             next if !$volid;
6246             eval {
6247                 if ($volid =~ m|^/|) {
6248                     unlink $volid || die 'unlink failed\n';
6249                 } else {
6250                     PVE::Storage::vdisk_free($cfg, $volid);
6251                 }
6252                 print STDERR "temporary volume '$volid' sucessfuly removed\n";
6253             };
6254             print STDERR "unable to cleanup '$volid' - $@" if $@;
6255         }
6256         die $err;
6257     }
6258
6259     rmtree $tmpdir;
6260
6261     rename($tmpfn, $conffile) ||
6262         die "unable to commit configuration file '$conffile'\n";
6263
6264     PVE::Cluster::cfs_update(); # make sure we read new file
6265
6266     eval { rescan($vmid, 1); };
6267     warn $@ if $@;
6268 }
6269
6270 sub restore_tar_archive {
6271     my ($archive, $vmid, $user, $opts) = @_;
6272
6273     if ($archive ne '-') {
6274         my $firstfile = tar_archive_read_firstfile($archive);
6275         die "ERROR: file '$archive' dos not lock like a QemuServer vzdump backup\n"
6276             if $firstfile ne 'qemu-server.conf';
6277     }
6278
6279     my $storecfg = PVE::Storage::config();
6280
6281     # destroy existing data - keep empty config
6282     my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
6283     destroy_vm($storecfg, $vmid, 1) if -f $vmcfgfn;
6284
6285     my $tocmd = "/usr/lib/qemu-server/qmextract";
6286
6287     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
6288     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
6289     $tocmd .= ' --prealloc' if $opts->{prealloc};
6290     $tocmd .= ' --info' if $opts->{info};
6291
6292     # tar option "xf" does not autodetect compression when read from STDIN,
6293     # so we pipe to zcat
6294     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
6295         PVE::Tools::shellquote("--to-command=$tocmd");
6296
6297     my $tmpdir = "/var/tmp/vzdumptmp$$";
6298     mkpath $tmpdir;
6299
6300     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
6301     local $ENV{VZDUMP_VMID} = $vmid;
6302     local $ENV{VZDUMP_USER} = $user;
6303
6304     my $conffile = PVE::QemuConfig->config_file($vmid);
6305     my $tmpfn = "$conffile.$$.tmp";
6306
6307     # disable interrupts (always do cleanups)
6308     local $SIG{INT} =
6309         local $SIG{TERM} =
6310         local $SIG{QUIT} =
6311         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
6312
6313     eval {
6314         # enable interrupts
6315         local $SIG{INT} =
6316             local $SIG{TERM} =
6317             local $SIG{QUIT} =
6318             local $SIG{HUP} =
6319             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6320
6321         if ($archive eq '-') {
6322             print "extracting archive from STDIN\n";
6323             run_command($cmd, input => "<&STDIN");
6324         } else {
6325             print "extracting archive '$archive'\n";
6326             run_command($cmd);
6327         }
6328
6329         return if $opts->{info};
6330
6331         # read new mapping
6332         my $map = {};
6333         my $statfile = "$tmpdir/qmrestore.stat";
6334         if (my $fd = IO::File->new($statfile, "r")) {
6335             while (defined (my $line = <$fd>)) {
6336                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
6337                     $map->{$1} = $2 if $1;
6338                 } else {
6339                     print STDERR "unable to parse line in statfile - $line\n";
6340                 }
6341             }
6342             $fd->close();
6343         }
6344
6345         my $confsrc = "$tmpdir/qemu-server.conf";
6346
6347         my $srcfd = new IO::File($confsrc, "r") ||
6348             die "unable to open file '$confsrc'\n";
6349
6350         my $outfd = new IO::File ($tmpfn, "w") ||
6351             die "unable to write config for VM $vmid\n";
6352
6353         my $cookie = { netcount => 0 };
6354         while (defined (my $line = <$srcfd>)) {
6355             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
6356         }
6357
6358         $srcfd->close();
6359         $outfd->close();
6360     };
6361     my $err = $@;
6362
6363     if ($err) {
6364
6365         unlink $tmpfn;
6366
6367         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
6368
6369         die $err;
6370     }
6371
6372     rmtree $tmpdir;
6373
6374     rename $tmpfn, $conffile ||
6375         die "unable to commit configuration file '$conffile'\n";
6376
6377     PVE::Cluster::cfs_update(); # make sure we read new file
6378
6379     eval { rescan($vmid, 1); };
6380     warn $@ if $@;
6381 };
6382
6383 sub foreach_storage_used_by_vm {
6384     my ($conf, $func) = @_;
6385
6386     my $sidhash = {};
6387
6388     foreach_drive($conf, sub {
6389         my ($ds, $drive) = @_;
6390         return if drive_is_cdrom($drive);
6391
6392         my $volid = $drive->{file};
6393
6394         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
6395         $sidhash->{$sid} = $sid if $sid;
6396     });
6397
6398     foreach my $sid (sort keys %$sidhash) {
6399         &$func($sid);
6400     }
6401 }
6402
6403 sub do_snapshots_with_qemu {
6404     my ($storecfg, $volid) = @_;
6405
6406     my $storage_name = PVE::Storage::parse_volume_id($volid);
6407
6408     if ($qemu_snap_storage->{$storecfg->{ids}->{$storage_name}->{type}}
6409         && !$storecfg->{ids}->{$storage_name}->{krbd}){
6410         return 1;
6411     }
6412
6413     if ($volid =~ m/\.(qcow2|qed)$/){
6414         return 1;
6415     }
6416
6417     return undef;
6418 }
6419
6420 sub qga_check_running {
6421     my ($vmid, $nowarn) = @_;
6422
6423     eval { vm_mon_cmd($vmid, "guest-ping", timeout => 3); };
6424     if ($@) {
6425         warn "Qemu Guest Agent is not running - $@" if !$nowarn;
6426         return 0;
6427     }
6428     return 1;
6429 }
6430
6431 sub template_create {
6432     my ($vmid, $conf, $disk) = @_;
6433
6434     my $storecfg = PVE::Storage::config();
6435
6436     foreach_drive($conf, sub {
6437         my ($ds, $drive) = @_;
6438
6439         return if drive_is_cdrom($drive);
6440         return if $disk && $ds ne $disk;
6441
6442         my $volid = $drive->{file};
6443         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
6444
6445         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
6446         $drive->{file} = $voliddst;
6447         $conf->{$ds} = print_drive($vmid, $drive);
6448         PVE::QemuConfig->write_config($vmid, $conf);
6449     });
6450 }
6451
6452 sub qemu_img_convert {
6453     my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized) = @_;
6454
6455     my $storecfg = PVE::Storage::config();
6456     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
6457     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
6458
6459     if ($src_storeid && $dst_storeid) {
6460
6461         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
6462
6463         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
6464         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6465
6466         my $src_format = qemu_img_format($src_scfg, $src_volname);
6467         my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
6468
6469         my $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
6470         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6471
6472         my $cmd = [];
6473         push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
6474         push @$cmd, '-l', "snapshot.name=$snapname" if($snapname && $src_format eq "qcow2");
6475         push @$cmd, '-t', 'none' if $dst_scfg->{type} eq 'zfspool';
6476         push @$cmd, '-T', 'none' if $src_scfg->{type} eq 'zfspool';
6477         push @$cmd, '-f', $src_format, '-O', $dst_format, $src_path;
6478         if ($is_zero_initialized) {
6479             push @$cmd, "zeroinit:$dst_path";
6480         } else {
6481             push @$cmd, $dst_path;
6482         }
6483
6484         my $parser = sub {
6485             my $line = shift;
6486             if($line =~ m/\((\S+)\/100\%\)/){
6487                 my $percent = $1;
6488                 my $transferred = int($size * $percent / 100);
6489                 my $remaining = $size - $transferred;
6490
6491                 print "transferred: $transferred bytes remaining: $remaining bytes total: $size bytes progression: $percent %\n";
6492             }
6493
6494         };
6495
6496         eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
6497         my $err = $@;
6498         die "copy failed: $err" if $err;
6499     }
6500 }
6501
6502 sub qemu_img_format {
6503     my ($scfg, $volname) = @_;
6504
6505     if ($scfg->{path} && $volname =~ m/\.($QEMU_FORMAT_RE)$/) {
6506         return $1;
6507     } else {
6508         return "raw";
6509     }
6510 }
6511
6512 sub qemu_drive_mirror {
6513     my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $skipcomplete, $qga) = @_;
6514
6515     $jobs = {} if !$jobs;
6516
6517     my $qemu_target;
6518     my $format;
6519     $jobs->{"drive-$drive"} = {};
6520
6521     if ($dst_volid =~ /^nbd:/) {
6522         $qemu_target = $dst_volid;
6523         $format = "nbd";
6524     } else {
6525         my $storecfg = PVE::Storage::config();
6526         my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
6527
6528         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6529
6530         $format = qemu_img_format($dst_scfg, $dst_volname);
6531
6532         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6533
6534         $qemu_target = $is_zero_initialized ? "zeroinit:$dst_path" : $dst_path;
6535     }
6536
6537     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $qemu_target };
6538     $opts->{format} = $format if $format;
6539
6540     print "drive mirror is starting for drive-$drive\n";
6541
6542     eval { vm_mon_cmd($vmid, "drive-mirror", %$opts); }; #if a job already run for this device,it's throw an error
6543
6544     if (my $err = $@) {
6545         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6546         die "mirroring error: $err";
6547     }
6548
6549     qemu_drive_mirror_monitor ($vmid, $vmiddst, $jobs, $skipcomplete, $qga);
6550 }
6551
6552 sub qemu_drive_mirror_monitor {
6553     my ($vmid, $vmiddst, $jobs, $skipcomplete, $qga) = @_;
6554
6555     eval {
6556         my $err_complete = 0;
6557
6558         while (1) {
6559             die "storage migration timed out\n" if $err_complete > 300;
6560
6561             my $stats = vm_mon_cmd($vmid, "query-block-jobs");
6562
6563             my $running_mirror_jobs = {};
6564             foreach my $stat (@$stats) {
6565                 next if $stat->{type} ne 'mirror';
6566                 $running_mirror_jobs->{$stat->{device}} = $stat;
6567             }
6568
6569             my $readycounter = 0;
6570
6571             foreach my $job (keys %$jobs) {
6572
6573                 if(defined($jobs->{$job}->{complete}) && !defined($running_mirror_jobs->{$job})) {
6574                     print "$job : finished\n";
6575                     delete $jobs->{$job};
6576                     next;
6577                 }
6578
6579                 die "$job: mirroring has been cancelled\n" if !defined($running_mirror_jobs->{$job});
6580
6581                 my $busy = $running_mirror_jobs->{$job}->{busy};
6582                 my $ready = $running_mirror_jobs->{$job}->{ready};
6583                 if (my $total = $running_mirror_jobs->{$job}->{len}) {
6584                     my $transferred = $running_mirror_jobs->{$job}->{offset} || 0;
6585                     my $remaining = $total - $transferred;
6586                     my $percent = sprintf "%.2f", ($transferred * 100 / $total);
6587
6588                     print "$job: transferred: $transferred bytes remaining: $remaining bytes total: $total bytes progression: $percent % busy: $busy ready: $ready \n";
6589                 }
6590
6591                 $readycounter++ if $running_mirror_jobs->{$job}->{ready};
6592             }
6593
6594             last if scalar(keys %$jobs) == 0;
6595
6596             if ($readycounter == scalar(keys %$jobs)) {
6597                 print "all mirroring jobs are ready \n";
6598                 last if $skipcomplete; #do the complete later
6599
6600                 if ($vmiddst && $vmiddst != $vmid) {
6601                     my $agent_running = $qga && qga_check_running($vmid);
6602                     if ($agent_running) {
6603                         print "freeze filesystem\n";
6604                         eval { PVE::QemuServer::vm_mon_cmd($vmid, "guest-fsfreeze-freeze"); };
6605                     } else {
6606                         print "suspend vm\n";
6607                         eval { PVE::QemuServer::vm_suspend($vmid, 1); };
6608                     }
6609
6610                     # if we clone a disk for a new target vm, we don't switch the disk
6611                     PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs);
6612
6613                     if ($agent_running) {
6614                         print "unfreeze filesystem\n";
6615                         eval { PVE::QemuServer::vm_mon_cmd($vmid, "guest-fsfreeze-thaw"); };
6616                     } else {
6617                         print "resume vm\n";
6618                         eval {  PVE::QemuServer::vm_resume($vmid, 1, 1); };
6619                     }
6620
6621                     last;
6622                 } else {
6623
6624                     foreach my $job (keys %$jobs) {
6625                         # try to switch the disk if source and destination are on the same guest
6626                         print "$job: Completing block job...\n";
6627
6628                         eval { vm_mon_cmd($vmid, "block-job-complete", device => $job) };
6629                         if ($@ =~ m/cannot be completed/) {
6630                             print "$job: Block job cannot be completed, try again.\n";
6631                             $err_complete++;
6632                         }else {
6633                             print "$job: Completed successfully.\n";
6634                             $jobs->{$job}->{complete} = 1;
6635                         }
6636                     }
6637                 }
6638             }
6639             sleep 1;
6640         }
6641     };
6642     my $err = $@;
6643
6644     if ($err) {
6645         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6646         die "mirroring error: $err";
6647     }
6648
6649 }
6650
6651 sub qemu_blockjobs_cancel {
6652     my ($vmid, $jobs) = @_;
6653
6654     foreach my $job (keys %$jobs) {
6655         print "$job: Cancelling block job\n";
6656         eval { vm_mon_cmd($vmid, "block-job-cancel", device => $job); };
6657         $jobs->{$job}->{cancel} = 1;
6658     }
6659
6660     while (1) {
6661         my $stats = vm_mon_cmd($vmid, "query-block-jobs");
6662
6663         my $running_jobs = {};
6664         foreach my $stat (@$stats) {
6665             $running_jobs->{$stat->{device}} = $stat;
6666         }
6667
6668         foreach my $job (keys %$jobs) {
6669
6670             if (defined($jobs->{$job}->{cancel}) && !defined($running_jobs->{$job})) {
6671                 print "$job: Done.\n";
6672                 delete $jobs->{$job};
6673             }
6674         }
6675
6676         last if scalar(keys %$jobs) == 0;
6677
6678         sleep 1;
6679     }
6680 }
6681
6682 sub clone_disk {
6683     my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
6684         $newvmid, $storage, $format, $full, $newvollist, $jobs, $skipcomplete, $qga) = @_;
6685
6686     my $newvolid;
6687
6688     if (!$full) {
6689         print "create linked clone of drive $drivename ($drive->{file})\n";
6690         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
6691         push @$newvollist, $newvolid;
6692     } else {
6693
6694         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6695         $storeid = $storage if $storage;
6696
6697         my $dst_format = resolve_dst_disk_format($storecfg, $storeid, $volname, $format);
6698         my ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 3);
6699
6700         print "create full clone of drive $drivename ($drive->{file})\n";
6701         my $name = undef;
6702         if (drive_is_cloudinit($drive)) {
6703             $name = "vm-$newvmid-cloudinit";
6704             # cloudinit only supports raw and qcow2 atm:
6705             if ($dst_format eq 'qcow2') {
6706                 $name .= '.qcow2';
6707             } elsif ($dst_format ne 'raw') {
6708                 die "clone: unhandled format for cloudinit image\n";
6709             }
6710         }
6711         $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $newvmid, $dst_format, $name, ($size/1024));
6712         push @$newvollist, $newvolid;
6713
6714         PVE::Storage::activate_volumes($storecfg, [$newvolid]);
6715
6716         my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
6717         if (!$running || $snapname) {
6718             qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit);
6719         } else {
6720
6721             my $kvmver = get_running_qemu_version ($vmid);
6722             if (!qemu_machine_feature_enabled (undef, $kvmver, 2, 7)) {
6723                 die "drive-mirror with iothread requires qemu version 2.7 or higher\n"
6724                     if $drive->{iothread};
6725             }
6726
6727             qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid, $sparseinit, $jobs, $skipcomplete, $qga);
6728         }
6729     }
6730
6731     my ($size) = PVE::Storage::volume_size_info($storecfg, $newvolid, 3);
6732
6733     my $disk = $drive;
6734     $disk->{format} = undef;
6735     $disk->{file} = $newvolid;
6736     $disk->{size} = $size;
6737
6738     return $disk;
6739 }
6740
6741 # this only works if VM is running
6742 sub get_current_qemu_machine {
6743     my ($vmid) = @_;
6744
6745     my $cmd = { execute => 'query-machines', arguments => {} };
6746     my $res = vm_qmp_command($vmid, $cmd);
6747
6748     my ($current, $default);
6749     foreach my $e (@$res) {
6750         $default = $e->{name} if $e->{'is-default'};
6751         $current = $e->{name} if $e->{'is-current'};
6752     }
6753
6754     # fallback to the default machine if current is not supported by qemu
6755     return $current || $default || 'pc';
6756 }
6757
6758 sub get_running_qemu_version {
6759     my ($vmid) = @_;
6760     my $cmd = { execute => 'query-version', arguments => {} };
6761     my $res = vm_qmp_command($vmid, $cmd);
6762     return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
6763 }
6764
6765 sub qemu_machine_feature_enabled {
6766     my ($machine, $kvmver, $version_major, $version_minor) = @_;
6767
6768     my $current_major;
6769     my $current_minor;
6770
6771     if ($machine && $machine =~ m/^((?:pc(-i440fx|-q35)?|virt)-(\d+)\.(\d+))/) {
6772
6773         $current_major = $3;
6774         $current_minor = $4;
6775
6776     } elsif ($kvmver =~ m/^(\d+)\.(\d+)/) {
6777
6778         $current_major = $1;
6779         $current_minor = $2;
6780     }
6781
6782     return 1 if $current_major > $version_major ||
6783                 ($current_major == $version_major &&
6784                  $current_minor >= $version_minor);
6785 }
6786
6787 sub qemu_machine_pxe {
6788     my ($vmid, $conf, $machine) = @_;
6789
6790     $machine =  PVE::QemuServer::get_current_qemu_machine($vmid) if !$machine;
6791
6792     if ($conf->{machine} && $conf->{machine} =~ m/\.pxe$/) {
6793         $machine .= '.pxe';
6794     }
6795
6796     return $machine;
6797 }
6798
6799 sub qemu_use_old_bios_files {
6800     my ($machine_type) = @_;
6801
6802     return if !$machine_type;
6803
6804     my $use_old_bios_files = undef;
6805
6806     if ($machine_type =~ m/^(\S+)\.pxe$/) {
6807         $machine_type = $1;
6808         $use_old_bios_files = 1;
6809     } else {
6810         my $kvmver = kvm_user_version();
6811         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
6812         # load new efi bios files on migration. So this hack is required to allow
6813         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
6814         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
6815         $use_old_bios_files = !qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 4);
6816     }
6817
6818     return ($use_old_bios_files, $machine_type);
6819 }
6820
6821 sub create_efidisk($$$$$) {
6822     my ($storecfg, $storeid, $vmid, $fmt, $arch) = @_;
6823
6824     my (undef, $ovmf_vars) = get_ovmf_files($arch);
6825     die "EFI vars default image not found\n" if ! -f $ovmf_vars;
6826
6827     my $vars_size = PVE::Tools::convert_size(-s $ovmf_vars, 'b' => 'kb');
6828     my $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $fmt, undef, $vars_size);
6829     PVE::Storage::activate_volumes($storecfg, [$volid]);
6830
6831     my $path = PVE::Storage::path($storecfg, $volid);
6832     eval {
6833         run_command(['/usr/bin/qemu-img', 'convert', '-n', '-f', 'raw', '-O', $fmt, $ovmf_vars, $path]);
6834     };
6835     die "Copying EFI vars image failed: $@" if $@;
6836
6837     return ($volid, $vars_size);
6838 }
6839
6840 sub lspci {
6841
6842     my $devices = {};
6843
6844     dir_glob_foreach("$pcisysfs/devices", '[a-f0-9]{4}:([a-f0-9]{2}:[a-f0-9]{2})\.([0-9])', sub {
6845             my (undef, $id, $function) = @_;
6846             my $res = { id => $id, function => $function};
6847             push @{$devices->{$id}}, $res;
6848     });
6849
6850     # Entries should be sorted by functions.
6851     foreach my $id (keys %$devices) {
6852         my $dev = $devices->{$id};
6853         $devices->{$id} = [ sort { $a->{function} <=> $b->{function} } @$dev ];
6854     }
6855
6856     return $devices;
6857 }
6858
6859 sub vm_iothreads_list {
6860     my ($vmid) = @_;
6861
6862     my $res = vm_mon_cmd($vmid, 'query-iothreads');
6863
6864     my $iothreads = {};
6865     foreach my $iothread (@$res) {
6866         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
6867     }
6868
6869     return $iothreads;
6870 }
6871
6872 sub scsihw_infos {
6873     my ($conf, $drive) = @_;
6874
6875     my $maxdev = 0;
6876
6877     if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)) {
6878         $maxdev = 7;
6879     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
6880         $maxdev = 1;
6881     } else {
6882         $maxdev = 256;
6883     }
6884
6885     my $controller = int($drive->{index} / $maxdev);
6886     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single') ? "virtioscsi" : "scsihw";
6887
6888     return ($maxdev, $controller, $controller_prefix);
6889 }
6890
6891 sub add_hyperv_enlightenments {
6892     my ($cpuFlags, $winversion, $machine_type, $kvmver, $bios, $gpu_passthrough) = @_;
6893
6894     return if $winversion < 6;
6895     return if $bios && $bios eq 'ovmf' && $winversion < 8;
6896
6897     push @$cpuFlags , 'hv_vendor_id=proxmox' if $gpu_passthrough;
6898
6899     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
6900         push @$cpuFlags , 'hv_spinlocks=0x1fff';
6901         push @$cpuFlags , 'hv_vapic';
6902         push @$cpuFlags , 'hv_time';
6903     } else {
6904         push @$cpuFlags , 'hv_spinlocks=0xffff';
6905     }
6906
6907     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 6)) {
6908         push @$cpuFlags , 'hv_reset';
6909         push @$cpuFlags , 'hv_vpindex';
6910         push @$cpuFlags , 'hv_runtime';
6911     }
6912
6913     if ($winversion >= 7) {
6914         push @$cpuFlags , 'hv_relaxed';
6915
6916         if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 12)) {
6917             push @$cpuFlags , 'hv_synic';
6918             push @$cpuFlags , 'hv_stimer';
6919         }
6920     }
6921 }
6922
6923 sub windows_version {
6924     my ($ostype) = @_;
6925
6926     return 0 if !$ostype;
6927
6928     my $winversion = 0;
6929
6930     if($ostype eq 'wxp' || $ostype eq 'w2k3' || $ostype eq 'w2k') {
6931         $winversion = 5;
6932     } elsif($ostype eq 'w2k8' || $ostype eq 'wvista') {
6933         $winversion = 6;
6934     } elsif ($ostype =~ m/^win(\d+)$/) {
6935         $winversion = $1;
6936     }
6937
6938     return $winversion;
6939 }
6940
6941 sub resolve_dst_disk_format {
6942         my ($storecfg, $storeid, $src_volname, $format) = @_;
6943         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
6944
6945         if (!$format) {
6946             # if no target format is specified, use the source disk format as hint
6947             if ($src_volname) {
6948                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6949                 $format = qemu_img_format($scfg, $src_volname);
6950             } else {
6951                 return $defFormat;
6952             }
6953         }
6954
6955         # test if requested format is supported - else use default
6956         my $supported = grep { $_ eq $format } @$validFormats;
6957         $format = $defFormat if !$supported;
6958         return $format;
6959 }
6960
6961 sub resolve_first_disk {
6962     my $conf = shift;
6963     my @disks = PVE::QemuServer::valid_drive_names();
6964     my $firstdisk;
6965     foreach my $ds (reverse @disks) {
6966         next if !$conf->{$ds};
6967         my $disk = PVE::QemuServer::parse_drive($ds, $conf->{$ds});
6968         next if PVE::QemuServer::drive_is_cdrom($disk);
6969         $firstdisk = $ds;
6970     }
6971     return $firstdisk;
6972 }
6973
6974 sub generate_uuid {
6975     my ($uuid, $uuid_str);
6976     UUID::generate($uuid);
6977     UUID::unparse($uuid, $uuid_str);
6978     return $uuid_str;
6979 }
6980
6981 sub generate_smbios1_uuid {
6982     return "uuid=".generate_uuid();
6983 }
6984
6985 sub nbd_stop {
6986     my ($vmid) = @_;
6987
6988     vm_mon_cmd($vmid, 'nbd-server-stop');
6989 }
6990
6991 # bash completion helper
6992
6993 sub complete_backup_archives {
6994     my ($cmdname, $pname, $cvalue) = @_;
6995
6996     my $cfg = PVE::Storage::config();
6997
6998     my $storeid;
6999
7000     if ($cvalue =~ m/^([^:]+):/) {
7001         $storeid = $1;
7002     }
7003
7004     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
7005
7006     my $res = [];
7007     foreach my $id (keys %$data) {
7008         foreach my $item (@{$data->{$id}}) {
7009             next if $item->{format} !~ m/^vma\.(gz|lzo)$/;
7010             push @$res, $item->{volid} if defined($item->{volid});
7011         }
7012     }
7013
7014     return $res;
7015 }
7016
7017 my $complete_vmid_full = sub {
7018     my ($running) = @_;
7019
7020     my $idlist = vmstatus();
7021
7022     my $res = [];
7023
7024     foreach my $id (keys %$idlist) {
7025         my $d = $idlist->{$id};
7026         if (defined($running)) {
7027             next if $d->{template};
7028             next if $running && $d->{status} ne 'running';
7029             next if !$running && $d->{status} eq 'running';
7030         }
7031         push @$res, $id;
7032
7033     }
7034     return $res;
7035 };
7036
7037 sub complete_vmid {
7038     return &$complete_vmid_full();
7039 }
7040
7041 sub complete_vmid_stopped {
7042     return &$complete_vmid_full(0);
7043 }
7044
7045 sub complete_vmid_running {
7046     return &$complete_vmid_full(1);
7047 }
7048
7049 sub complete_storage {
7050
7051     my $cfg = PVE::Storage::config();
7052     my $ids = $cfg->{ids};
7053
7054     my $res = [];
7055     foreach my $sid (keys %$ids) {
7056         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
7057         next if !$ids->{$sid}->{content}->{images};
7058         push @$res, $sid;
7059     }
7060
7061     return $res;
7062 }
7063
7064 1;