PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5 use POSIX;
   6 use IO::Handle;
   7 use IO::Select;
   8 use IO::File;
   9 use IO::Dir;
  10 use IO::Socket::UNIX;
  11 use File::Basename;
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use Digest::SHA;
  16 use Fcntl ':flock';
  17 use Cwd 'abs_path';
  18 use IPC::Open3;
  19 use JSON;
  20 use Fcntl;
  21 use PVE::SafeSyslog;
  22 use Storable qw(dclone);
  23 use PVE::Exception qw(raise raise_param_exc);
  24 use PVE::Storage;
  25 use PVE::Tools qw(run_command lock_file lock_file_full file_read_firstline dir_glob_foreach $IPV6RE);
  26 use PVE::JSONSchema qw(get_standard_option);
  27 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file cfs_lock_file);
  28 use PVE::INotify;
  29 use PVE::ProcFSTools;
  30 use PVE::QemuConfig;
  31 use PVE::QMPClient;
  32 use PVE::RPCEnvironment;
  33 use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr);
  34 use PVE::QemuServer::Memory;
  35 use PVE::QemuServer::USB qw(parse_usb_device);
  36 use PVE::QemuServer::Cloudinit;
  37 use PVE::Systemd;
  38 use Time::HiRes qw(gettimeofday);
  39 use File::Copy qw(copy);
  40 use URI::Escape;
  41
  42 my $EDK2_FW_BASE = '/usr/share/pve-edk2-firmware/';
  43 my $OVMF = {
  44     x86_64 => [
  45         "$EDK2_FW_BASE/OVMF_CODE.fd",
  46         "$EDK2_FW_BASE/OVMF_VARS.fd"
  47     ],
  48     aarch64 => [
  49         "$EDK2_FW_BASE/AAVMF_CODE.fd",
  50         "$EDK2_FW_BASE/AAVMF_VARS.fd"
  51     ],
  52 };
  53
  54 my $qemu_snap_storage = {rbd => 1, sheepdog => 1};
  55
  56 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  57
  58 my $QEMU_FORMAT_RE = qr/raw|cow|qcow|qcow2|qed|vmdk|cloop/;
  59
  60 # Note about locking: we use flock on the config file protect
  61 # against concurent actions.
  62 # Aditionaly, we have a 'lock' setting in the config file. This
  63 # can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
  64 # allowed when such lock is set. But you can ignore this kind of
  65 # lock with the --skiplock flag.
  66
  67 cfs_register_file('/qemu-server/',
  68                   \&parse_vm_config,
  69                   \&write_vm_config);
  70
  71 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
  72     description => "Some command save/restore state from this location.",
  73     type => 'string',
  74     maxLength => 128,
  75     optional => 1,
  76 });
  77
  78 PVE::JSONSchema::register_standard_option('pve-snapshot-name', {
  79     description => "The name of the snapshot.",
  80     type => 'string', format => 'pve-configid',
  81     maxLength => 40,
  82 });
  83
  84 PVE::JSONSchema::register_standard_option('pve-qm-image-format', {
  85     type => 'string',
  86     enum => [qw(raw cow qcow qed qcow2 vmdk cloop)],
  87     description => "The drive's backing file's data format.",
  88     optional => 1,
  89 });
  90
  91 PVE::JSONSchema::register_standard_option('pve-qemu-machine', {
  92         description => "Specifies the Qemu machine type.",
  93         type => 'string',
  94         pattern => '(pc|pc(-i440fx)?-\d+\.\d+(\.pxe)?|q35|pc-q35-\d+\.\d+(\.pxe)?|virt(?:-\d+\.\d+)?)',
  95         maxLength => 40,
  96         optional => 1,
  97 });
  98
  99 #no warnings 'redefine';
 100
 101 sub cgroups_write {
 102    my ($controller, $vmid, $option, $value) = @_;
 103
 104    my $path = "/sys/fs/cgroup/$controller/qemu.slice/$vmid.scope/$option";
 105    PVE::ProcFSTools::write_proc_entry($path, $value);
 106
 107 }
 108
 109 my $nodename = PVE::INotify::nodename();
 110
 111 mkdir "/etc/pve/nodes/$nodename";
 112 my $confdir = "/etc/pve/nodes/$nodename/qemu-server";
 113 mkdir $confdir;
 114
 115 my $var_run_tmpdir = "/var/run/qemu-server";
 116 mkdir $var_run_tmpdir;
 117
 118 my $lock_dir = "/var/lock/qemu-server";
 119 mkdir $lock_dir;
 120
 121 my $pcisysfs = "/sys/bus/pci";
 122
 123 my $cpu_vendor_list = {
 124     # Intel CPUs
 125     486 => 'GenuineIntel',
 126     pentium => 'GenuineIntel',
 127     pentium2  => 'GenuineIntel',
 128     pentium3  => 'GenuineIntel',
 129     coreduo => 'GenuineIntel',
 130     core2duo => 'GenuineIntel',
 131     Conroe  => 'GenuineIntel',
 132     Penryn  => 'GenuineIntel',
 133     Nehalem  => 'GenuineIntel',
 134     'Nehalem-IBRS'  => 'GenuineIntel',
 135     Westmere => 'GenuineIntel',
 136     'Westmere-IBRS' => 'GenuineIntel',
 137     SandyBridge => 'GenuineIntel',
 138     'SandyBridge-IBRS' => 'GenuineIntel',
 139     IvyBridge => 'GenuineIntel',
 140     'IvyBridge-IBRS' => 'GenuineIntel',
 141     Haswell => 'GenuineIntel',
 142     'Haswell-IBRS' => 'GenuineIntel',
 143     'Haswell-noTSX' => 'GenuineIntel',
 144     'Haswell-noTSX-IBRS' => 'GenuineIntel',
 145     Broadwell => 'GenuineIntel',
 146     'Broadwell-IBRS' => 'GenuineIntel',
 147     'Broadwell-noTSX' => 'GenuineIntel',
 148     'Broadwell-noTSX-IBRS' => 'GenuineIntel',
 149     'Skylake-Client' => 'GenuineIntel',
 150     'Skylake-Client-IBRS' => 'GenuineIntel',
 151     'Skylake-Server' => 'GenuineIntel',
 152     'Skylake-Server-IBRS' => 'GenuineIntel',
 153
 154     # AMD CPUs
 155     athlon => 'AuthenticAMD',
 156     phenom  => 'AuthenticAMD',
 157     Opteron_G1  => 'AuthenticAMD',
 158     Opteron_G2  => 'AuthenticAMD',
 159     Opteron_G3  => 'AuthenticAMD',
 160     Opteron_G4  => 'AuthenticAMD',
 161     Opteron_G5  => 'AuthenticAMD',
 162     EPYC => 'AuthenticAMD',
 163     'EPYC-IBPB' => 'AuthenticAMD',
 164
 165     # generic types, use vendor from host node
 166     host => 'default',
 167     kvm32 => 'default',
 168     kvm64 => 'default',
 169     qemu32 => 'default',
 170     qemu64 => 'default',
 171     max => 'default',
 172 };
 173
 174 my $cpu_flag = qr/[+-](pcid|spec-ctrl|ibpb|ssbd|virt-ssbd|amd-ssbd|amd-no-ssb|pdpe1gb)/;
 175
 176 my $cpu_fmt = {
 177     cputype => {
 178         description => "Emulated CPU type.",
 179         type => 'string',
 180         enum => [ sort { "\L$a" cmp "\L$b" } keys %$cpu_vendor_list ],
 181         default => 'kvm64',
 182         default_key => 1,
 183     },
 184     hidden => {
 185         description => "Do not identify as a KVM virtual machine.",
 186         type => 'boolean',
 187         optional => 1,
 188         default => 0
 189     },
 190     flags => {
 191         description => "List of additional CPU flags separated by ';'."
 192                      . " Use '+FLAG' to enable, '-FLAG' to disable a flag."
 193                      . " Currently supported flags: 'pcid', 'spec-ctrl', 'ibpb', 'ssbd', 'virt-ssbd', 'amd-ssbd', 'amd-no-ssb', 'pdpe1gb'.",
 194         format_description => '+FLAG[;-FLAG...]',
 195         type => 'string',
 196         pattern => qr/$cpu_flag(;$cpu_flag)*/,
 197         optional => 1,
 198     },
 199 };
 200
 201 my $watchdog_fmt = {
 202     model => {
 203         default_key => 1,
 204         type => 'string',
 205         enum => [qw(i6300esb ib700)],
 206         description => "Watchdog type to emulate.",
 207         default => 'i6300esb',
 208         optional => 1,
 209     },
 210     action => {
 211         type => 'string',
 212         enum => [qw(reset shutdown poweroff pause debug none)],
 213         description => "The action to perform if after activation the guest fails to poll the watchdog in time.",
 214         optional => 1,
 215     },
 216 };
 217 PVE::JSONSchema::register_format('pve-qm-watchdog', $watchdog_fmt);
 218
 219 my $agent_fmt = {
 220     enabled => {
 221         description => "Enable/disable Qemu GuestAgent.",
 222         type => 'boolean',
 223         default => 0,
 224         default_key => 1,
 225     },
 226     fstrim_cloned_disks => {
 227         description => "Run fstrim after cloning/moving a disk.",
 228         type => 'boolean',
 229         optional => 1,
 230         default => 0
 231     },
 232 };
 233
 234 my $vga_fmt = {
 235     type => {
 236         description => "Select the VGA type.",
 237         type => 'string',
 238         default => 'std',
 239         optional => 1,
 240         default_key => 1,
 241         enum => [qw(cirrus qxl qxl2 qxl3 qxl4 serial0 serial1 serial2 serial3 std virtio vmware)],
 242     },
 243     memory => {
 244         description => "Sets the VGA memory (in MiB). Has no effect with serial display.",
 245         type => 'integer',
 246         optional => 1,
 247         minimum => 4,
 248         maximum => 512,
 249     },
 250 };
 251
 252 my $confdesc = {
 253     onboot => {
 254         optional => 1,
 255         type => 'boolean',
 256         description => "Specifies whether a VM will be started during system bootup.",
 257         default => 0,
 258     },
 259     autostart => {
 260         optional => 1,
 261         type => 'boolean',
 262         description => "Automatic restart after crash (currently ignored).",
 263         default => 0,
 264     },
 265     hotplug => {
 266         optional => 1,
 267         type => 'string', format => 'pve-hotplug-features',
 268         description => "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
 269         default => 'network,disk,usb',
 270     },
 271     reboot => {
 272         optional => 1,
 273         type => 'boolean',
 274         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 275         default => 1,
 276     },
 277     lock => {
 278         optional => 1,
 279         type => 'string',
 280         description => "Lock/unlock the VM.",
 281         enum => [qw(migrate backup snapshot rollback)],
 282     },
 283     cpulimit => {
 284         optional => 1,
 285         type => 'number',
 286         description => "Limit of CPU usage.",
 287         verbose_description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
 288         minimum => 0,
 289         maximum => 128,
 290         default => 0,
 291     },
 292     cpuunits => {
 293         optional => 1,
 294         type => 'integer',
 295         description => "CPU weight for a VM.",
 296         verbose_description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.",
 297         minimum => 2,
 298         maximum => 262144,
 299         default => 1024,
 300     },
 301     memory => {
 302         optional => 1,
 303         type => 'integer',
 304         description => "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
 305         minimum => 16,
 306         default => 512,
 307     },
 308     balloon => {
 309         optional => 1,
 310         type => 'integer',
 311         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 312         minimum => 0,
 313     },
 314     shares => {
 315         optional => 1,
 316         type => 'integer',
 317         description => "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning. Auto-ballooning is done by pvestatd.",
 318         minimum => 0,
 319         maximum => 50000,
 320         default => 1000,
 321     },
 322     keyboard => {
 323         optional => 1,
 324         type => 'string',
 325         description => "Keybord layout for vnc server. Default is read from the '/etc/pve/datacenter.cfg' configuration file.".
 326                        "It should not be necessary to set it.",
 327         enum => PVE::Tools::kvmkeymaplist(),
 328         default => undef,
 329     },
 330     name => {
 331         optional => 1,
 332         type => 'string', format => 'dns-name',
 333         description => "Set a name for the VM. Only used on the configuration web interface.",
 334     },
 335     scsihw => {
 336         optional => 1,
 337         type => 'string',
 338         description => "SCSI controller model",
 339         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 340         default => 'lsi',
 341     },
 342     description => {
 343         optional => 1,
 344         type => 'string',
 345         description => "Description for the VM. Only used on the configuration web interface. This is saved as comment inside the configuration file.",
 346     },
 347     ostype => {
 348         optional => 1,
 349         type => 'string',
 350         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 l24 l26 solaris)],
 351         description => "Specify guest operating system.",
 352         verbose_description => <<EODESC,
 353 Specify guest operating system. This is used to enable special
 354 optimization/features for specific operating systems:
 355
 356 [horizontal]
 357 other;; unspecified OS
 358 wxp;; Microsoft Windows XP
 359 w2k;; Microsoft Windows 2000
 360 w2k3;; Microsoft Windows 2003
 361 w2k8;; Microsoft Windows 2008
 362 wvista;; Microsoft Windows Vista
 363 win7;; Microsoft Windows 7
 364 win8;; Microsoft Windows 8/2012/2012r2
 365 win10;; Microsoft Windows 10/2016
 366 l24;; Linux 2.4 Kernel
 367 l26;; Linux 2.6/3.X Kernel
 368 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
 369 EODESC
 370     },
 371     boot => {
 372         optional => 1,
 373         type => 'string',
 374         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n).",
 375         pattern => '[acdn]{1,4}',
 376         default => 'cdn',
 377     },
 378     bootdisk => {
 379         optional => 1,
 380         type => 'string', format => 'pve-qm-bootdisk',
 381         description => "Enable booting from specified disk.",
 382         pattern => '(ide|sata|scsi|virtio)\d+',
 383     },
 384     smp => {
 385         optional => 1,
 386         type => 'integer',
 387         description => "The number of CPUs. Please use option -sockets instead.",
 388         minimum => 1,
 389         default => 1,
 390     },
 391     sockets => {
 392         optional => 1,
 393         type => 'integer',
 394         description => "The number of CPU sockets.",
 395         minimum => 1,
 396         default => 1,
 397     },
 398     cores => {
 399         optional => 1,
 400         type => 'integer',
 401         description => "The number of cores per socket.",
 402         minimum => 1,
 403         default => 1,
 404     },
 405     numa => {
 406         optional => 1,
 407         type => 'boolean',
 408         description => "Enable/disable NUMA.",
 409         default => 0,
 410     },
 411     hugepages => {
 412         optional => 1,
 413         type => 'string',
 414         description => "Enable/disable hugepages memory.",
 415         enum => [qw(any 2 1024)],
 416     },
 417     vcpus => {
 418         optional => 1,
 419         type => 'integer',
 420         description => "Number of hotplugged vcpus.",
 421         minimum => 1,
 422         default => 0,
 423     },
 424     acpi => {
 425         optional => 1,
 426         type => 'boolean',
 427         description => "Enable/disable ACPI.",
 428         default => 1,
 429     },
 430     agent => {
 431         optional => 1,
 432         description => "Enable/disable Qemu GuestAgent and its properties.",
 433         type => 'string',
 434         format => $agent_fmt,
 435     },
 436     kvm => {
 437         optional => 1,
 438         type => 'boolean',
 439         description => "Enable/disable KVM hardware virtualization.",
 440         default => 1,
 441     },
 442     tdf => {
 443         optional => 1,
 444         type => 'boolean',
 445         description => "Enable/disable time drift fix.",
 446         default => 0,
 447     },
 448     localtime => {
 449         optional => 1,
 450         type => 'boolean',
 451         description => "Set the real time clock to local time. This is enabled by default if ostype indicates a Microsoft OS.",
 452     },
 453     freeze => {
 454         optional => 1,
 455         type => 'boolean',
 456         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 457     },
 458     vga => {
 459         optional => 1,
 460         type => 'string', format => $vga_fmt,
 461         description => "Configure the VGA hardware.",
 462         verbose_description => "Configure the VGA Hardware. If you want to use ".
 463             "high resolution modes (>= 1280x1024x16) you may need to increase " .
 464             "the vga memory option. Since QEMU 2.9 the default VGA display type " .
 465             "is 'std' for all OS types besides some Windows versions (XP and " .
 466             "older) which use 'cirrus'. The 'qxl' option enables the SPICE " .
 467             "display server. For win* OS you can select how many independent " .
 468             "displays you want, Linux guests can add displays them self.\n".
 469             "You can also run without any graphic card, using a serial device as terminal.",
 470     },
 471     watchdog => {
 472         optional => 1,
 473         type => 'string', format => 'pve-qm-watchdog',
 474         description => "Create a virtual hardware watchdog device.",
 475         verbose_description => "Create a virtual hardware watchdog device. Once enabled" .
 476             " (by a guest action), the watchdog must be periodically polled " .
 477             "by an agent inside the guest or else the watchdog will reset " .
 478             "the guest (or execute the respective action specified)",
 479     },
 480     startdate => {
 481         optional => 1,
 482         type => 'string',
 483         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 484         description => "Set the initial date of the real time clock. Valid format for date are: 'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 485         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 486         default => 'now',
 487     },
 488     startup =>  get_standard_option('pve-startup-order'),
 489     template => {
 490         optional => 1,
 491         type => 'boolean',
 492         description => "Enable/disable Template.",
 493         default => 0,
 494     },
 495     args => {
 496         optional => 1,
 497         type => 'string',
 498         description => "Arbitrary arguments passed to kvm.",
 499         verbose_description => <<EODESCR,
 500 Arbitrary arguments passed to kvm, for example:
 501
 502 args: -no-reboot -no-hpet
 503
 504 NOTE: this option is for experts only.
 505 EODESCR
 506     },
 507     tablet => {
 508         optional => 1,
 509         type => 'boolean',
 510         default => 1,
 511         description => "Enable/disable the USB tablet device.",
 512         verbose_description => "Enable/disable the USB tablet device. This device is " .
 513             "usually needed to allow absolute mouse positioning with VNC. " .
 514             "Else the mouse runs out of sync with normal VNC clients. " .
 515             "If you're running lots of console-only guests on one host, " .
 516             "you may consider disabling this to save some context switches. " .
 517             "This is turned off by default if you use spice (-vga=qxl).",
 518     },
 519     migrate_speed => {
 520         optional => 1,
 521         type => 'integer',
 522         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 523         minimum => 0,
 524         default => 0,
 525     },
 526     migrate_downtime => {
 527         optional => 1,
 528         type => 'number',
 529         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 530         minimum => 0,
 531         default => 0.1,
 532     },
 533     cdrom => {
 534         optional => 1,
 535         type => 'string', format => 'pve-qm-ide',
 536         typetext => '<volume>',
 537         description => "This is an alias for option -ide2",
 538     },
 539     cpu => {
 540         optional => 1,
 541         description => "Emulated CPU type.",
 542         type => 'string',
 543         format => $cpu_fmt,
 544     },
 545     parent => get_standard_option('pve-snapshot-name', {
 546         optional => 1,
 547         description => "Parent snapshot name. This is used internally, and should not be modified.",
 548     }),
 549     snaptime => {
 550         optional => 1,
 551         description => "Timestamp for snapshots.",
 552         type => 'integer',
 553         minimum => 0,
 554     },
 555     vmstate => {
 556         optional => 1,
 557         type => 'string', format => 'pve-volume-id',
 558         description => "Reference to a volume which stores the VM state. This is used internally for snapshots.",
 559     },
 560     vmstatestorage => get_standard_option('pve-storage-id', {
 561         description => "Default storage for VM state volumes/files.",
 562         optional => 1,
 563     }),
 564     runningmachine => get_standard_option('pve-qemu-machine', {
 565         description => "Specifies the Qemu machine type of the running vm. This is used internally for snapshots.",
 566     }),
 567     machine => get_standard_option('pve-qemu-machine'),
 568     arch => {
 569         description => "Virtual processor architecture. Defaults to the host.",
 570         optional => 1,
 571         type => 'string',
 572         enum => [qw(x86_64 aarch64)],
 573     },
 574     smbios1 => {
 575         description => "Specify SMBIOS type 1 fields.",
 576         type => 'string', format => 'pve-qm-smbios1',
 577         maxLength => 256,
 578         optional => 1,
 579     },
 580     protection => {
 581         optional => 1,
 582         type => 'boolean',
 583         description => "Sets the protection flag of the VM. This will disable the remove VM and remove disk operations.",
 584         default => 0,
 585     },
 586     bios => {
 587         optional => 1,
 588         type => 'string',
 589         enum => [ qw(seabios ovmf) ],
 590         description => "Select BIOS implementation.",
 591         default => 'seabios',
 592     },
 593     vmgenid => {
 594         type => 'string',
 595         pattern => '(?:[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}|[01])',
 596         format_description => 'UUID',
 597         description => "Set VM Generation ID. Use '1' to autogenerate on create or update, pass '0' to disable explicitly.",
 598         verbose_description => "The VM generation ID (vmgenid) device exposes a".
 599             " 128-bit integer value identifier to the guest OS. This allows to".
 600             " notify the guest operating system when the virtual machine is".
 601             " executed with a different configuration (e.g. snapshot execution".
 602             " or creation from a template). The guest operating system notices".
 603             " the change, and is then able to react as appropriate by marking".
 604             " its copies of distributed databases as dirty, re-initializing its".
 605             " random number generator, etc.\n".
 606             "Note that auto-creation only works when done throug API/CLI create".
 607             " or update methods, but not when manually editing the config file.",
 608         default => "1 (autogenerated)",
 609         optional => 1,
 610     },
 611 };
 612
 613 my $confdesc_cloudinit = {
 614     citype => {
 615         optional => 1,
 616         type => 'string',
 617         description => 'Specifies the cloud-init configuration format. The default depends on the configured operating system type (`ostype`. We use the `nocloud` format for Linux, and `configdrive2` for windows.',
 618         enum => ['configdrive2', 'nocloud'],
 619     },
 620     ciuser => {
 621         optional => 1,
 622         type => 'string',
 623         description => "cloud-init: User name to change ssh keys and password for instead of the image's configured default user.",
 624     },
 625     cipassword => {
 626         optional => 1,
 627         type => 'string',
 628         description => 'cloud-init: Password to assign the user. Using this is generally not recommended. Use ssh keys instead. Also note that older cloud-init versions do not support hashed passwords.',
 629     },
 630     searchdomain => {
 631         optional => 1,
 632         type => 'string',
 633         description => "cloud-init: Sets DNS search domains for a container. Create will automatically use the setting from the host if neither searchdomain nor nameserver are set.",
 634     },
 635     nameserver => {
 636         optional => 1,
 637         type => 'string', format => 'address-list',
 638         description => "cloud-init: Sets DNS server IP address for a container. Create will automatically use the setting from the host if neither searchdomain nor nameserver are set.",
 639     },
 640     sshkeys => {
 641         optional => 1,
 642         type => 'string',
 643         format => 'urlencoded',
 644         description => "cloud-init: Setup public SSH keys (one key per line, OpenSSH format).",
 645     },
 646 };
 647
 648 # what about other qemu settings ?
 649 #cpu => 'string',
 650 #machine => 'string',
 651 #fda => 'file',
 652 #fdb => 'file',
 653 #mtdblock => 'file',
 654 #sd => 'file',
 655 #pflash => 'file',
 656 #snapshot => 'bool',
 657 #bootp => 'file',
 658 ##tftp => 'dir',
 659 ##smb => 'dir',
 660 #kernel => 'file',
 661 #append => 'string',
 662 #initrd => 'file',
 663 ##soundhw => 'string',
 664
 665 while (my ($k, $v) = each %$confdesc) {
 666     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 667 }
 668
 669 my $MAX_IDE_DISKS = 4;
 670 my $MAX_SCSI_DISKS = 14;
 671 my $MAX_VIRTIO_DISKS = 16;
 672 my $MAX_SATA_DISKS = 6;
 673 my $MAX_USB_DEVICES = 5;
 674 my $MAX_NETS = 32;
 675 my $MAX_UNUSED_DISKS = 256;
 676 my $MAX_HOSTPCI_DEVICES = 4;
 677 my $MAX_SERIAL_PORTS = 4;
 678 my $MAX_PARALLEL_PORTS = 3;
 679 my $MAX_NUMA = 8;
 680
 681 my $numa_fmt = {
 682     cpus => {
 683         type => "string",
 684         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 685         description => "CPUs accessing this NUMA node.",
 686         format_description => "id[-id];...",
 687     },
 688     memory => {
 689         type => "number",
 690         description => "Amount of memory this NUMA node provides.",
 691         optional => 1,
 692     },
 693     hostnodes => {
 694         type => "string",
 695         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 696         description => "Host NUMA nodes to use.",
 697         format_description => "id[-id];...",
 698         optional => 1,
 699     },
 700     policy => {
 701         type => 'string',
 702         enum => [qw(preferred bind interleave)],
 703         description => "NUMA allocation policy.",
 704         optional => 1,
 705     },
 706 };
 707 PVE::JSONSchema::register_format('pve-qm-numanode', $numa_fmt);
 708 my $numadesc = {
 709     optional => 1,
 710     type => 'string', format => $numa_fmt,
 711     description => "NUMA topology.",
 712 };
 713 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 714
 715 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 716     $confdesc->{"numa$i"} = $numadesc;
 717 }
 718
 719 my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000',  'pcnet',  'virtio',
 720                       'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3',
 721                       'e1000-82540em', 'e1000-82544gc', 'e1000-82545em'];
 722 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 723
 724 my $net_fmt_bridge_descr = <<__EOD__;
 725 Bridge to attach the network device to. The Proxmox VE standard bridge
 726 is called 'vmbr0'.
 727
 728 If you do not specify a bridge, we create a kvm user (NATed) network
 729 device, which provides DHCP and DNS services. The following addresses
 730 are used:
 731
 732  10.0.2.2   Gateway
 733  10.0.2.3   DNS Server
 734  10.0.2.4   SMB Server
 735
 736 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 737 __EOD__
 738
 739 my $net_fmt = {
 740     macaddr => {
 741         type => 'string',
 742         pattern => qr/[0-9a-f]{2}(?::[0-9a-f]{2}){5}/i,
 743         description => "MAC address. That address must be unique withing your network. This is automatically generated if not specified.",
 744         format_description => "XX:XX:XX:XX:XX:XX",
 745         optional => 1,
 746     },
 747     model => {
 748         type => 'string',
 749         description => "Network Card Model. The 'virtio' model provides the best performance with very low CPU overhead. If your guest does not support this driver, it is usually best to use 'e1000'.",
 750         enum => $nic_model_list,
 751         default_key => 1,
 752     },
 753     (map { $_ => { keyAlias => 'model', alias => 'macaddr' }} @$nic_model_list),
 754     bridge => {
 755         type => 'string',
 756         description => $net_fmt_bridge_descr,
 757         format_description => 'bridge',
 758         optional => 1,
 759     },
 760     queues => {
 761         type => 'integer',
 762         minimum => 0, maximum => 16,
 763         description => 'Number of packet queues to be used on the device.',
 764         optional => 1,
 765     },
 766     rate => {
 767         type => 'number',
 768         minimum => 0,
 769         description => "Rate limit in mbps (megabytes per second) as floating point number.",
 770         optional => 1,
 771     },
 772     tag => {
 773         type => 'integer',
 774         minimum => 1, maximum => 4094,
 775         description => 'VLAN tag to apply to packets on this interface.',
 776         optional => 1,
 777     },
 778     trunks => {
 779         type => 'string',
 780         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 781         description => 'VLAN trunks to pass through this interface.',
 782         format_description => 'vlanid[;vlanid...]',
 783         optional => 1,
 784     },
 785     firewall => {
 786         type => 'boolean',
 787         description => 'Whether this interface should be protected by the firewall.',
 788         optional => 1,
 789     },
 790     link_down => {
 791         type => 'boolean',
 792         description => 'Whether this interface should be disconnected (like pulling the plug).',
 793         optional => 1,
 794     },
 795 };
 796
 797 my $netdesc = {
 798     optional => 1,
 799     type => 'string', format => $net_fmt,
 800     description => "Specify network devices.",
 801 };
 802
 803 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 804
 805 my $ipconfig_fmt = {
 806     ip => {
 807         type => 'string',
 808         format => 'pve-ipv4-config',
 809         format_description => 'IPv4Format/CIDR',
 810         description => 'IPv4 address in CIDR format.',
 811         optional => 1,
 812         default => 'dhcp',
 813     },
 814     gw => {
 815         type => 'string',
 816         format => 'ipv4',
 817         format_description => 'GatewayIPv4',
 818         description => 'Default gateway for IPv4 traffic.',
 819         optional => 1,
 820         requires => 'ip',
 821     },
 822     ip6 => {
 823         type => 'string',
 824         format => 'pve-ipv6-config',
 825         format_description => 'IPv6Format/CIDR',
 826         description => 'IPv6 address in CIDR format.',
 827         optional => 1,
 828         default => 'dhcp',
 829     },
 830     gw6 => {
 831         type => 'string',
 832         format => 'ipv6',
 833         format_description => 'GatewayIPv6',
 834         description => 'Default gateway for IPv6 traffic.',
 835         optional => 1,
 836         requires => 'ip6',
 837     },
 838 };
 839 PVE::JSONSchema::register_format('pve-qm-ipconfig', $ipconfig_fmt);
 840 my $ipconfigdesc = {
 841     optional => 1,
 842     type => 'string', format => 'pve-qm-ipconfig',
 843     description => <<'EODESCR',
 844 cloud-init: Specify IP addresses and gateways for the corresponding interface.
 845
 846 IP addresses use CIDR notation, gateways are optional but need an IP of the same type specified.
 847
 848 The special string 'dhcp' can be used for IP addresses to use DHCP, in which case no explicit gateway should be provided.
 849 For IPv6 the special string 'auto' can be used to use stateless autoconfiguration.
 850
 851 If cloud-init is enabled and neither an IPv4 nor an IPv6 address is specified, it defaults to using dhcp on IPv4.
 852 EODESCR
 853 };
 854 PVE::JSONSchema::register_standard_option("pve-qm-ipconfig", $netdesc);
 855
 856 for (my $i = 0; $i < $MAX_NETS; $i++)  {
 857     $confdesc->{"net$i"} = $netdesc;
 858     $confdesc_cloudinit->{"ipconfig$i"} = $ipconfigdesc;
 859 }
 860
 861 foreach my $key (keys %$confdesc_cloudinit) {
 862     $confdesc->{$key} = $confdesc_cloudinit->{$key};
 863 }
 864
 865 PVE::JSONSchema::register_format('pve-volume-id-or-qm-path', \&verify_volume_id_or_qm_path);
 866 sub verify_volume_id_or_qm_path {
 867     my ($volid, $noerr) = @_;
 868
 869     if ($volid eq 'none' || $volid eq 'cdrom' || $volid =~ m|^/|) {
 870         return $volid;
 871     }
 872
 873     # if its neither 'none' nor 'cdrom' nor a path, check if its a volume-id
 874     $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') };
 875     if ($@) {
 876         return undef if $noerr;
 877         die $@;
 878     }
 879     return $volid;
 880 }
 881
 882 my $drivename_hash;
 883
 884 my %drivedesc_base = (
 885     volume => { alias => 'file' },
 886     file => {
 887         type => 'string',
 888         format => 'pve-volume-id-or-qm-path',
 889         default_key => 1,
 890         format_description => 'volume',
 891         description => "The drive's backing volume.",
 892     },
 893     media => {
 894         type => 'string',
 895         enum => [qw(cdrom disk)],
 896         description => "The drive's media type.",
 897         default => 'disk',
 898         optional => 1
 899     },
 900     cyls => {
 901         type => 'integer',
 902         description => "Force the drive's physical geometry to have a specific cylinder count.",
 903         optional => 1
 904     },
 905     heads => {
 906         type => 'integer',
 907         description => "Force the drive's physical geometry to have a specific head count.",
 908         optional => 1
 909     },
 910     secs => {
 911         type => 'integer',
 912         description => "Force the drive's physical geometry to have a specific sector count.",
 913         optional => 1
 914     },
 915     trans => {
 916         type => 'string',
 917         enum => [qw(none lba auto)],
 918         description => "Force disk geometry bios translation mode.",
 919         optional => 1,
 920     },
 921     snapshot => {
 922         type => 'boolean',
 923         description => "Controls qemu's snapshot mode feature."
 924             . " If activated, changes made to the disk are temporary and will"
 925             . " be discarded when the VM is shutdown.",
 926         optional => 1,
 927     },
 928     cache => {
 929         type => 'string',
 930         enum => [qw(none writethrough writeback unsafe directsync)],
 931         description => "The drive's cache mode",
 932         optional => 1,
 933     },
 934     format => get_standard_option('pve-qm-image-format'),
 935     size => {
 936         type => 'string',
 937         format => 'disk-size',
 938         format_description => 'DiskSize',
 939         description => "Disk size. This is purely informational and has no effect.",
 940         optional => 1,
 941     },
 942     backup => {
 943         type => 'boolean',
 944         description => "Whether the drive should be included when making backups.",
 945         optional => 1,
 946     },
 947     replicate => {
 948         type => 'boolean',
 949         description => 'Whether the drive should considered for replication jobs.',
 950         optional => 1,
 951         default => 1,
 952     },
 953     rerror => {
 954         type => 'string',
 955         enum => [qw(ignore report stop)],
 956         description => 'Read error action.',
 957         optional => 1,
 958     },
 959     werror => {
 960         type => 'string',
 961         enum => [qw(enospc ignore report stop)],
 962         description => 'Write error action.',
 963         optional => 1,
 964     },
 965     aio => {
 966         type => 'string',
 967         enum => [qw(native threads)],
 968         description => 'AIO type to use.',
 969         optional => 1,
 970     },
 971     discard => {
 972         type => 'string',
 973         enum => [qw(ignore on)],
 974         description => 'Controls whether to pass discard/trim requests to the underlying storage.',
 975         optional => 1,
 976     },
 977     detect_zeroes => {
 978         type => 'boolean',
 979         description => 'Controls whether to detect and try to optimize writes of zeroes.',
 980         optional => 1,
 981     },
 982     serial => {
 983         type => 'string',
 984         format => 'urlencoded',
 985         format_description => 'serial',
 986         maxLength => 20*3, # *3 since it's %xx url enoded
 987         description => "The drive's reported serial number, url-encoded, up to 20 bytes long.",
 988         optional => 1,
 989     },
 990     shared => {
 991         type => 'boolean',
 992         description => 'Mark this locally-managed volume as available on all nodes',
 993         verbose_description => "Mark this locally-managed volume as available on all nodes.\n\nWARNING: This option does not share the volume automatically, it assumes it is shared already!",
 994         optional => 1,
 995         default => 0,
 996     }
 997 );
 998
 999 my %iothread_fmt = ( iothread => {
1000         type => 'boolean',
1001         description => "Whether to use iothreads for this drive",
1002         optional => 1,
1003 });
1004
1005 my %model_fmt = (
1006     model => {
1007         type => 'string',
1008         format => 'urlencoded',
1009         format_description => 'model',
1010         maxLength => 40*3, # *3 since it's %xx url enoded
1011         description => "The drive's reported model name, url-encoded, up to 40 bytes long.",
1012         optional => 1,
1013     },
1014 );
1015
1016 my %queues_fmt = (
1017     queues => {
1018         type => 'integer',
1019         description => "Number of queues.",
1020         minimum => 2,
1021         optional => 1
1022     }
1023 );
1024
1025 my %scsiblock_fmt = (
1026     scsiblock => {
1027         type => 'boolean',
1028         description => "whether to use scsi-block for full passthrough of host block device\n\nWARNING: can lead to I/O errors in combination with low memory or high memory fragmentation on host",
1029         optional => 1,
1030         default => 0,
1031     },
1032 );
1033
1034 my %ssd_fmt = (
1035     ssd => {
1036         type => 'boolean',
1037         description => "Whether to expose this drive as an SSD, rather than a rotational hard disk.",
1038         optional => 1,
1039     },
1040 );
1041
1042 my $add_throttle_desc = sub {
1043     my ($key, $type, $what, $unit, $longunit, $minimum) = @_;
1044     my $d = {
1045         type => $type,
1046         format_description => $unit,
1047         description => "Maximum $what in $longunit.",
1048         optional => 1,
1049     };
1050     $d->{minimum} = $minimum if defined($minimum);
1051     $drivedesc_base{$key} = $d;
1052 };
1053 # throughput: (leaky bucket)
1054 $add_throttle_desc->('bps',     'integer', 'r/w speed',   'bps',  'bytes per second');
1055 $add_throttle_desc->('bps_rd',  'integer', 'read speed',  'bps',  'bytes per second');
1056 $add_throttle_desc->('bps_wr',  'integer', 'write speed', 'bps',  'bytes per second');
1057 $add_throttle_desc->('mbps',    'number',  'r/w speed',   'mbps', 'megabytes per second');
1058 $add_throttle_desc->('mbps_rd', 'number',  'read speed',  'mbps', 'megabytes per second');
1059 $add_throttle_desc->('mbps_wr', 'number',  'write speed', 'mbps', 'megabytes per second');
1060 $add_throttle_desc->('iops',    'integer', 'r/w I/O',     'iops', 'operations per second');
1061 $add_throttle_desc->('iops_rd', 'integer', 'read I/O',    'iops', 'operations per second');
1062 $add_throttle_desc->('iops_wr', 'integer', 'write I/O',   'iops', 'operations per second');
1063
1064 # pools: (pool of IO before throttling starts taking effect)
1065 $add_throttle_desc->('mbps_max',    'number',  'unthrottled r/w pool',       'mbps', 'megabytes per second');
1066 $add_throttle_desc->('mbps_rd_max', 'number',  'unthrottled read pool',      'mbps', 'megabytes per second');
1067 $add_throttle_desc->('mbps_wr_max', 'number',  'unthrottled write pool',     'mbps', 'megabytes per second');
1068 $add_throttle_desc->('iops_max',    'integer', 'unthrottled r/w I/O pool',   'iops', 'operations per second');
1069 $add_throttle_desc->('iops_rd_max', 'integer', 'unthrottled read I/O pool',  'iops', 'operations per second');
1070 $add_throttle_desc->('iops_wr_max', 'integer', 'unthrottled write I/O pool', 'iops', 'operations per second');
1071
1072 # burst lengths
1073 $add_throttle_desc->('bps_max_length',     'integer', 'length of I/O bursts',       'seconds', 'seconds', 1);
1074 $add_throttle_desc->('bps_rd_max_length',  'integer', 'length of read I/O bursts',  'seconds', 'seconds', 1);
1075 $add_throttle_desc->('bps_wr_max_length',  'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
1076 $add_throttle_desc->('iops_max_length',    'integer', 'length of I/O bursts',       'seconds', 'seconds', 1);
1077 $add_throttle_desc->('iops_rd_max_length', 'integer', 'length of read I/O bursts',  'seconds', 'seconds', 1);
1078 $add_throttle_desc->('iops_wr_max_length', 'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
1079
1080 # legacy support
1081 $drivedesc_base{'bps_rd_length'} = { alias => 'bps_rd_max_length' };
1082 $drivedesc_base{'bps_wr_length'} = { alias => 'bps_wr_max_length' };
1083 $drivedesc_base{'iops_rd_length'} = { alias => 'iops_rd_max_length' };
1084 $drivedesc_base{'iops_wr_length'} = { alias => 'iops_wr_max_length' };
1085
1086 my $ide_fmt = {
1087     %drivedesc_base,
1088     %model_fmt,
1089     %ssd_fmt,
1090 };
1091 PVE::JSONSchema::register_format("pve-qm-ide", $ide_fmt);
1092
1093 my $idedesc = {
1094     optional => 1,
1095     type => 'string', format => $ide_fmt,
1096     description => "Use volume as IDE hard disk or CD-ROM (n is 0 to " .($MAX_IDE_DISKS -1) . ").",
1097 };
1098 PVE::JSONSchema::register_standard_option("pve-qm-ide", $idedesc);
1099
1100 my $scsi_fmt = {
1101     %drivedesc_base,
1102     %iothread_fmt,
1103     %queues_fmt,
1104     %scsiblock_fmt,
1105     %ssd_fmt,
1106 };
1107 my $scsidesc = {
1108     optional => 1,
1109     type => 'string', format => $scsi_fmt,
1110     description => "Use volume as SCSI hard disk or CD-ROM (n is 0 to " . ($MAX_SCSI_DISKS - 1) . ").",
1111 };
1112 PVE::JSONSchema::register_standard_option("pve-qm-scsi", $scsidesc);
1113
1114 my $sata_fmt = {
1115     %drivedesc_base,
1116     %ssd_fmt,
1117 };
1118 my $satadesc = {
1119     optional => 1,
1120     type => 'string', format => $sata_fmt,
1121     description => "Use volume as SATA hard disk or CD-ROM (n is 0 to " . ($MAX_SATA_DISKS - 1). ").",
1122 };
1123 PVE::JSONSchema::register_standard_option("pve-qm-sata", $satadesc);
1124
1125 my $virtio_fmt = {
1126     %drivedesc_base,
1127     %iothread_fmt,
1128 };
1129 my $virtiodesc = {
1130     optional => 1,
1131     type => 'string', format => $virtio_fmt,
1132     description => "Use volume as VIRTIO hard disk (n is 0 to " . ($MAX_VIRTIO_DISKS - 1) . ").",
1133 };
1134 PVE::JSONSchema::register_standard_option("pve-qm-virtio", $virtiodesc);
1135
1136 my $alldrive_fmt = {
1137     %drivedesc_base,
1138     %iothread_fmt,
1139     %model_fmt,
1140     %queues_fmt,
1141     %scsiblock_fmt,
1142     %ssd_fmt,
1143 };
1144
1145 my $efidisk_fmt = {
1146     volume => { alias => 'file' },
1147     file => {
1148         type => 'string',
1149         format => 'pve-volume-id-or-qm-path',
1150         default_key => 1,
1151         format_description => 'volume',
1152         description => "The drive's backing volume.",
1153     },
1154     format => get_standard_option('pve-qm-image-format'),
1155     size => {
1156         type => 'string',
1157         format => 'disk-size',
1158         format_description => 'DiskSize',
1159         description => "Disk size. This is purely informational and has no effect.",
1160         optional => 1,
1161     },
1162 };
1163
1164 my $efidisk_desc = {
1165     optional => 1,
1166     type => 'string', format => $efidisk_fmt,
1167     description => "Configure a Disk for storing EFI vars",
1168 };
1169
1170 PVE::JSONSchema::register_standard_option("pve-qm-efidisk", $efidisk_desc);
1171
1172 my $usb_fmt = {
1173     host => {
1174         default_key => 1,
1175         type => 'string', format => 'pve-qm-usb-device',
1176         format_description => 'HOSTUSBDEVICE|spice',
1177         description => <<EODESCR,
1178 The Host USB device or port or the value 'spice'. HOSTUSBDEVICE syntax is:
1179
1180  'bus-port(.port)*' (decimal numbers) or
1181  'vendor_id:product_id' (hexadeciaml numbers) or
1182  'spice'
1183
1184 You can use the 'lsusb -t' command to list existing usb devices.
1185
1186 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1187
1188 The value 'spice' can be used to add a usb redirection devices for spice.
1189 EODESCR
1190     },
1191     usb3 => {
1192         optional => 1,
1193         type => 'boolean',
1194         description => "Specifies whether if given host option is a USB3 device or port (this does currently not work reliably with spice redirection and is then ignored).",
1195         default => 0,
1196     },
1197 };
1198
1199 my $usbdesc = {
1200     optional => 1,
1201     type => 'string', format => $usb_fmt,
1202     description => "Configure an USB device (n is 0 to 4).",
1203 };
1204 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
1205
1206 # NOTE: the match-groups of this regex are used in parse_hostpci
1207 my $PCIRE = qr/([a-f0-9]{2}:[a-f0-9]{2})(?:\.([a-f0-9]))?/;
1208 my $hostpci_fmt = {
1209     host => {
1210         default_key => 1,
1211         type => 'string',
1212         pattern => qr/$PCIRE(;$PCIRE)*/,
1213         format_description => 'HOSTPCIID[;HOSTPCIID2...]',
1214         description => <<EODESCR,
1215 Host PCI device pass through. The PCI ID of a host's PCI device or a list
1216 of PCI virtual functions of the host. HOSTPCIID syntax is:
1217
1218 'bus:dev.func' (hexadecimal numbers)
1219
1220 You can us the 'lspci' command to list existing PCI devices.
1221 EODESCR
1222     },
1223     rombar => {
1224         type => 'boolean',
1225         description =>  "Specify whether or not the device's ROM will be visible in the guest's memory map.",
1226         optional => 1,
1227         default => 1,
1228     },
1229     romfile => {
1230         type => 'string',
1231         pattern => '[^,;]+',
1232         format_description => 'string',
1233         description => "Custom pci device rom filename (must be located in /usr/share/kvm/).",
1234         optional => 1,
1235     },
1236     pcie => {
1237         type => 'boolean',
1238         description =>  "Choose the PCI-express bus (needs the 'q35' machine model).",
1239         optional => 1,
1240         default => 0,
1241     },
1242     'x-vga' => {
1243         type => 'boolean',
1244         description =>  "Enable vfio-vga device support.",
1245         optional => 1,
1246         default => 0,
1247     },
1248 };
1249 PVE::JSONSchema::register_format('pve-qm-hostpci', $hostpci_fmt);
1250
1251 my $hostpcidesc = {
1252         optional => 1,
1253         type => 'string', format => 'pve-qm-hostpci',
1254         description => "Map host PCI devices into guest.",
1255         verbose_description =>  <<EODESCR,
1256 Map host PCI devices into guest.
1257
1258 NOTE: This option allows direct access to host hardware. So it is no longer
1259 possible to migrate such machines - use with special care.
1260
1261 CAUTION: Experimental! User reported problems with this option.
1262 EODESCR
1263 };
1264 PVE::JSONSchema::register_standard_option("pve-qm-hostpci", $hostpcidesc);
1265
1266 my $serialdesc = {
1267         optional => 1,
1268         type => 'string',
1269         pattern => '(/dev/.+|socket)',
1270         description =>  "Create a serial device inside the VM (n is 0 to 3)",
1271         verbose_description =>  <<EODESCR,
1272 Create a serial device inside the VM (n is 0 to 3), and pass through a
1273 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1274 host side (use 'qm terminal' to open a terminal connection).
1275
1276 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines - use with special care.
1277
1278 CAUTION: Experimental! User reported problems with this option.
1279 EODESCR
1280 };
1281
1282 my $paralleldesc= {
1283         optional => 1,
1284         type => 'string',
1285         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
1286         description =>  "Map host parallel devices (n is 0 to 2).",
1287         verbose_description =>  <<EODESCR,
1288 Map host parallel devices (n is 0 to 2).
1289
1290 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1291
1292 CAUTION: Experimental! User reported problems with this option.
1293 EODESCR
1294 };
1295
1296 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
1297     $confdesc->{"parallel$i"} = $paralleldesc;
1298 }
1299
1300 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
1301     $confdesc->{"serial$i"} = $serialdesc;
1302 }
1303
1304 for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
1305     $confdesc->{"hostpci$i"} = $hostpcidesc;
1306 }
1307
1308 for (my $i = 0; $i < $MAX_IDE_DISKS; $i++)  {
1309     $drivename_hash->{"ide$i"} = 1;
1310     $confdesc->{"ide$i"} = $idedesc;
1311 }
1312
1313 for (my $i = 0; $i < $MAX_SATA_DISKS; $i++)  {
1314     $drivename_hash->{"sata$i"} = 1;
1315     $confdesc->{"sata$i"} = $satadesc;
1316 }
1317
1318 for (my $i = 0; $i < $MAX_SCSI_DISKS; $i++)  {
1319     $drivename_hash->{"scsi$i"} = 1;
1320     $confdesc->{"scsi$i"} = $scsidesc ;
1321 }
1322
1323 for (my $i = 0; $i < $MAX_VIRTIO_DISKS; $i++)  {
1324     $drivename_hash->{"virtio$i"} = 1;
1325     $confdesc->{"virtio$i"} = $virtiodesc;
1326 }
1327
1328 $drivename_hash->{efidisk0} = 1;
1329 $confdesc->{efidisk0} = $efidisk_desc;
1330
1331 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
1332     $confdesc->{"usb$i"} = $usbdesc;
1333 }
1334
1335 my $unuseddesc = {
1336     optional => 1,
1337     type => 'string', format => 'pve-volume-id',
1338     description => "Reference to unused volumes. This is used internally, and should not be modified manually.",
1339 };
1340
1341 for (my $i = 0; $i < $MAX_UNUSED_DISKS; $i++)  {
1342     $confdesc->{"unused$i"} = $unuseddesc;
1343 }
1344
1345 my $kvm_api_version = 0;
1346
1347 sub kvm_version {
1348     return $kvm_api_version if $kvm_api_version;
1349
1350     open my $fh, '<', '/dev/kvm'
1351         or return undef;
1352
1353     # 0xae00 => KVM_GET_API_VERSION
1354     $kvm_api_version = ioctl($fh, 0xae00, 0);
1355
1356     return $kvm_api_version;
1357 }
1358
1359 my $kvm_user_version;
1360
1361 sub kvm_user_version {
1362
1363     return $kvm_user_version if $kvm_user_version;
1364
1365     $kvm_user_version = 'unknown';
1366
1367     my $code = sub {
1368         my $line = shift;
1369         if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1370             $kvm_user_version = $2;
1371         }
1372     };
1373
1374     eval { run_command("kvm -version", outfunc => $code); };
1375     warn $@ if $@;
1376
1377     return $kvm_user_version;
1378
1379 }
1380
1381 my $kernel_has_vhost_net = -c '/dev/vhost-net';
1382
1383 sub valid_drive_names {
1384     # order is important - used to autoselect boot disk
1385     return ((map { "ide$_" } (0 .. ($MAX_IDE_DISKS - 1))),
1386             (map { "scsi$_" } (0 .. ($MAX_SCSI_DISKS - 1))),
1387             (map { "virtio$_" } (0 .. ($MAX_VIRTIO_DISKS - 1))),
1388             (map { "sata$_" } (0 .. ($MAX_SATA_DISKS - 1))),
1389             'efidisk0');
1390 }
1391
1392 sub is_valid_drivename {
1393     my $dev = shift;
1394
1395     return defined($drivename_hash->{$dev});
1396 }
1397
1398 sub option_exists {
1399     my $key = shift;
1400     return defined($confdesc->{$key});
1401 }
1402
1403 sub nic_models {
1404     return $nic_model_list;
1405 }
1406
1407 sub os_list_description {
1408
1409     return {
1410         other => 'Other',
1411         wxp => 'Windows XP',
1412         w2k => 'Windows 2000',
1413         w2k3 =>, 'Windows 2003',
1414         w2k8 => 'Windows 2008',
1415         wvista => 'Windows Vista',
1416         win7 => 'Windows 7',
1417         win8 => 'Windows 8/2012',
1418         win10 => 'Windows 10/2016',
1419         l24 => 'Linux 2.4',
1420         l26 => 'Linux 2.6',
1421     };
1422 }
1423
1424 my $cdrom_path;
1425
1426 sub get_cdrom_path {
1427
1428     return  $cdrom_path if $cdrom_path;
1429
1430     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
1431     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
1432     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
1433 }
1434
1435 sub get_iso_path {
1436     my ($storecfg, $vmid, $cdrom) = @_;
1437
1438     if ($cdrom eq 'cdrom') {
1439         return get_cdrom_path();
1440     } elsif ($cdrom eq 'none') {
1441         return '';
1442     } elsif ($cdrom =~ m|^/|) {
1443         return $cdrom;
1444     } else {
1445         return PVE::Storage::path($storecfg, $cdrom);
1446     }
1447 }
1448
1449 # try to convert old style file names to volume IDs
1450 sub filename_to_volume_id {
1451     my ($vmid, $file, $media) = @_;
1452
1453      if (!($file eq 'none' || $file eq 'cdrom' ||
1454           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1455
1456         return undef if $file =~ m|/|;
1457
1458         if ($media && $media eq 'cdrom') {
1459             $file = "local:iso/$file";
1460         } else {
1461             $file = "local:$vmid/$file";
1462         }
1463     }
1464
1465     return $file;
1466 }
1467
1468 sub verify_media_type {
1469     my ($opt, $vtype, $media) = @_;
1470
1471     return if !$media;
1472
1473     my $etype;
1474     if ($media eq 'disk') {
1475         $etype = 'images';
1476     } elsif ($media eq 'cdrom') {
1477         $etype = 'iso';
1478     } else {
1479         die "internal error";
1480     }
1481
1482     return if ($vtype eq $etype);
1483
1484     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1485 }
1486
1487 sub cleanup_drive_path {
1488     my ($opt, $storecfg, $drive) = @_;
1489
1490     # try to convert filesystem paths to volume IDs
1491
1492     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1493         ($drive->{file} !~ m|^/dev/.+|) &&
1494         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1495         ($drive->{file} !~ m/^\d+$/)) {
1496         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1497         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"}) if !$vtype;
1498         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1499         verify_media_type($opt, $vtype, $drive->{media});
1500         $drive->{file} = $volid;
1501     }
1502
1503     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1504 }
1505
1506 sub parse_hotplug_features {
1507     my ($data) = @_;
1508
1509     my $res = {};
1510
1511     return $res if $data eq '0';
1512
1513     $data = $confdesc->{hotplug}->{default} if $data eq '1';
1514
1515     foreach my $feature (PVE::Tools::split_list($data)) {
1516         if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
1517             $res->{$1} = 1;
1518         } else {
1519             die "invalid hotplug feature '$feature'\n";
1520         }
1521     }
1522     return $res;
1523 }
1524
1525 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1526 sub pve_verify_hotplug_features {
1527     my ($value, $noerr) = @_;
1528
1529     return $value if parse_hotplug_features($value);
1530
1531     return undef if $noerr;
1532
1533     die "unable to parse hotplug option\n";
1534 }
1535
1536 # ideX = [volume=]volume-id[,media=d][,cyls=c,heads=h,secs=s[,trans=t]]
1537 #        [,snapshot=on|off][,cache=on|off][,format=f][,backup=yes|no]
1538 #        [,rerror=ignore|report|stop][,werror=enospc|ignore|report|stop]
1539 #        [,aio=native|threads][,discard=ignore|on][,detect_zeroes=on|off]
1540 #        [,iothread=on][,serial=serial][,model=model]
1541
1542 sub parse_drive {
1543     my ($key, $data) = @_;
1544
1545     my ($interface, $index);
1546
1547     if ($key =~ m/^([^\d]+)(\d+)$/) {
1548         $interface = $1;
1549         $index = $2;
1550     } else {
1551         return undef;
1552     }
1553
1554     my $desc = $key =~ /^unused\d+$/ ? $alldrive_fmt
1555                                      : $confdesc->{$key}->{format};
1556     if (!$desc) {
1557         warn "invalid drive key: $key\n";
1558         return undef;
1559     }
1560     my $res = eval { PVE::JSONSchema::parse_property_string($desc, $data) };
1561     return undef if !$res;
1562     $res->{interface} = $interface;
1563     $res->{index} = $index;
1564
1565     my $error = 0;
1566     foreach my $opt (qw(bps bps_rd bps_wr)) {
1567         if (my $bps = defined(delete $res->{$opt})) {
1568             if (defined($res->{"m$opt"})) {
1569                 warn "both $opt and m$opt specified\n";
1570                 ++$error;
1571                 next;
1572             }
1573             $res->{"m$opt"} = sprintf("%.3f", $bps / (1024*1024.0));
1574         }
1575     }
1576
1577     # can't use the schema's 'requires' because of the mbps* => bps* "transforming aliases"
1578     for my $requirement (
1579         [mbps_max => 'mbps'],
1580         [mbps_rd_max => 'mbps_rd'],
1581         [mbps_wr_max => 'mbps_wr'],
1582         [miops_max => 'miops'],
1583         [miops_rd_max => 'miops_rd'],
1584         [miops_wr_max => 'miops_wr'],
1585         [bps_max_length => 'mbps_max'],
1586         [bps_rd_max_length => 'mbps_rd_max'],
1587         [bps_wr_max_length => 'mbps_wr_max'],
1588         [iops_max_length => 'iops_max'],
1589         [iops_rd_max_length => 'iops_rd_max'],
1590         [iops_wr_max_length => 'iops_wr_max']) {
1591         my ($option, $requires) = @$requirement;
1592         if ($res->{$option} && !$res->{$requires}) {
1593             warn "$option requires $requires\n";
1594             ++$error;
1595         }
1596     }
1597
1598     return undef if $error;
1599
1600     return undef if $res->{mbps_rd} && $res->{mbps};
1601     return undef if $res->{mbps_wr} && $res->{mbps};
1602     return undef if $res->{iops_rd} && $res->{iops};
1603     return undef if $res->{iops_wr} && $res->{iops};
1604
1605     if ($res->{media} && ($res->{media} eq 'cdrom')) {
1606         return undef if $res->{snapshot} || $res->{trans} || $res->{format};
1607         return undef if $res->{heads} || $res->{secs} || $res->{cyls};
1608         return undef if $res->{interface} eq 'virtio';
1609     }
1610
1611     if (my $size = $res->{size}) {
1612         return undef if !defined($res->{size} = PVE::JSONSchema::parse_size($size));
1613     }
1614
1615     return $res;
1616 }
1617
1618 sub print_drive {
1619     my ($vmid, $drive) = @_;
1620     my $data = { %$drive };
1621     delete $data->{$_} for qw(index interface);
1622     return PVE::JSONSchema::print_property_string($data, $alldrive_fmt);
1623 }
1624
1625 sub scsi_inquiry {
1626     my($fh, $noerr) = @_;
1627
1628     my $SG_IO = 0x2285;
1629     my $SG_GET_VERSION_NUM = 0x2282;
1630
1631     my $versionbuf = "\x00" x 8;
1632     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1633     if (!$ret) {
1634         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1635         return undef;
1636     }
1637     my $version = unpack("I", $versionbuf);
1638     if ($version < 30000) {
1639         die "scsi generic interface too old\n"  if !$noerr;
1640         return undef;
1641     }
1642
1643     my $buf = "\x00" x 36;
1644     my $sensebuf = "\x00" x 8;
1645     my $cmd = pack("C x3 C x1", 0x12, 36);
1646
1647     # see /usr/include/scsi/sg.h
1648     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1649
1650     my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1651                       length($sensebuf), 0, length($buf), $buf,
1652                       $cmd, $sensebuf, 6000);
1653
1654     $ret = ioctl($fh, $SG_IO, $packet);
1655     if (!$ret) {
1656         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1657         return undef;
1658     }
1659
1660     my @res = unpack($sg_io_hdr_t, $packet);
1661     if ($res[17] || $res[18]) {
1662         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1663         return undef;
1664     }
1665
1666     my $res = {};
1667     (my $byte0, my $byte1, $res->{vendor},
1668      $res->{product}, $res->{revision}) = unpack("C C x6 A8 A16 A4", $buf);
1669
1670     $res->{removable} = $byte1 & 128 ? 1 : 0;
1671     $res->{type} = $byte0 & 31;
1672
1673     return $res;
1674 }
1675
1676 sub path_is_scsi {
1677     my ($path) = @_;
1678
1679     my $fh = IO::File->new("+<$path") || return undef;
1680     my $res = scsi_inquiry($fh, 1);
1681     close($fh);
1682
1683     return $res;
1684 }
1685
1686 sub machine_type_is_q35 {
1687     my ($conf) = @_;
1688
1689     return $conf->{machine} && ($conf->{machine} =~ m/q35/) ? 1 : 0;
1690 }
1691
1692 sub print_tabletdevice_full {
1693     my ($conf, $arch) = @_;
1694
1695     my $q35 = machine_type_is_q35($conf);
1696
1697     # we use uhci for old VMs because tablet driver was buggy in older qemu
1698     my $usbbus;
1699     if (machine_type_is_q35($conf) || $arch eq 'aarch64') {
1700         $usbbus = 'ehci';
1701     } else {
1702         $usbbus = 'uhci';
1703     }
1704
1705     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1706 }
1707
1708 sub print_keyboarddevice_full {
1709     my ($conf, $arch, $machine) = @_;
1710
1711     return undef if $arch ne 'aarch64';
1712
1713     return "usb-kbd,id=keyboard,bus=ehci.0,port=2";
1714 }
1715
1716 sub print_drivedevice_full {
1717     my ($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type) = @_;
1718
1719     my $device = '';
1720     my $maxdev = 0;
1721
1722     if ($drive->{interface} eq 'virtio') {
1723         my $pciaddr = print_pci_addr("$drive->{interface}$drive->{index}", $bridges, $arch, $machine_type);
1724         $device = "virtio-blk-pci,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}$pciaddr";
1725         $device .= ",iothread=iothread-$drive->{interface}$drive->{index}" if $drive->{iothread};
1726     } elsif ($drive->{interface} eq 'scsi') {
1727
1728         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1729         my $unit = $drive->{index} % $maxdev;
1730         my $devicetype = 'hd';
1731         my $path = '';
1732         if (drive_is_cdrom($drive)) {
1733             $devicetype = 'cd';
1734         } else {
1735             if ($drive->{file} =~ m|^/|) {
1736                 $path = $drive->{file};
1737                 if (my $info = path_is_scsi($path)) {
1738                     if ($info->{type} == 0 && $drive->{scsiblock}) {
1739                         $devicetype = 'block';
1740                     } elsif ($info->{type} == 1) { # tape
1741                         $devicetype = 'generic';
1742                     }
1743                 }
1744             } else {
1745                  $path = PVE::Storage::path($storecfg, $drive->{file});
1746             }
1747
1748             if($path =~ m/^iscsi\:\/\//){
1749                 $devicetype = 'generic';
1750             }
1751         }
1752
1753         if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)){
1754             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1755         } else {
1756             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0,lun=$drive->{index},drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1757         }
1758
1759         if ($drive->{ssd} && ($devicetype eq 'block' || $devicetype eq 'hd')) {
1760             $device .= ",rotation_rate=1";
1761         }
1762
1763     } elsif ($drive->{interface} eq 'ide' || $drive->{interface} eq 'sata') {
1764         my $maxdev = ($drive->{interface} eq 'sata') ? $MAX_SATA_DISKS : 2;
1765         my $controller = int($drive->{index} / $maxdev);
1766         my $unit = $drive->{index} % $maxdev;
1767         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1768
1769         $device = "ide-$devicetype";
1770         if ($drive->{interface} eq 'ide') {
1771             $device .= ",bus=ide.$controller,unit=$unit";
1772         } else {
1773             $device .= ",bus=ahci$controller.$unit";
1774         }
1775         $device .= ",drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1776
1777         if ($devicetype eq 'hd') {
1778             if (my $model = $drive->{model}) {
1779                 $model = URI::Escape::uri_unescape($model);
1780                 $device .= ",model=$model";
1781             }
1782             if ($drive->{ssd}) {
1783                 $device .= ",rotation_rate=1";
1784             }
1785         }
1786     } elsif ($drive->{interface} eq 'usb') {
1787         die "implement me";
1788         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1789     } else {
1790         die "unsupported interface type";
1791     }
1792
1793     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1794
1795     if (my $serial = $drive->{serial}) {
1796         $serial = URI::Escape::uri_unescape($serial);
1797         $device .= ",serial=$serial";
1798     }
1799
1800
1801     return $device;
1802 }
1803
1804 sub get_initiator_name {
1805     my $initiator;
1806
1807     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return undef;
1808     while (defined(my $line = <$fh>)) {
1809         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1810         $initiator = $1;
1811         last;
1812     }
1813     $fh->close();
1814
1815     return $initiator;
1816 }
1817
1818 sub print_drive_full {
1819     my ($storecfg, $vmid, $drive) = @_;
1820
1821     my $path;
1822     my $volid = $drive->{file};
1823     my $format;
1824
1825     if (drive_is_cdrom($drive)) {
1826         $path = get_iso_path($storecfg, $vmid, $volid);
1827     } else {
1828         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1829         if ($storeid) {
1830             $path = PVE::Storage::path($storecfg, $volid);
1831             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
1832             $format = qemu_img_format($scfg, $volname);
1833         } else {
1834             $path = $volid;
1835             $format = "raw";
1836         }
1837    }
1838
1839     my $opts = '';
1840     my @qemu_drive_options = qw(heads secs cyls trans media format cache rerror werror aio discard);
1841     foreach my $o (@qemu_drive_options) {
1842         $opts .= ",$o=$drive->{$o}" if defined($drive->{$o});
1843     }
1844
1845     # snapshot only accepts on|off
1846     if (defined($drive->{snapshot})) {
1847         my $v = $drive->{snapshot} ? 'on' : 'off';
1848         $opts .= ",snapshot=$v";
1849     }
1850
1851     foreach my $type (['', '-total'], [_rd => '-read'], [_wr => '-write']) {
1852         my ($dir, $qmpname) = @$type;
1853         if (my $v = $drive->{"mbps$dir"}) {
1854             $opts .= ",throttling.bps$qmpname=".int($v*1024*1024);
1855         }
1856         if (my $v = $drive->{"mbps${dir}_max"}) {
1857             $opts .= ",throttling.bps$qmpname-max=".int($v*1024*1024);
1858         }
1859         if (my $v = $drive->{"bps${dir}_max_length"}) {
1860             $opts .= ",throttling.bps$qmpname-max-length=$v";
1861         }
1862         if (my $v = $drive->{"iops${dir}"}) {
1863             $opts .= ",throttling.iops$qmpname=$v";
1864         }
1865         if (my $v = $drive->{"iops${dir}_max"}) {
1866             $opts .= ",throttling.iops$qmpname-max=$v";
1867         }
1868         if (my $v = $drive->{"iops${dir}_max_length"}) {
1869             $opts .= ",throttling.iops$qmpname-max-length=$v";
1870         }
1871     }
1872
1873     $opts .= ",format=$format" if $format && !$drive->{format};
1874
1875     my $cache_direct = 0;
1876
1877     if (my $cache = $drive->{cache}) {
1878         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1879     } elsif (!drive_is_cdrom($drive)) {
1880         $opts .= ",cache=none";
1881         $cache_direct = 1;
1882     }
1883
1884     # aio native works only with O_DIRECT
1885     if (!$drive->{aio}) {
1886         if($cache_direct) {
1887             $opts .= ",aio=native";
1888         } else {
1889             $opts .= ",aio=threads";
1890         }
1891     }
1892
1893     if (!drive_is_cdrom($drive)) {
1894         my $detectzeroes;
1895         if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
1896             $detectzeroes = 'off';
1897         } elsif ($drive->{discard}) {
1898             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
1899         } else {
1900             # This used to be our default with discard not being specified:
1901             $detectzeroes = 'on';
1902         }
1903         $opts .= ",detect-zeroes=$detectzeroes" if $detectzeroes;
1904     }
1905
1906     my $pathinfo = $path ? "file=$path," : '';
1907
1908     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1909 }
1910
1911 sub print_netdevice_full {
1912     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files, $arch, $machine_type) = @_;
1913
1914     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
1915
1916     my $device = $net->{model};
1917     if ($net->{model} eq 'virtio') {
1918          $device = 'virtio-net-pci';
1919      };
1920
1921     my $pciaddr = print_pci_addr("$netid", $bridges, $arch, $machine_type);
1922     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1923     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
1924         #Consider we have N queues, the number of vectors needed is 2*N + 2 (plus one config interrupt and control vq)
1925         my $vectors = $net->{queues} * 2 + 2;
1926         $tmpstr .= ",vectors=$vectors,mq=on";
1927     }
1928     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
1929
1930     if ($use_old_bios_files) {
1931         my $romfile;
1932         if ($device eq 'virtio-net-pci') {
1933             $romfile = 'pxe-virtio.rom';
1934         } elsif ($device eq 'e1000') {
1935             $romfile = 'pxe-e1000.rom';
1936         } elsif ($device eq 'ne2k') {
1937             $romfile = 'pxe-ne2k_pci.rom';
1938         } elsif ($device eq 'pcnet') {
1939             $romfile = 'pxe-pcnet.rom';
1940         } elsif ($device eq 'rtl8139') {
1941             $romfile = 'pxe-rtl8139.rom';
1942         }
1943         $tmpstr .= ",romfile=$romfile" if $romfile;
1944     }
1945
1946     return $tmpstr;
1947 }
1948
1949 sub print_netdev_full {
1950     my ($vmid, $conf, $arch, $net, $netid, $hotplug) = @_;
1951
1952     my $i = '';
1953     if ($netid =~ m/^net(\d+)$/) {
1954         $i = int($1);
1955     }
1956
1957     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
1958
1959     my $ifname = "tap${vmid}i$i";
1960
1961     # kvm uses TUNSETIFF ioctl, and that limits ifname length
1962     die "interface name '$ifname' is too long (max 15 character)\n"
1963         if length($ifname) >= 16;
1964
1965     my $vhostparam = '';
1966     if (is_native($arch)) {
1967         $vhostparam = ',vhost=on' if $kernel_has_vhost_net && $net->{model} eq 'virtio';
1968     }
1969
1970     my $vmname = $conf->{name} || "vm$vmid";
1971
1972     my $netdev = "";
1973     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
1974
1975     if ($net->{bridge}) {
1976         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script,downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1977     } else {
1978         $netdev = "type=user,id=$netid,hostname=$vmname";
1979     }
1980
1981     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
1982
1983     return $netdev;
1984 }
1985
1986
1987 sub print_cpu_device {
1988     my ($conf, $id) = @_;
1989
1990     my $kvm = $conf->{kvm} // 1;
1991     my $cpu = $kvm ? "kvm64" : "qemu64";
1992     if (my $cputype = $conf->{cpu}) {
1993         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
1994             or die "Cannot parse cpu description: $cputype\n";
1995         $cpu = $cpuconf->{cputype};
1996     }
1997
1998     my $cores = $conf->{cores} || 1;
1999
2000     my $current_core = ($id - 1) % $cores;
2001     my $current_socket = int(($id - 1 - $current_core)/$cores);
2002
2003     return "$cpu-x86_64-cpu,id=cpu$id,socket-id=$current_socket,core-id=$current_core,thread-id=0";
2004 }
2005
2006 my $vga_map = {
2007     'cirrus' => 'cirrus-vga',
2008     'std' => 'VGA',
2009     'vmware' => 'vmware-svga',
2010     'virtio' => 'virtio-vga',
2011 };
2012
2013 sub print_vga_device {
2014     my ($conf, $vga, $arch, $machine, $id, $qxlnum, $bridges) = @_;
2015
2016     my $type = $vga_map->{$vga->{type}};
2017     if ($type eq 'virtio-vga' && $arch eq 'aarch64') {
2018         $type = 'virtio-gpu';
2019     }
2020     my $vgamem_mb = $vga->{memory};
2021     if ($qxlnum) {
2022         $type = $id ? 'qxl' : 'qxl-vga';
2023     }
2024     die "no devicetype for $vga->{type}\n" if !$type;
2025
2026     my $memory = "";
2027     if ($vgamem_mb) {
2028         if ($vga->{type} eq 'virtio') {
2029             my $bytes = PVE::Tools::convert_size($vgamem_mb, "mb" => "b");
2030             $memory = ",max_hostmem=$bytes";
2031         } elsif ($qxlnum) {
2032             # from https://www.spice-space.org/multiple-monitors.html
2033             $memory = ",vgamem_mb=$vga->{memory}";
2034             my $ram = $vgamem_mb * 4;
2035             my $vram = $vgamem_mb * 2;
2036             $memory .= ",ram_size_mb=$ram,vram_size_mb=$vram";
2037         } else {
2038             $memory = ",vgamem_mb=$vga->{memory}";
2039         }
2040     } elsif ($qxlnum && $id) {
2041         $memory = ",ram_size=67108864,vram_size=33554432";
2042     }
2043
2044     my $q35 = machine_type_is_q35($conf);
2045     my $vgaid = "vga" . ($id // '');
2046     my $pciaddr;
2047
2048     if ($q35 && $vgaid eq 'vga') {
2049         # the first display uses pcie.0 bus on q35 machines
2050         $pciaddr = print_pcie_addr($vgaid, $bridges, $arch, $machine);
2051     } else {
2052         $pciaddr = print_pci_addr($vgaid, $bridges, $arch, $machine);
2053     }
2054
2055     return "$type,id=${vgaid}${memory}${pciaddr}";
2056 }
2057
2058 sub drive_is_cloudinit {
2059     my ($drive) = @_;
2060     return $drive->{file} =~ m@[:/]vm-\d+-cloudinit(?:\.$QEMU_FORMAT_RE)?$@;
2061 }
2062
2063 sub drive_is_cdrom {
2064     my ($drive, $exclude_cloudinit) = @_;
2065
2066     return 0 if $exclude_cloudinit && drive_is_cloudinit($drive);
2067
2068     return $drive && $drive->{media} && ($drive->{media} eq 'cdrom');
2069
2070 }
2071
2072 sub parse_number_sets {
2073     my ($set) = @_;
2074     my $res = [];
2075     foreach my $part (split(/;/, $set)) {
2076         if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
2077             die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
2078             push @$res, [ $1, $2 ];
2079         } else {
2080             die "invalid range: $part\n";
2081         }
2082     }
2083     return $res;
2084 }
2085
2086 sub parse_numa {
2087     my ($data) = @_;
2088
2089     my $res = PVE::JSONSchema::parse_property_string($numa_fmt, $data);
2090     $res->{cpus} = parse_number_sets($res->{cpus}) if defined($res->{cpus});
2091     $res->{hostnodes} = parse_number_sets($res->{hostnodes}) if defined($res->{hostnodes});
2092     return $res;
2093 }
2094
2095 sub parse_hostpci {
2096     my ($value) = @_;
2097
2098     return undef if !$value;
2099
2100     my $res = PVE::JSONSchema::parse_property_string($hostpci_fmt, $value);
2101
2102     my @idlist = split(/;/, $res->{host});
2103     delete $res->{host};
2104     foreach my $id (@idlist) {
2105         if ($id =~ /^$PCIRE$/) {
2106             if (defined($2)) {
2107                 push @{$res->{pciid}}, { id => $1, function => $2 };
2108             } else {
2109                 my $pcidevices = lspci($1);
2110                 $res->{pciid} = $pcidevices->{$1};
2111             }
2112         } else {
2113             # should have been caught by parse_property_string already
2114             die "failed to parse PCI id: $id\n";
2115         }
2116     }
2117     return $res;
2118 }
2119
2120 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
2121 sub parse_net {
2122     my ($data) = @_;
2123
2124     my $res = eval { PVE::JSONSchema::parse_property_string($net_fmt, $data) };
2125     if ($@) {
2126         warn $@;
2127         return undef;
2128     }
2129     if (!defined($res->{macaddr})) {
2130         my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
2131         $res->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix});
2132     }
2133     return $res;
2134 }
2135
2136 # ipconfigX ip=cidr,gw=ip,ip6=cidr,gw6=ip
2137 sub parse_ipconfig {
2138     my ($data) = @_;
2139
2140     my $res = eval { PVE::JSONSchema::parse_property_string($ipconfig_fmt, $data) };
2141     if ($@) {
2142         warn $@;
2143         return undef;
2144     }
2145
2146     if ($res->{gw} && !$res->{ip}) {
2147         warn 'gateway specified without specifying an IP address';
2148         return undef;
2149     }
2150     if ($res->{gw6} && !$res->{ip6}) {
2151         warn 'IPv6 gateway specified without specifying an IPv6 address';
2152         return undef;
2153     }
2154     if ($res->{gw} && $res->{ip} eq 'dhcp') {
2155         warn 'gateway specified together with DHCP';
2156         return undef;
2157     }
2158     if ($res->{gw6} && $res->{ip6} !~ /^$IPV6RE/) {
2159         # gw6 + auto/dhcp
2160         warn "IPv6 gateway specified together with $res->{ip6} address";
2161         return undef;
2162     }
2163
2164     if (!$res->{ip} && !$res->{ip6}) {
2165         return { ip => 'dhcp', ip6 => 'dhcp' };
2166     }
2167
2168     return $res;
2169 }
2170
2171 sub print_net {
2172     my $net = shift;
2173
2174     return PVE::JSONSchema::print_property_string($net, $net_fmt);
2175 }
2176
2177 sub add_random_macs {
2178     my ($settings) = @_;
2179
2180     foreach my $opt (keys %$settings) {
2181         next if $opt !~ m/^net(\d+)$/;
2182         my $net = parse_net($settings->{$opt});
2183         next if !$net;
2184         $settings->{$opt} = print_net($net);
2185     }
2186 }
2187
2188 sub vm_is_volid_owner {
2189     my ($storecfg, $vmid, $volid) = @_;
2190
2191     if ($volid !~  m|^/|) {
2192         my ($path, $owner);
2193         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
2194         if ($owner && ($owner == $vmid)) {
2195             return 1;
2196         }
2197     }
2198
2199     return undef;
2200 }
2201
2202 sub split_flagged_list {
2203     my $text = shift || '';
2204     $text =~ s/[,;]/ /g;
2205     $text =~ s/^\s+//;
2206     return { map { /^(!?)(.*)$/ && ($2, $1) } ($text =~ /\S+/g) };
2207 }
2208
2209 sub join_flagged_list {
2210     my ($how, $lst) = @_;
2211     join $how, map { $lst->{$_} . $_ } keys %$lst;
2212 }
2213
2214 sub vmconfig_delete_pending_option {
2215     my ($conf, $key, $force) = @_;
2216
2217     delete $conf->{pending}->{$key};
2218     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
2219     $pending_delete_hash->{$key} = $force ? '!' : '';
2220     $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
2221 }
2222
2223 sub vmconfig_undelete_pending_option {
2224     my ($conf, $key) = @_;
2225
2226     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
2227     delete $pending_delete_hash->{$key};
2228
2229     if (%$pending_delete_hash) {
2230         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
2231     } else {
2232         delete $conf->{pending}->{delete};
2233     }
2234 }
2235
2236 sub vmconfig_register_unused_drive {
2237     my ($storecfg, $vmid, $conf, $drive) = @_;
2238
2239     if (drive_is_cloudinit($drive)) {
2240         eval { PVE::Storage::vdisk_free($storecfg, $drive->{file}) };
2241         warn $@ if $@;
2242     } elsif (!drive_is_cdrom($drive)) {
2243         my $volid = $drive->{file};
2244         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
2245             PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
2246         }
2247     }
2248 }
2249
2250 sub vmconfig_cleanup_pending {
2251     my ($conf) = @_;
2252
2253     # remove pending changes when nothing changed
2254     my $changes;
2255     foreach my $opt (keys %{$conf->{pending}}) {
2256         if (defined($conf->{$opt}) && ($conf->{pending}->{$opt} eq  $conf->{$opt})) {
2257             $changes = 1;
2258             delete $conf->{pending}->{$opt};
2259         }
2260     }
2261
2262     my $current_delete_hash = split_flagged_list($conf->{pending}->{delete});
2263     my $pending_delete_hash = {};
2264     while (my ($opt, $force) = each %$current_delete_hash) {
2265         if (defined($conf->{$opt})) {
2266             $pending_delete_hash->{$opt} = $force;
2267         } else {
2268             $changes = 1;
2269         }
2270     }
2271
2272     if (%$pending_delete_hash) {
2273         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
2274     } else {
2275         delete $conf->{pending}->{delete};
2276     }
2277
2278     return $changes;
2279 }
2280
2281 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str]
2282 my $smbios1_fmt = {
2283     uuid => {
2284         type => 'string',
2285         pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
2286         format_description => 'UUID',
2287         description => "Set SMBIOS1 UUID.",
2288         optional => 1,
2289     },
2290     version => {
2291         type => 'string',
2292         pattern => '\S+',
2293         format_description => 'string',
2294         description => "Set SMBIOS1 version.",
2295         optional => 1,
2296     },
2297     serial => {
2298         type => 'string',
2299         pattern => '\S+',
2300         format_description => 'string',
2301         description => "Set SMBIOS1 serial number.",
2302         optional => 1,
2303     },
2304     manufacturer => {
2305         type => 'string',
2306         pattern => '\S+',
2307         format_description => 'string',
2308         description => "Set SMBIOS1 manufacturer.",
2309         optional => 1,
2310     },
2311     product => {
2312         type => 'string',
2313         pattern => '\S+',
2314         format_description => 'string',
2315         description => "Set SMBIOS1 product ID.",
2316         optional => 1,
2317     },
2318     sku => {
2319         type => 'string',
2320         pattern => '\S+',
2321         format_description => 'string',
2322         description => "Set SMBIOS1 SKU string.",
2323         optional => 1,
2324     },
2325     family => {
2326         type => 'string',
2327         pattern => '\S+',
2328         format_description => 'string',
2329         description => "Set SMBIOS1 family string.",
2330         optional => 1,
2331     },
2332 };
2333
2334 sub parse_smbios1 {
2335     my ($data) = @_;
2336
2337     my $res = eval { PVE::JSONSchema::parse_property_string($smbios1_fmt, $data) };
2338     warn $@ if $@;
2339     return $res;
2340 }
2341
2342 sub print_smbios1 {
2343     my ($smbios1) = @_;
2344     return PVE::JSONSchema::print_property_string($smbios1, $smbios1_fmt);
2345 }
2346
2347 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_fmt);
2348
2349 PVE::JSONSchema::register_format('pve-qm-bootdisk', \&verify_bootdisk);
2350 sub verify_bootdisk {
2351     my ($value, $noerr) = @_;
2352
2353     return $value if is_valid_drivename($value);
2354
2355     return undef if $noerr;
2356
2357     die "invalid boot disk '$value'\n";
2358 }
2359
2360 sub parse_watchdog {
2361     my ($value) = @_;
2362
2363     return undef if !$value;
2364
2365     my $res = eval { PVE::JSONSchema::parse_property_string($watchdog_fmt, $value) };
2366     warn $@ if $@;
2367     return $res;
2368 }
2369
2370 sub parse_guest_agent {
2371     my ($value) = @_;
2372
2373     return {} if !defined($value->{agent});
2374
2375     my $res = eval { PVE::JSONSchema::parse_property_string($agent_fmt, $value->{agent}) };
2376     warn $@ if $@;
2377
2378     # if the agent is disabled ignore the other potentially set properties
2379     return {} if !$res->{enabled};
2380     return $res;
2381 }
2382
2383 sub parse_vga {
2384     my ($value) = @_;
2385
2386     return {} if !$value;
2387     my $res = eval { PVE::JSONSchema::parse_property_string($vga_fmt, $value) };
2388     warn $@ if $@;
2389     return $res;
2390 }
2391
2392 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
2393 sub verify_usb_device {
2394     my ($value, $noerr) = @_;
2395
2396     return $value if parse_usb_device($value);
2397
2398     return undef if $noerr;
2399
2400     die "unable to parse usb device\n";
2401 }
2402
2403 # add JSON properties for create and set function
2404 sub json_config_properties {
2405     my $prop = shift;
2406
2407     foreach my $opt (keys %$confdesc) {
2408         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate' || $opt eq 'runningmachine';
2409         $prop->{$opt} = $confdesc->{$opt};
2410     }
2411
2412     return $prop;
2413 }
2414
2415 # return copy of $confdesc_cloudinit to generate documentation
2416 sub cloudinit_config_properties {
2417
2418     return dclone($confdesc_cloudinit);
2419 }
2420
2421 sub check_type {
2422     my ($key, $value) = @_;
2423
2424     die "unknown setting '$key'\n" if !$confdesc->{$key};
2425
2426     my $type = $confdesc->{$key}->{type};
2427
2428     if (!defined($value)) {
2429         die "got undefined value\n";
2430     }
2431
2432     if ($value =~ m/[\n\r]/) {
2433         die "property contains a line feed\n";
2434     }
2435
2436     if ($type eq 'boolean') {
2437         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
2438         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
2439         die "type check ('boolean') failed - got '$value'\n";
2440     } elsif ($type eq 'integer') {
2441         return int($1) if $value =~ m/^(\d+)$/;
2442         die "type check ('integer') failed - got '$value'\n";
2443     } elsif ($type eq 'number') {
2444         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
2445         die "type check ('number') failed - got '$value'\n";
2446     } elsif ($type eq 'string') {
2447         if (my $fmt = $confdesc->{$key}->{format}) {
2448             PVE::JSONSchema::check_format($fmt, $value);
2449             return $value;
2450         }
2451         $value =~ s/^\"(.*)\"$/$1/;
2452         return $value;
2453     } else {
2454         die "internal error"
2455     }
2456 }
2457
2458 sub check_iommu_support{
2459     #fixme : need to check IOMMU support
2460     #http://www.linux-kvm.org/page/How_to_assign_devices_with_VT-d_in_KVM
2461
2462     my $iommu=1;
2463     return $iommu;
2464
2465 }
2466
2467 sub touch_config {
2468     my ($vmid) = @_;
2469
2470     my $conf = PVE::QemuConfig->config_file($vmid);
2471     utime undef, undef, $conf;
2472 }
2473
2474 sub destroy_vm {
2475     my ($storecfg, $vmid, $keep_empty_config, $skiplock) = @_;
2476
2477     my $conffile = PVE::QemuConfig->config_file($vmid);
2478
2479     my $conf = PVE::QemuConfig->load_config($vmid);
2480
2481     PVE::QemuConfig->check_lock($conf) if !$skiplock;
2482
2483     if ($conf->{template}) {
2484         # check if any base image is still used by a linked clone
2485         foreach_drive($conf, sub {
2486                 my ($ds, $drive) = @_;
2487
2488                 return if drive_is_cdrom($drive);
2489
2490                 my $volid = $drive->{file};
2491
2492                 return if !$volid || $volid =~ m|^/|;
2493
2494                 die "base volume '$volid' is still in use by linked cloned\n"
2495                     if PVE::Storage::volume_is_base_and_used($storecfg, $volid);
2496
2497         });
2498     }
2499
2500     # only remove disks owned by this VM
2501     foreach_drive($conf, sub {
2502         my ($ds, $drive) = @_;
2503
2504         return if drive_is_cdrom($drive, 1);
2505
2506         my $volid = $drive->{file};
2507
2508         return if !$volid || $volid =~ m|^/|;
2509
2510         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
2511         return if !$path || !$owner || ($owner != $vmid);
2512
2513         eval {
2514             PVE::Storage::vdisk_free($storecfg, $volid);
2515         };
2516         warn "Could not remove disk '$volid', check manually: $@" if $@;
2517
2518     });
2519
2520     if ($keep_empty_config) {
2521         PVE::Tools::file_set_contents($conffile, "memory: 128\n");
2522     } else {
2523         unlink $conffile;
2524     }
2525
2526     # also remove unused disk
2527     eval {
2528         my $dl = PVE::Storage::vdisk_list($storecfg, undef, $vmid);
2529
2530         eval {
2531             PVE::Storage::foreach_volid($dl, sub {
2532                 my ($volid, $sid, $volname, $d) = @_;
2533                 PVE::Storage::vdisk_free($storecfg, $volid);
2534             });
2535         };
2536         warn $@ if $@;
2537
2538     };
2539     warn $@ if $@;
2540 }
2541
2542 sub parse_vm_config {
2543     my ($filename, $raw) = @_;
2544
2545     return undef if !defined($raw);
2546
2547     my $res = {
2548         digest => Digest::SHA::sha1_hex($raw),
2549         snapshots => {},
2550         pending => {},
2551     };
2552
2553     $filename =~ m|/qemu-server/(\d+)\.conf$|
2554         || die "got strange filename '$filename'";
2555
2556     my $vmid = $1;
2557
2558     my $conf = $res;
2559     my $descr;
2560     my $section = '';
2561
2562     my @lines = split(/\n/, $raw);
2563     foreach my $line (@lines) {
2564         next if $line =~ m/^\s*$/;
2565
2566         if ($line =~ m/^\[PENDING\]\s*$/i) {
2567             $section = 'pending';
2568             if (defined($descr)) {
2569                 $descr =~ s/\s+$//;
2570                 $conf->{description} = $descr;
2571             }
2572             $descr = undef;
2573             $conf = $res->{$section} = {};
2574             next;
2575
2576         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2577             $section = $1;
2578             if (defined($descr)) {
2579                 $descr =~ s/\s+$//;
2580                 $conf->{description} = $descr;
2581             }
2582             $descr = undef;
2583             $conf = $res->{snapshots}->{$section} = {};
2584             next;
2585         }
2586
2587         if ($line =~ m/^\#(.*)\s*$/) {
2588             $descr = '' if !defined($descr);
2589             $descr .= PVE::Tools::decode_text($1) . "\n";
2590             next;
2591         }
2592
2593         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2594             $descr = '' if !defined($descr);
2595             $descr .= PVE::Tools::decode_text($2);
2596         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2597             $conf->{snapstate} = $1;
2598         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2599             my $key = $1;
2600             my $value = $2;
2601             $conf->{$key} = $value;
2602         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2603             my $value = $1;
2604             if ($section eq 'pending') {
2605                 $conf->{delete} = $value; # we parse this later
2606             } else {
2607                 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
2608             }
2609         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(.+?)\s*$/) {
2610             my $key = $1;
2611             my $value = $2;
2612             eval { $value = check_type($key, $value); };
2613             if ($@) {
2614                 warn "vm $vmid - unable to parse value of '$key' - $@";
2615             } else {
2616                 $key = 'ide2' if $key eq 'cdrom';
2617                 my $fmt = $confdesc->{$key}->{format};
2618                 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2619                     my $v = parse_drive($key, $value);
2620                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2621                         $v->{file} = $volid;
2622                         $value = print_drive($vmid, $v);
2623                     } else {
2624                         warn "vm $vmid - unable to parse value of '$key'\n";
2625                         next;
2626                     }
2627                 }
2628
2629                 $conf->{$key} = $value;
2630             }
2631         }
2632     }
2633
2634     if (defined($descr)) {
2635         $descr =~ s/\s+$//;
2636         $conf->{description} = $descr;
2637     }
2638     delete $res->{snapstate}; # just to be sure
2639
2640     return $res;
2641 }
2642
2643 sub write_vm_config {
2644     my ($filename, $conf) = @_;
2645
2646     delete $conf->{snapstate}; # just to be sure
2647
2648     if ($conf->{cdrom}) {
2649         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2650         $conf->{ide2} = $conf->{cdrom};
2651         delete $conf->{cdrom};
2652     }
2653
2654     # we do not use 'smp' any longer
2655     if ($conf->{sockets}) {
2656         delete $conf->{smp};
2657     } elsif ($conf->{smp}) {
2658         $conf->{sockets} = $conf->{smp};
2659         delete $conf->{cores};
2660         delete $conf->{smp};
2661     }
2662
2663     my $used_volids = {};
2664
2665     my $cleanup_config = sub {
2666         my ($cref, $pending, $snapname) = @_;
2667
2668         foreach my $key (keys %$cref) {
2669             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2670                 $key eq 'snapstate' || $key eq 'pending';
2671             my $value = $cref->{$key};
2672             if ($key eq 'delete') {
2673                 die "propertry 'delete' is only allowed in [PENDING]\n"
2674                     if !$pending;
2675                 # fixme: check syntax?
2676                 next;
2677             }
2678             eval { $value = check_type($key, $value); };
2679             die "unable to parse value of '$key' - $@" if $@;
2680
2681             $cref->{$key} = $value;
2682
2683             if (!$snapname && is_valid_drivename($key)) {
2684                 my $drive = parse_drive($key, $value);
2685                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2686             }
2687         }
2688     };
2689
2690     &$cleanup_config($conf);
2691
2692     &$cleanup_config($conf->{pending}, 1);
2693
2694     foreach my $snapname (keys %{$conf->{snapshots}}) {
2695         die "internal error" if $snapname eq 'pending';
2696         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2697     }
2698
2699     # remove 'unusedX' settings if we re-add a volume
2700     foreach my $key (keys %$conf) {
2701         my $value = $conf->{$key};
2702         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2703             delete $conf->{$key};
2704         }
2705     }
2706
2707     my $generate_raw_config = sub {
2708         my ($conf, $pending) = @_;
2709
2710         my $raw = '';
2711
2712         # add description as comment to top of file
2713         if (defined(my $descr = $conf->{description})) {
2714             if ($descr) {
2715                 foreach my $cl (split(/\n/, $descr)) {
2716                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2717                 }
2718             } else {
2719                 $raw .= "#\n" if $pending;
2720             }
2721         }
2722
2723         foreach my $key (sort keys %$conf) {
2724             next if $key eq 'digest' || $key eq 'description' || $key eq 'pending' || $key eq 'snapshots';
2725             $raw .= "$key: $conf->{$key}\n";
2726         }
2727         return $raw;
2728     };
2729
2730     my $raw = &$generate_raw_config($conf);
2731
2732     if (scalar(keys %{$conf->{pending}})){
2733         $raw .= "\n[PENDING]\n";
2734         $raw .= &$generate_raw_config($conf->{pending}, 1);
2735     }
2736
2737     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2738         $raw .= "\n[$snapname]\n";
2739         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2740     }
2741
2742     return $raw;
2743 }
2744
2745 sub load_defaults {
2746
2747     my $res = {};
2748
2749     # we use static defaults from our JSON schema configuration
2750     foreach my $key (keys %$confdesc) {
2751         if (defined(my $default = $confdesc->{$key}->{default})) {
2752             $res->{$key} = $default;
2753         }
2754     }
2755
2756     return $res;
2757 }
2758
2759 sub config_list {
2760     my $vmlist = PVE::Cluster::get_vmlist();
2761     my $res = {};
2762     return $res if !$vmlist || !$vmlist->{ids};
2763     my $ids = $vmlist->{ids};
2764
2765     foreach my $vmid (keys %$ids) {
2766         my $d = $ids->{$vmid};
2767         next if !$d->{node} || $d->{node} ne $nodename;
2768         next if !$d->{type} || $d->{type} ne 'qemu';
2769         $res->{$vmid}->{exists} = 1;
2770     }
2771     return $res;
2772 }
2773
2774 # test if VM uses local resources (to prevent migration)
2775 sub check_local_resources {
2776     my ($conf, $noerr) = @_;
2777
2778     my $loc_res = 0;
2779
2780     $loc_res = 1 if $conf->{hostusb}; # old syntax
2781     $loc_res = 1 if $conf->{hostpci}; # old syntax
2782
2783     foreach my $k (keys %$conf) {
2784         next if $k =~ m/^usb/ && ($conf->{$k} eq 'spice');
2785         # sockets are safe: they will recreated be on the target side post-migrate
2786         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2787         $loc_res = 1 if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2788     }
2789
2790     die "VM uses local resources\n" if $loc_res && !$noerr;
2791
2792     return $loc_res;
2793 }
2794
2795 # check if used storages are available on all nodes (use by migrate)
2796 sub check_storage_availability {
2797     my ($storecfg, $conf, $node) = @_;
2798
2799     foreach_drive($conf, sub {
2800         my ($ds, $drive) = @_;
2801
2802         my $volid = $drive->{file};
2803         return if !$volid;
2804
2805         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2806         return if !$sid;
2807
2808         # check if storage is available on both nodes
2809         my $scfg = PVE::Storage::storage_check_node($storecfg, $sid);
2810         PVE::Storage::storage_check_node($storecfg, $sid, $node);
2811    });
2812 }
2813
2814 # list nodes where all VM images are available (used by has_feature API)
2815 sub shared_nodes {
2816     my ($conf, $storecfg) = @_;
2817
2818     my $nodelist = PVE::Cluster::get_nodelist();
2819     my $nodehash = { map { $_ => 1 } @$nodelist };
2820     my $nodename = PVE::INotify::nodename();
2821
2822     foreach_drive($conf, sub {
2823         my ($ds, $drive) = @_;
2824
2825         my $volid = $drive->{file};
2826         return if !$volid;
2827
2828         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2829         if ($storeid) {
2830             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2831             if ($scfg->{disable}) {
2832                 $nodehash = {};
2833             } elsif (my $avail = $scfg->{nodes}) {
2834                 foreach my $node (keys %$nodehash) {
2835                     delete $nodehash->{$node} if !$avail->{$node};
2836                 }
2837             } elsif (!$scfg->{shared}) {
2838                 foreach my $node (keys %$nodehash) {
2839                     delete $nodehash->{$node} if $node ne $nodename
2840                 }
2841             }
2842         }
2843     });
2844
2845     return $nodehash
2846 }
2847
2848 sub check_cmdline {
2849     my ($pidfile, $pid) = @_;
2850
2851     my $fh = IO::File->new("/proc/$pid/cmdline", "r");
2852     if (defined($fh)) {
2853         my $line = <$fh>;
2854         $fh->close;
2855         return undef if !$line;
2856         my @param = split(/\0/, $line);
2857
2858         my $cmd = $param[0];
2859         return if !$cmd || ($cmd !~ m|kvm$| && $cmd !~ m@(?:^|/)qemu-system-[^/]+$@);
2860
2861         for (my $i = 0; $i < scalar (@param); $i++) {
2862             my $p = $param[$i];
2863             next if !$p;
2864             if (($p eq '-pidfile') || ($p eq '--pidfile')) {
2865                 my $p = $param[$i+1];
2866                 return 1 if $p && ($p eq $pidfile);
2867                 return undef;
2868             }
2869         }
2870     }
2871     return undef;
2872 }
2873
2874 sub check_running {
2875     my ($vmid, $nocheck, $node) = @_;
2876
2877     my $filename = PVE::QemuConfig->config_file($vmid, $node);
2878
2879     die "unable to find configuration file for VM $vmid - no such machine\n"
2880         if !$nocheck && ! -f $filename;
2881
2882     my $pidfile = pidfile_name($vmid);
2883
2884     if (my $fd = IO::File->new("<$pidfile")) {
2885         my $st = stat($fd);
2886         my $line = <$fd>;
2887         close($fd);
2888
2889         my $mtime = $st->mtime;
2890         if ($mtime > time()) {
2891             warn "file '$filename' modified in future\n";
2892         }
2893
2894         if ($line =~ m/^(\d+)$/) {
2895             my $pid = $1;
2896             if (check_cmdline($pidfile, $pid)) {
2897                 if (my $pinfo = PVE::ProcFSTools::check_process_running($pid)) {
2898                     return $pid;
2899                 }
2900             }
2901         }
2902     }
2903
2904     return undef;
2905 }
2906
2907 sub vzlist {
2908
2909     my $vzlist = config_list();
2910
2911     my $fd = IO::Dir->new($var_run_tmpdir) || return $vzlist;
2912
2913     while (defined(my $de = $fd->read)) {
2914         next if $de !~ m/^(\d+)\.pid$/;
2915         my $vmid = $1;
2916         next if !defined($vzlist->{$vmid});
2917         if (my $pid = check_running($vmid)) {
2918             $vzlist->{$vmid}->{pid} = $pid;
2919         }
2920     }
2921
2922     return $vzlist;
2923 }
2924
2925 sub disksize {
2926     my ($storecfg, $conf) = @_;
2927
2928     my $bootdisk = $conf->{bootdisk};
2929     return undef if !$bootdisk;
2930     return undef if !is_valid_drivename($bootdisk);
2931
2932     return undef if !$conf->{$bootdisk};
2933
2934     my $drive = parse_drive($bootdisk, $conf->{$bootdisk});
2935     return undef if !defined($drive);
2936
2937     return undef if drive_is_cdrom($drive);
2938
2939     my $volid = $drive->{file};
2940     return undef if !$volid;
2941
2942     return $drive->{size};
2943 }
2944
2945 our $vmstatus_return_properties = {
2946     vmid => get_standard_option('pve-vmid'),
2947     status => {
2948         description => "Qemu process status.",
2949         type => 'string',
2950         enum => ['stopped', 'running'],
2951     },
2952     maxmem => {
2953         description => "Maximum memory in bytes.",
2954         type => 'integer',
2955         optional => 1,
2956         renderer => 'bytes',
2957     },
2958     maxdisk => {
2959         description => "Root disk size in bytes.",
2960         type => 'integer',
2961         optional => 1,
2962         renderer => 'bytes',
2963     },
2964     name => {
2965         description => "VM name.",
2966         type => 'string',
2967         optional => 1,
2968     },
2969     qmpstatus => {
2970         description => "Qemu QMP agent status.",
2971         type => 'string',
2972         optional => 1,
2973     },
2974     pid => {
2975         description => "PID of running qemu process.",
2976         type => 'integer',
2977         optional => 1,
2978     },
2979     uptime => {
2980         description => "Uptime.",
2981         type => 'integer',
2982         optional => 1,
2983         renderer => 'duration',
2984     },
2985     cpus => {
2986         description => "Maximum usable CPUs.",
2987         type => 'number',
2988         optional => 1,
2989     },
2990 };
2991
2992 my $last_proc_pid_stat;
2993
2994 # get VM status information
2995 # This must be fast and should not block ($full == false)
2996 # We only query KVM using QMP if $full == true (this can be slow)
2997 sub vmstatus {
2998     my ($opt_vmid, $full) = @_;
2999
3000     my $res = {};
3001
3002     my $storecfg = PVE::Storage::config();
3003
3004     my $list = vzlist();
3005     my $defaults = load_defaults();
3006
3007     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
3008
3009     my $cpucount = $cpuinfo->{cpus} || 1;
3010
3011     foreach my $vmid (keys %$list) {
3012         next if $opt_vmid && ($vmid ne $opt_vmid);
3013
3014         my $cfspath = PVE::QemuConfig->cfs_config_path($vmid);
3015         my $conf = PVE::Cluster::cfs_read_file($cfspath) || {};
3016
3017         my $d = { vmid => $vmid };
3018         $d->{pid} = $list->{$vmid}->{pid};
3019
3020         # fixme: better status?
3021         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
3022
3023         my $size = disksize($storecfg, $conf);
3024         if (defined($size)) {
3025             $d->{disk} = 0; # no info available
3026             $d->{maxdisk} = $size;
3027         } else {
3028             $d->{disk} = 0;
3029             $d->{maxdisk} = 0;
3030         }
3031
3032         $d->{cpus} = ($conf->{sockets} || $defaults->{sockets})
3033             * ($conf->{cores} || $defaults->{cores});
3034         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
3035         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
3036
3037         $d->{name} = $conf->{name} || "VM $vmid";
3038         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024)
3039             : $defaults->{memory}*(1024*1024);
3040
3041         if ($conf->{balloon}) {
3042             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
3043             $d->{shares} = defined($conf->{shares}) ? $conf->{shares}
3044                 : $defaults->{shares};
3045         }
3046
3047         $d->{uptime} = 0;
3048         $d->{cpu} = 0;
3049         $d->{mem} = 0;
3050
3051         $d->{netout} = 0;
3052         $d->{netin} = 0;
3053
3054         $d->{diskread} = 0;
3055         $d->{diskwrite} = 0;
3056
3057         $d->{template} = PVE::QemuConfig->is_template($conf);
3058
3059         $d->{serial} = 1 if conf_has_serial($conf);
3060
3061         $res->{$vmid} = $d;
3062     }
3063
3064     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
3065     foreach my $dev (keys %$netdev) {
3066         next if $dev !~ m/^tap([1-9]\d*)i/;
3067         my $vmid = $1;
3068         my $d = $res->{$vmid};
3069         next if !$d;
3070
3071         $d->{netout} += $netdev->{$dev}->{receive};
3072         $d->{netin} += $netdev->{$dev}->{transmit};
3073
3074         if ($full) {
3075             $d->{nics}->{$dev}->{netout} = $netdev->{$dev}->{receive};
3076             $d->{nics}->{$dev}->{netin} = $netdev->{$dev}->{transmit};
3077         }
3078
3079     }
3080
3081     my $ctime = gettimeofday;
3082
3083     foreach my $vmid (keys %$list) {
3084
3085         my $d = $res->{$vmid};
3086         my $pid = $d->{pid};
3087         next if !$pid;
3088
3089         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
3090         next if !$pstat; # not running
3091
3092         my $used = $pstat->{utime} + $pstat->{stime};
3093
3094         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
3095
3096         if ($pstat->{vsize}) {
3097             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
3098         }
3099
3100         my $old = $last_proc_pid_stat->{$pid};
3101         if (!$old) {
3102             $last_proc_pid_stat->{$pid} = {
3103                 time => $ctime,
3104                 used => $used,
3105                 cpu => 0,
3106             };
3107             next;
3108         }
3109
3110         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
3111
3112         if ($dtime > 1000) {
3113             my $dutime = $used -  $old->{used};
3114
3115             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
3116             $last_proc_pid_stat->{$pid} = {
3117                 time => $ctime,
3118                 used => $used,
3119                 cpu => $d->{cpu},
3120             };
3121         } else {
3122             $d->{cpu} = $old->{cpu};
3123         }
3124     }
3125
3126     return $res if !$full;
3127
3128     my $qmpclient = PVE::QMPClient->new();
3129
3130     my $ballooncb = sub {
3131         my ($vmid, $resp) = @_;
3132
3133         my $info = $resp->{'return'};
3134         return if !$info->{max_mem};
3135
3136         my $d = $res->{$vmid};
3137
3138         # use memory assigned to VM
3139         $d->{maxmem} = $info->{max_mem};
3140         $d->{balloon} = $info->{actual};
3141
3142         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
3143             $d->{mem} = $info->{total_mem} - $info->{free_mem};
3144             $d->{freemem} = $info->{free_mem};
3145         }
3146
3147         $d->{ballooninfo} = $info;
3148     };
3149
3150     my $blockstatscb = sub {
3151         my ($vmid, $resp) = @_;
3152         my $data = $resp->{'return'} || [];
3153         my $totalrdbytes = 0;
3154         my $totalwrbytes = 0;
3155
3156         for my $blockstat (@$data) {
3157             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
3158             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
3159
3160             $blockstat->{device} =~ s/drive-//;
3161             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
3162         }
3163         $res->{$vmid}->{diskread} = $totalrdbytes;
3164         $res->{$vmid}->{diskwrite} = $totalwrbytes;
3165     };
3166
3167     my $statuscb = sub {
3168         my ($vmid, $resp) = @_;
3169
3170         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
3171         # this fails if ballon driver is not loaded, so this must be
3172         # the last commnand (following command are aborted if this fails).
3173         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
3174
3175         my $status = 'unknown';
3176         if (!defined($status = $resp->{'return'}->{status})) {
3177             warn "unable to get VM status\n";
3178             return;
3179         }
3180
3181         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
3182     };
3183
3184     foreach my $vmid (keys %$list) {
3185         next if $opt_vmid && ($vmid ne $opt_vmid);
3186         next if !$res->{$vmid}->{pid}; # not running
3187         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
3188     }
3189
3190     $qmpclient->queue_execute(undef, 2);
3191
3192     foreach my $vmid (keys %$list) {
3193         next if $opt_vmid && ($vmid ne $opt_vmid);
3194         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
3195     }
3196
3197     return $res;
3198 }
3199
3200 sub foreach_drive {
3201     my ($conf, $func, @param) = @_;
3202
3203     foreach my $ds (valid_drive_names()) {
3204         next if !defined($conf->{$ds});
3205
3206         my $drive = parse_drive($ds, $conf->{$ds});
3207         next if !$drive;
3208
3209         &$func($ds, $drive, @param);
3210     }
3211 }
3212
3213 sub foreach_volid {
3214     my ($conf, $func, @param) = @_;
3215
3216     my $volhash = {};
3217
3218     my $test_volid = sub {
3219         my ($volid, $is_cdrom, $replicate, $shared, $snapname) = @_;
3220
3221         return if !$volid;
3222
3223         $volhash->{$volid}->{cdrom} //= 1;
3224         $volhash->{$volid}->{cdrom} = 0 if !$is_cdrom;
3225
3226         $volhash->{$volid}->{replicate} //= 0;
3227         $volhash->{$volid}->{replicate} = 1 if $replicate;
3228
3229         $volhash->{$volid}->{shared} //= 0;
3230         $volhash->{$volid}->{shared} = 1 if $shared;
3231
3232         $volhash->{$volid}->{referenced_in_config} //= 0;
3233         $volhash->{$volid}->{referenced_in_config} = 1 if !defined($snapname);
3234
3235         $volhash->{$volid}->{referenced_in_snapshot}->{$snapname} = 1
3236             if defined($snapname);
3237     };
3238
3239     foreach_drive($conf, sub {
3240         my ($ds, $drive) = @_;
3241         $test_volid->($drive->{file}, drive_is_cdrom($drive), $drive->{replicate} // 1, $drive->{shared}, undef);
3242     });
3243
3244     foreach my $snapname (keys %{$conf->{snapshots}}) {
3245         my $snap = $conf->{snapshots}->{$snapname};
3246         $test_volid->($snap->{vmstate}, 0, 1, $snapname);
3247         foreach_drive($snap, sub {
3248             my ($ds, $drive) = @_;
3249             $test_volid->($drive->{file}, drive_is_cdrom($drive), $drive->{replicate} // 1, $drive->{shared}, $snapname);
3250         });
3251     }
3252
3253     foreach my $volid (keys %$volhash) {
3254         &$func($volid, $volhash->{$volid}, @param);
3255     }
3256 }
3257
3258 sub conf_has_serial {
3259     my ($conf) = @_;
3260
3261     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3262         if ($conf->{"serial$i"}) {
3263             return 1;
3264         }
3265     }
3266
3267     return 0;
3268 }
3269
3270 sub vga_conf_has_spice {
3271     my ($vga) = @_;
3272
3273     my $vgaconf = parse_vga($vga);
3274     my $vgatype = $vgaconf->{type};
3275     return 0 if !$vgatype || $vgatype !~ m/^qxl([234])?$/;
3276
3277     return $1 || 1;
3278 }
3279
3280 my $host_arch; # FIXME: fix PVE::Tools::get_host_arch
3281 sub get_host_arch() {
3282     $host_arch = (POSIX::uname())[4] if !$host_arch;
3283     return $host_arch;
3284 }
3285
3286 sub is_native($) {
3287     my ($arch) = @_;
3288     return get_host_arch() eq $arch;
3289 }
3290
3291 my $default_machines = {
3292     x86_64 => 'pc',
3293     aarch64 => 'virt',
3294 };
3295
3296 sub get_basic_machine_info {
3297     my ($conf, $forcemachine) = @_;
3298
3299     my $arch = $conf->{arch} // get_host_arch();
3300     my $machine = $forcemachine || $conf->{machine} || $default_machines->{$arch};
3301     return ($arch, $machine);
3302 }
3303
3304 sub get_ovmf_files($) {
3305     my ($arch) = @_;
3306
3307     my $ovmf = $OVMF->{$arch}
3308         or die "no OVMF images known for architecture '$arch'\n";
3309
3310     return @$ovmf;
3311 }
3312
3313 my $Arch2Qemu = {
3314     aarch64 => '/usr/bin/qemu-system-aarch64',
3315     x86_64 => '/usr/bin/qemu-system-x86_64',
3316 };
3317 sub get_command_for_arch($) {
3318     my ($arch) = @_;
3319     return '/usr/bin/kvm' if is_native($arch);
3320
3321     my $cmd = $Arch2Qemu->{$arch}
3322         or die "don't know how to emulate architecture '$arch'\n";
3323     return $cmd;
3324 }
3325
3326 sub get_cpu_options {
3327     my ($conf, $arch, $kvm, $machine_type, $kvm_off, $kvmver, $winversion, $gpu_passthrough) = @_;
3328
3329     my $cpuFlags = [];
3330     my $ostype = $conf->{ostype};
3331
3332     my $cpu = $kvm ? "kvm64" : "qemu64";
3333     if ($arch eq 'aarch64') {
3334         $cpu = 'cortex-a57';
3335     }
3336     if (my $cputype = $conf->{cpu}) {
3337         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
3338             or die "Cannot parse cpu description: $cputype\n";
3339         $cpu = $cpuconf->{cputype};
3340         $kvm_off = 1 if $cpuconf->{hidden};
3341
3342         if (defined(my $flags = $cpuconf->{flags})) {
3343             push @$cpuFlags, split(";", $flags);
3344         }
3345     }
3346
3347     push @$cpuFlags , '+lahf_lm' if $cpu eq 'kvm64' && $arch eq 'x86_64';
3348
3349     push @$cpuFlags , '-x2apic'
3350         if $conf->{ostype} && $conf->{ostype} eq 'solaris';
3351
3352     push @$cpuFlags, '+sep' if $cpu eq 'kvm64' || $cpu eq 'kvm32';
3353
3354     push @$cpuFlags, '-rdtscp' if $cpu =~ m/^Opteron/;
3355
3356     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3) && $arch eq 'x86_64') {
3357
3358         push @$cpuFlags , '+kvm_pv_unhalt' if $kvm;
3359         push @$cpuFlags , '+kvm_pv_eoi' if $kvm;
3360     }
3361
3362     add_hyperv_enlightenments($cpuFlags, $winversion, $machine_type, $kvmver, $conf->{bios}, $gpu_passthrough) if $kvm;
3363
3364     push @$cpuFlags, 'enforce' if $cpu ne 'host' && $kvm && $arch eq 'x86_64';
3365
3366     push @$cpuFlags, 'kvm=off' if $kvm_off;
3367
3368     if (my $cpu_vendor = $cpu_vendor_list->{$cpu}) {
3369         push @$cpuFlags, "vendor=${cpu_vendor}"
3370             if $cpu_vendor ne 'default';
3371     } elsif ($arch ne 'aarch64') {
3372         die "internal error"; # should not happen
3373     }
3374
3375     $cpu .= "," . join(',', @$cpuFlags) if scalar(@$cpuFlags);
3376
3377     return ('-cpu', $cpu);
3378 }
3379
3380 sub config_to_command {
3381     my ($storecfg, $vmid, $conf, $defaults, $forcemachine) = @_;
3382
3383     my $cmd = [];
3384     my $globalFlags = [];
3385     my $machineFlags = [];
3386     my $rtcFlags = [];
3387     my $devices = [];
3388     my $pciaddr = '';
3389     my $bridges = {};
3390     my $kvmver = kvm_user_version();
3391     my $vernum = 0; # unknown
3392     my $ostype = $conf->{ostype};
3393     my $winversion = windows_version($ostype);
3394     my $kvm = $conf->{kvm};
3395
3396     my ($arch, $machine_type) = get_basic_machine_info($conf, $forcemachine);
3397     $kvm //= 1 if is_native($arch);
3398
3399     if ($kvm) {
3400         die "KVM virtualisation configured, but not available. Either disable in VM configuration or enable in BIOS.\n"
3401             if !defined kvm_version();
3402     }
3403
3404     if ($kvmver =~ m/^(\d+)\.(\d+)$/) {
3405         $vernum = $1*1000000+$2*1000;
3406     } elsif ($kvmver =~ m/^(\d+)\.(\d+)\.(\d+)$/) {
3407         $vernum = $1*1000000+$2*1000+$3;
3408     }
3409
3410     die "detected old qemu-kvm binary ($kvmver)\n" if $vernum < 15000;
3411
3412     my $have_ovz = -f '/proc/vz/vestat';
3413
3414     my $q35 = machine_type_is_q35($conf);
3415     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
3416     my $use_old_bios_files = undef;
3417     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3418
3419     my $cpuunits = defined($conf->{cpuunits}) ?
3420             $conf->{cpuunits} : $defaults->{cpuunits};
3421
3422     push @$cmd, get_command_for_arch($arch);
3423
3424     push @$cmd, '-id', $vmid;
3425
3426     my $vmname = $conf->{name} || "vm$vmid";
3427
3428     push @$cmd, '-name', $vmname;
3429
3430     my $use_virtio = 0;
3431
3432     my $qmpsocket = qmp_socket($vmid);
3433     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server,nowait";
3434     push @$cmd, '-mon', "chardev=qmp,mode=control";
3435
3436     if (qemu_machine_feature_enabled($machine_type, $kvmver, 2, 12)) {
3437         my $eventsocket = qmp_socket($vmid, 0, 'event');
3438         push @$cmd, '-chardev', "socket,id=qmp-event,path=$eventsocket,server,nowait";
3439         push @$cmd, '-mon', "chardev=qmp-event,mode=control";
3440     }
3441
3442     push @$cmd, '-pidfile' , pidfile_name($vmid);
3443
3444     push @$cmd, '-daemonize';
3445
3446     if ($conf->{smbios1}) {
3447         push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
3448     }
3449
3450     if ($conf->{vmgenid}) {
3451         push @$devices, '-device', 'vmgenid,guid='.$conf->{vmgenid};
3452     }
3453
3454     my ($ovmf_code, $ovmf_vars) = get_ovmf_files($arch);
3455     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3456         die "uefi base image not found\n" if ! -f $ovmf_code;
3457
3458         my $path;
3459         my $format;
3460         if (my $efidisk = $conf->{efidisk0}) {
3461             my $d = PVE::JSONSchema::parse_property_string($efidisk_fmt, $efidisk);
3462             my ($storeid, $volname) = PVE::Storage::parse_volume_id($d->{file}, 1);
3463             $format = $d->{format};
3464             if ($storeid) {
3465                 $path = PVE::Storage::path($storecfg, $d->{file});
3466                 if (!defined($format)) {
3467                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
3468                     $format = qemu_img_format($scfg, $volname);
3469                 }
3470             } else {
3471                 $path = $d->{file};
3472                 die "efidisk format must be specified\n"
3473                     if !defined($format);
3474             }
3475         } else {
3476             warn "no efidisk configured! Using temporary efivars disk.\n";
3477             $path = "/tmp/$vmid-ovmf.fd";
3478             PVE::Tools::file_copy($ovmf_vars, $path, -s $ovmf_vars);
3479             $format = 'raw';
3480         }
3481
3482         push @$cmd, '-drive', "if=pflash,unit=0,format=raw,readonly,file=$ovmf_code";
3483         push @$cmd, '-drive', "if=pflash,unit=1,format=$format,id=drive-efidisk0,file=$path";
3484     }
3485
3486
3487     # add usb controllers
3488     my @usbcontrollers = PVE::QemuServer::USB::get_usb_controllers($conf, $bridges, $arch, $machine_type, $usbdesc->{format}, $MAX_USB_DEVICES);
3489     push @$devices, @usbcontrollers if @usbcontrollers;
3490     my $vga = parse_vga($conf->{vga});
3491
3492     my $qxlnum = vga_conf_has_spice($conf->{vga});
3493     $vga->{type} = 'qxl' if $qxlnum;
3494
3495     if (!$vga->{type}) {
3496         if ($arch eq 'aarch64') {
3497             $vga->{type} = 'virtio';
3498         } elsif (qemu_machine_feature_enabled($machine_type, $kvmver, 2, 9)) {
3499             $vga->{type} = (!$winversion || $winversion >= 6) ? 'std' : 'cirrus';
3500         } else {
3501             $vga->{type} = ($winversion >= 6) ? 'std' : 'cirrus';
3502         }
3503     }
3504
3505     # enable absolute mouse coordinates (needed by vnc)
3506     my $tablet;
3507     if (defined($conf->{tablet})) {
3508         $tablet = $conf->{tablet};
3509     } else {
3510         $tablet = $defaults->{tablet};
3511         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
3512         $tablet = 0 if $vga->{type} =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
3513     }
3514
3515     if ($tablet) {
3516         push @$devices, '-device', print_tabletdevice_full($conf, $arch) if $tablet;
3517         my $kbd = print_keyboarddevice_full($conf, $arch);
3518         push @$devices, '-device', $kbd if defined($kbd);
3519     }
3520
3521     my $kvm_off = 0;
3522     my $gpu_passthrough;
3523
3524     # host pci devices
3525     for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
3526         my $d = parse_hostpci($conf->{"hostpci$i"});
3527         next if !$d;
3528
3529         my $pcie = $d->{pcie};
3530         if($pcie){
3531             die "q35 machine model is not enabled" if !$q35;
3532             $pciaddr = print_pcie_addr("hostpci$i");
3533         }else{
3534             $pciaddr = print_pci_addr("hostpci$i", $bridges, $arch, $machine_type);
3535         }
3536
3537         my $rombar = defined($d->{rombar}) && !$d->{rombar} ? ',rombar=0' : '';
3538         my $romfile = $d->{romfile};
3539
3540         my $xvga = '';
3541         if ($d->{'x-vga'}) {
3542             $xvga = ',x-vga=on';
3543             $kvm_off = 1;
3544             $vga->{type} = 'none';
3545             $gpu_passthrough = 1;
3546
3547             if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3548                 $xvga = "";
3549             }
3550         }
3551         my $pcidevices = $d->{pciid};
3552         my $multifunction = 1 if @$pcidevices > 1;
3553
3554         my $j=0;
3555         foreach my $pcidevice (@$pcidevices) {
3556
3557             my $id = "hostpci$i";
3558             $id .= ".$j" if $multifunction;
3559             my $addr = $pciaddr;
3560             $addr .= ".$j" if $multifunction;
3561             my $devicestr = "vfio-pci,host=$pcidevice->{id}.$pcidevice->{function},id=$id$addr";
3562
3563             if($j == 0){
3564                 $devicestr .= "$rombar$xvga";
3565                 $devicestr .= ",multifunction=on" if $multifunction;
3566                 $devicestr .= ",romfile=/usr/share/kvm/$romfile" if $romfile;
3567             }
3568
3569             push @$devices, '-device', $devicestr;
3570             $j++;
3571         }
3572     }
3573
3574     # usb devices
3575     my @usbdevices = PVE::QemuServer::USB::get_usb_devices($conf, $usbdesc->{format}, $MAX_USB_DEVICES);
3576     push @$devices, @usbdevices if @usbdevices;
3577     # serial devices
3578     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3579         if (my $path = $conf->{"serial$i"}) {
3580             if ($path eq 'socket') {
3581                 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
3582                 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server,nowait";
3583                 # On aarch64, serial0 is the UART device. Qemu only allows
3584                 # connecting UART devices via the '-serial' command line, as
3585                 # the device has a fixed slot on the hardware...
3586                 if ($arch eq 'aarch64' && $i == 0) {
3587                     push @$devices, '-serial', "chardev:serial$i";
3588                 } else {
3589                     push @$devices, '-device', "isa-serial,chardev=serial$i";
3590                 }
3591             } else {
3592                 die "no such serial device\n" if ! -c $path;
3593                 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
3594                 push @$devices, '-device', "isa-serial,chardev=serial$i";
3595             }
3596         }
3597     }
3598
3599     # parallel devices
3600     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
3601         if (my $path = $conf->{"parallel$i"}) {
3602             die "no such parallel device\n" if ! -c $path;
3603             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
3604             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
3605             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
3606         }
3607     }
3608
3609
3610     my $sockets = 1;
3611     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3612     $sockets = $conf->{sockets} if  $conf->{sockets};
3613
3614     my $cores = $conf->{cores} || 1;
3615
3616     my $maxcpus = $sockets * $cores;
3617
3618     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
3619
3620     my $allowed_vcpus = $cpuinfo->{cpus};
3621
3622     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n"
3623         if ($allowed_vcpus < $maxcpus);
3624
3625     if($hotplug_features->{cpu} && qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 7)) {
3626
3627         push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3628         for (my $i = 2; $i <= $vcpus; $i++)  {
3629             my $cpustr = print_cpu_device($conf,$i);
3630             push @$cmd, '-device', $cpustr;
3631         }
3632
3633     } else {
3634
3635         push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3636     }
3637     push @$cmd, '-nodefaults';
3638
3639     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
3640
3641     my $bootindex_hash = {};
3642     my $i = 1;
3643     foreach my $o (split(//, $bootorder)) {
3644         $bootindex_hash->{$o} = $i*100;
3645         $i++;
3646     }
3647
3648     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
3649
3650     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
3651
3652     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
3653
3654     if ($vga->{type} && $vga->{type} !~ m/^serial\d+$/ && $vga->{type} ne 'none'){
3655         push @$devices, '-device', print_vga_device($conf, $vga, $arch, $machine_type, undef, $qxlnum, $bridges);
3656         my $socket = vnc_socket($vmid);
3657         push @$cmd,  '-vnc', "unix:$socket,x509,password";
3658     } else {
3659         push @$cmd, '-vga', 'none' if $vga->{type} eq 'none';
3660         push @$cmd, '-nographic';
3661     }
3662
3663     # time drift fix
3664     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
3665
3666     my $useLocaltime = $conf->{localtime};
3667
3668     if ($winversion >= 5) { # windows
3669         $useLocaltime = 1 if !defined($conf->{localtime});
3670
3671         # use time drift fix when acpi is enabled
3672         if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
3673             $tdf = 1 if !defined($conf->{tdf});
3674         }
3675     }
3676
3677     if ($winversion >= 6) {
3678         push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3679         push @$cmd, '-no-hpet';
3680     }
3681
3682     push @$rtcFlags, 'driftfix=slew' if $tdf;
3683
3684     if (!$kvm) {
3685         push @$machineFlags, 'accel=tcg';
3686     }
3687
3688     if ($machine_type) {
3689         push @$machineFlags, "type=${machine_type}";
3690     }
3691
3692     if ($conf->{startdate}) {
3693         push @$rtcFlags, "base=$conf->{startdate}";
3694     } elsif ($useLocaltime) {
3695         push @$rtcFlags, 'base=localtime';
3696     }
3697
3698     push @$cmd, get_cpu_options($conf, $arch, $kvm, $machine_type, $kvm_off, $kvmver, $winversion, $gpu_passthrough);
3699
3700     PVE::QemuServer::Memory::config($conf, $vmid, $sockets, $cores, $defaults, $hotplug_features, $cmd);
3701
3702     push @$cmd, '-S' if $conf->{freeze};
3703
3704     push @$cmd, '-k', $conf->{keyboard} if defined($conf->{keyboard});
3705
3706     # enable sound
3707     #my $soundhw = $conf->{soundhw} || $defaults->{soundhw};
3708     #push @$cmd, '-soundhw', 'es1370';
3709     #push @$cmd, '-soundhw', $soundhw if $soundhw;
3710
3711     if (parse_guest_agent($conf)->{enabled}) {
3712         my $qgasocket = qmp_socket($vmid, 1);
3713         my $pciaddr = print_pci_addr("qga0", $bridges, $arch, $machine_type);
3714         push @$devices, '-chardev', "socket,path=$qgasocket,server,nowait,id=qga0";
3715         push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3716         push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3717     }
3718
3719     my $spice_port;
3720
3721     if ($qxlnum) {
3722         if ($qxlnum > 1) {
3723             if ($winversion){
3724                 for(my $i = 1; $i < $qxlnum; $i++){
3725                     push @$devices, '-device', print_vga_device($conf, $vga, $arch, $machine_type, $i, $qxlnum, $bridges);
3726                 }
3727             } else {
3728                 # assume other OS works like Linux
3729                 my ($ram, $vram) = ("134217728", "67108864");
3730                 if ($vga->{memory}) {
3731                     $ram = PVE::Tools::convert_size($qxlnum*4*$vga->{memory}, 'mb' => 'b');
3732                     $vram = PVE::Tools::convert_size($qxlnum*2*$vga->{memory}, 'mb' => 'b');
3733                 }
3734                 push @$cmd, '-global', "qxl-vga.ram_size=$ram";
3735                 push @$cmd, '-global', "qxl-vga.vram_size=$vram";
3736             }
3737         }
3738
3739         my $pciaddr = print_pci_addr("spice", $bridges, $arch, $machine_type);
3740
3741         my $nodename = PVE::INotify::nodename();
3742         my $pfamily = PVE::Tools::get_host_address_family($nodename);
3743         my @nodeaddrs = PVE::Tools::getaddrinfo_all('localhost', family => $pfamily);
3744         die "failed to get an ip address of type $pfamily for 'localhost'\n" if !@nodeaddrs;
3745         my $localhost = PVE::Network::addr_to_ip($nodeaddrs[0]->{addr});
3746         $spice_port = PVE::Tools::next_spice_port($pfamily, $localhost);
3747
3748         push @$devices, '-spice', "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";
3749
3750         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3751         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3752         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3753     }
3754
3755     # enable balloon by default, unless explicitly disabled
3756     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3757         $pciaddr = print_pci_addr("balloon0", $bridges, $arch, $machine_type);
3758         push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
3759     }
3760
3761     if ($conf->{watchdog}) {
3762         my $wdopts = parse_watchdog($conf->{watchdog});
3763         $pciaddr = print_pci_addr("watchdog", $bridges, $arch, $machine_type);
3764         my $watchdog = $wdopts->{model} || 'i6300esb';
3765         push @$devices, '-device', "$watchdog$pciaddr";
3766         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3767     }
3768
3769     my $vollist = [];
3770     my $scsicontroller = {};
3771     my $ahcicontroller = {};
3772     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3773
3774     # Add iscsi initiator name if available
3775     if (my $initiator = get_initiator_name()) {
3776         push @$devices, '-iscsi', "initiator-name=$initiator";
3777     }
3778
3779     foreach_drive($conf, sub {
3780         my ($ds, $drive) = @_;
3781
3782         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3783             push @$vollist, $drive->{file};
3784         }
3785
3786         # ignore efidisk here, already added in bios/fw handling code above
3787         return if $drive->{interface} eq 'efidisk';
3788
3789         $use_virtio = 1 if $ds =~ m/^virtio/;
3790
3791         if (drive_is_cdrom ($drive)) {
3792             if ($bootindex_hash->{d}) {
3793                 $drive->{bootindex} = $bootindex_hash->{d};
3794                 $bootindex_hash->{d} += 1;
3795             }
3796         } else {
3797             if ($bootindex_hash->{c}) {
3798                 $drive->{bootindex} = $bootindex_hash->{c} if $conf->{bootdisk} && ($conf->{bootdisk} eq $ds);
3799                 $bootindex_hash->{c} += 1;
3800             }
3801         }
3802
3803         if($drive->{interface} eq 'virtio'){
3804            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3805         }
3806
3807         if ($drive->{interface} eq 'scsi') {
3808
3809             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
3810
3811             $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges, $arch, $machine_type);
3812             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
3813
3814             my $iothread = '';
3815             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
3816                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3817                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3818             } elsif ($drive->{iothread}) {
3819                 warn "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n";
3820             }
3821
3822             my $queues = '';
3823             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
3824                 $queues = ",num_queues=$drive->{queues}";
3825             }
3826
3827             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues" if !$scsicontroller->{$controller};
3828             $scsicontroller->{$controller}=1;
3829         }
3830
3831         if ($drive->{interface} eq 'sata') {
3832            my $controller = int($drive->{index} / $MAX_SATA_DISKS);
3833            $pciaddr = print_pci_addr("ahci$controller", $bridges, $arch, $machine_type);
3834            push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr" if !$ahcicontroller->{$controller};
3835            $ahcicontroller->{$controller}=1;
3836         }
3837
3838         my $drive_cmd = print_drive_full($storecfg, $vmid, $drive);
3839         push @$devices, '-drive',$drive_cmd;
3840         push @$devices, '-device', print_drivedevice_full($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type);
3841     });
3842
3843     for (my $i = 0; $i < $MAX_NETS; $i++) {
3844          next if !$conf->{"net$i"};
3845          my $d = parse_net($conf->{"net$i"});
3846          next if !$d;
3847
3848          $use_virtio = 1 if $d->{model} eq 'virtio';
3849
3850          if ($bootindex_hash->{n}) {
3851             $d->{bootindex} = $bootindex_hash->{n};
3852             $bootindex_hash->{n} += 1;
3853          }
3854
3855          my $netdevfull = print_netdev_full($vmid, $conf, $arch, $d, "net$i");
3856          push @$devices, '-netdev', $netdevfull;
3857
3858          my $netdevicefull = print_netdevice_full($vmid, $conf, $d, "net$i", $bridges, $use_old_bios_files, $arch, $machine_type);
3859          push @$devices, '-device', $netdevicefull;
3860     }
3861
3862     if (!$q35) {
3863         # add pci bridges
3864         if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3865            $bridges->{1} = 1;
3866            $bridges->{2} = 1;
3867         }
3868
3869         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
3870
3871         while (my ($k, $v) = each %$bridges) {
3872             $pciaddr = print_pci_addr("pci.$k", undef, $arch, $machine_type);
3873             unshift @$devices, '-device', "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr" if $k > 0;
3874         }
3875     }
3876
3877     # add custom args
3878     if ($conf->{args}) {
3879         my $aa = PVE::Tools::split_args($conf->{args});
3880         push @$cmd, @$aa;
3881     }
3882
3883     push @$cmd, @$devices;
3884     push @$cmd, '-rtc', join(',', @$rtcFlags)
3885         if scalar(@$rtcFlags);
3886     push @$cmd, '-machine', join(',', @$machineFlags)
3887         if scalar(@$machineFlags);
3888     push @$cmd, '-global', join(',', @$globalFlags)
3889         if scalar(@$globalFlags);
3890
3891     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
3892 }
3893
3894 sub vnc_socket {
3895     my ($vmid) = @_;
3896     return "${var_run_tmpdir}/$vmid.vnc";
3897 }
3898
3899 sub spice_port {
3900     my ($vmid) = @_;
3901
3902     my $res = vm_mon_cmd($vmid, 'query-spice');
3903
3904     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
3905 }
3906
3907 sub qmp_socket {
3908     my ($vmid, $qga, $name) = @_;
3909     my $sockettype = $qga ? 'qga' : 'qmp';
3910     my $ext = $name ? '-'.$name : '';
3911     return "${var_run_tmpdir}/$vmid$ext.$sockettype";
3912 }
3913
3914 sub pidfile_name {
3915     my ($vmid) = @_;
3916     return "${var_run_tmpdir}/$vmid.pid";
3917 }
3918
3919 sub vm_devices_list {
3920     my ($vmid) = @_;
3921
3922     my $res = vm_mon_cmd($vmid, 'query-pci');
3923     my $devices_to_check = [];
3924     my $devices = {};
3925     foreach my $pcibus (@$res) {
3926         push @$devices_to_check, @{$pcibus->{devices}},
3927     }
3928
3929     while (@$devices_to_check) {
3930         my $to_check = [];
3931         for my $d (@$devices_to_check) {
3932             $devices->{$d->{'qdev_id'}} = 1 if $d->{'qdev_id'};
3933             next if !$d->{'pci_bridge'};
3934
3935             $devices->{$d->{'qdev_id'}} += scalar(@{$d->{'pci_bridge'}->{devices}});
3936             push @$to_check, @{$d->{'pci_bridge'}->{devices}};
3937         }
3938         $devices_to_check = $to_check;
3939     }
3940
3941     my $resblock = vm_mon_cmd($vmid, 'query-block');
3942     foreach my $block (@$resblock) {
3943         if($block->{device} =~ m/^drive-(\S+)/){
3944                 $devices->{$1} = 1;
3945         }
3946     }
3947
3948     my $resmice = vm_mon_cmd($vmid, 'query-mice');
3949     foreach my $mice (@$resmice) {
3950         if ($mice->{name} eq 'QEMU HID Tablet') {
3951             $devices->{tablet} = 1;
3952             last;
3953         }
3954     }
3955
3956     # for usb devices there is no query-usb
3957     # but we can iterate over the entries in
3958     # qom-list path=/machine/peripheral
3959     my $resperipheral = vm_mon_cmd($vmid, 'qom-list', path => '/machine/peripheral');
3960     foreach my $per (@$resperipheral) {
3961         if ($per->{name} =~ m/^usb\d+$/) {
3962             $devices->{$per->{name}} = 1;
3963         }
3964     }
3965
3966     return $devices;
3967 }
3968
3969 sub vm_deviceplug {
3970     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
3971
3972     my $q35 = machine_type_is_q35($conf);
3973
3974     my $devices_list = vm_devices_list($vmid);
3975     return 1 if defined($devices_list->{$deviceid});
3976
3977     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid, $arch, $machine_type); # add PCI bridge if we need it for the device
3978
3979     if ($deviceid eq 'tablet') {
3980
3981         qemu_deviceadd($vmid, print_tabletdevice_full($conf, $arch));
3982
3983     } elsif ($deviceid eq 'keyboard') {
3984
3985         qemu_deviceadd($vmid, print_keyboarddevice_full($conf, $arch));
3986
3987     } elsif ($deviceid =~ m/^usb(\d+)$/) {
3988
3989         die "usb hotplug currently not reliable\n";
3990         # since we can't reliably hot unplug all added usb devices
3991         # and usb passthrough disables live migration
3992         # we disable usb hotplugging for now
3993         qemu_deviceadd($vmid, PVE::QemuServer::USB::print_usbdevice_full($conf, $deviceid, $device));
3994
3995     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3996
3997         qemu_iothread_add($vmid, $deviceid, $device);
3998
3999         qemu_driveadd($storecfg, $vmid, $device);
4000         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, $arch, $machine_type);
4001
4002         qemu_deviceadd($vmid, $devicefull);
4003         eval { qemu_deviceaddverify($vmid, $deviceid); };
4004         if (my $err = $@) {
4005             eval { qemu_drivedel($vmid, $deviceid); };
4006             warn $@ if $@;
4007             die $err;
4008         }
4009
4010     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4011
4012
4013         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
4014         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
4015         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
4016
4017         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
4018
4019         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
4020             qemu_iothread_add($vmid, $deviceid, $device);
4021             $devicefull .= ",iothread=iothread-$deviceid";
4022         }
4023
4024         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
4025             $devicefull .= ",num_queues=$device->{queues}";
4026         }
4027
4028         qemu_deviceadd($vmid, $devicefull);
4029         qemu_deviceaddverify($vmid, $deviceid);
4030
4031     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4032
4033         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device, $arch, $machine_type);
4034         qemu_driveadd($storecfg, $vmid, $device);
4035
4036         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, $arch, $machine_type);
4037         eval { qemu_deviceadd($vmid, $devicefull); };
4038         if (my $err = $@) {
4039             eval { qemu_drivedel($vmid, $deviceid); };
4040             warn $@ if $@;
4041             die $err;
4042         }
4043
4044     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4045
4046         return undef if !qemu_netdevadd($vmid, $conf, $arch, $device, $deviceid);
4047
4048         my $machine_type = PVE::QemuServer::qemu_machine_pxe($vmid, $conf);
4049         my $use_old_bios_files = undef;
4050         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
4051
4052         my $netdevicefull = print_netdevice_full($vmid, $conf, $device, $deviceid, undef, $use_old_bios_files, $arch, $machine_type);
4053         qemu_deviceadd($vmid, $netdevicefull);
4054         eval { qemu_deviceaddverify($vmid, $deviceid); };
4055         if (my $err = $@) {
4056             eval { qemu_netdevdel($vmid, $deviceid); };
4057             warn $@ if $@;
4058             die $err;
4059         }
4060
4061     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
4062
4063         my $bridgeid = $2;
4064         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
4065         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
4066
4067         qemu_deviceadd($vmid, $devicefull);
4068         qemu_deviceaddverify($vmid, $deviceid);
4069
4070     } else {
4071         die "can't hotplug device '$deviceid'\n";
4072     }
4073
4074     return 1;
4075 }
4076
4077 # fixme: this should raise exceptions on error!
4078 sub vm_deviceunplug {
4079     my ($vmid, $conf, $deviceid) = @_;
4080
4081     my $devices_list = vm_devices_list($vmid);
4082     return 1 if !defined($devices_list->{$deviceid});
4083
4084     die "can't unplug bootdisk" if $conf->{bootdisk} && $conf->{bootdisk} eq $deviceid;
4085
4086     if ($deviceid eq 'tablet' || $deviceid eq 'keyboard') {
4087
4088         qemu_devicedel($vmid, $deviceid);
4089
4090     } elsif ($deviceid =~ m/^usb\d+$/) {
4091
4092         die "usb hotplug currently not reliable\n";
4093         # when unplugging usb devices this way,
4094         # there may be remaining usb controllers/hubs
4095         # so we disable it for now
4096         qemu_devicedel($vmid, $deviceid);
4097         qemu_devicedelverify($vmid, $deviceid);
4098
4099     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4100
4101         qemu_devicedel($vmid, $deviceid);
4102         qemu_devicedelverify($vmid, $deviceid);
4103         qemu_drivedel($vmid, $deviceid);
4104         qemu_iothread_del($conf, $vmid, $deviceid);
4105
4106     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4107
4108         qemu_devicedel($vmid, $deviceid);
4109         qemu_devicedelverify($vmid, $deviceid);
4110         qemu_iothread_del($conf, $vmid, $deviceid);
4111
4112     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4113
4114         qemu_devicedel($vmid, $deviceid);
4115         qemu_drivedel($vmid, $deviceid);
4116         qemu_deletescsihw($conf, $vmid, $deviceid);
4117
4118     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4119
4120         qemu_devicedel($vmid, $deviceid);
4121         qemu_devicedelverify($vmid, $deviceid);
4122         qemu_netdevdel($vmid, $deviceid);
4123
4124     } else {
4125         die "can't unplug device '$deviceid'\n";
4126     }
4127
4128     return 1;
4129 }
4130
4131 sub qemu_deviceadd {
4132     my ($vmid, $devicefull) = @_;
4133
4134     $devicefull = "driver=".$devicefull;
4135     my %options =  split(/[=,]/, $devicefull);
4136
4137     vm_mon_cmd($vmid, "device_add" , %options);
4138 }
4139
4140 sub qemu_devicedel {
4141     my ($vmid, $deviceid) = @_;
4142
4143     my $ret = vm_mon_cmd($vmid, "device_del", id => $deviceid);
4144 }
4145
4146 sub qemu_iothread_add {
4147     my($vmid, $deviceid, $device) = @_;
4148
4149     if ($device->{iothread}) {
4150         my $iothreads = vm_iothreads_list($vmid);
4151         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
4152     }
4153 }
4154
4155 sub qemu_iothread_del {
4156     my($conf, $vmid, $deviceid) = @_;
4157
4158     my $device = parse_drive($deviceid, $conf->{$deviceid});
4159     if ($device->{iothread}) {
4160         my $iothreads = vm_iothreads_list($vmid);
4161         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
4162     }
4163 }
4164
4165 sub qemu_objectadd {
4166     my($vmid, $objectid, $qomtype) = @_;
4167
4168     vm_mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
4169
4170     return 1;
4171 }
4172
4173 sub qemu_objectdel {
4174     my($vmid, $objectid) = @_;
4175
4176     vm_mon_cmd($vmid, "object-del", id => $objectid);
4177
4178     return 1;
4179 }
4180
4181 sub qemu_driveadd {
4182     my ($storecfg, $vmid, $device) = @_;
4183
4184     my $drive = print_drive_full($storecfg, $vmid, $device);
4185     $drive =~ s/\\/\\\\/g;
4186     my $ret = vm_human_monitor_command($vmid, "drive_add auto \"$drive\"");
4187
4188     # If the command succeeds qemu prints: "OK"
4189     return 1 if $ret =~ m/OK/s;
4190
4191     die "adding drive failed: $ret\n";
4192 }
4193
4194 sub qemu_drivedel {
4195     my($vmid, $deviceid) = @_;
4196
4197     my $ret = vm_human_monitor_command($vmid, "drive_del drive-$deviceid");
4198     $ret =~ s/^\s+//;
4199
4200     return 1 if $ret eq "";
4201
4202     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
4203     return 1 if $ret =~ m/Device \'.*?\' not found/s;
4204
4205     die "deleting drive $deviceid failed : $ret\n";
4206 }
4207
4208 sub qemu_deviceaddverify {
4209     my ($vmid, $deviceid) = @_;
4210
4211     for (my $i = 0; $i <= 5; $i++) {
4212          my $devices_list = vm_devices_list($vmid);
4213          return 1 if defined($devices_list->{$deviceid});
4214          sleep 1;
4215     }
4216
4217     die "error on hotplug device '$deviceid'\n";
4218 }
4219
4220
4221 sub qemu_devicedelverify {
4222     my ($vmid, $deviceid) = @_;
4223
4224     # need to verify that the device is correctly removed as device_del
4225     # is async and empty return is not reliable
4226
4227     for (my $i = 0; $i <= 5; $i++) {
4228          my $devices_list = vm_devices_list($vmid);
4229          return 1 if !defined($devices_list->{$deviceid});
4230          sleep 1;
4231     }
4232
4233     die "error on hot-unplugging device '$deviceid'\n";
4234 }
4235
4236 sub qemu_findorcreatescsihw {
4237     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4238
4239     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4240
4241     my $scsihwid="$controller_prefix$controller";
4242     my $devices_list = vm_devices_list($vmid);
4243
4244     if(!defined($devices_list->{$scsihwid})) {
4245         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device, $arch, $machine_type);
4246     }
4247
4248     return 1;
4249 }
4250
4251 sub qemu_deletescsihw {
4252     my ($conf, $vmid, $opt) = @_;
4253
4254     my $device = parse_drive($opt, $conf->{$opt});
4255
4256     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
4257         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
4258         return 1;
4259     }
4260
4261     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4262
4263     my $devices_list = vm_devices_list($vmid);
4264     foreach my $opt (keys %{$devices_list}) {
4265         if (PVE::QemuServer::is_valid_drivename($opt)) {
4266             my $drive = PVE::QemuServer::parse_drive($opt, $conf->{$opt});
4267             if($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
4268                 return 1;
4269             }
4270         }
4271     }
4272
4273     my $scsihwid="scsihw$controller";
4274
4275     vm_deviceunplug($vmid, $conf, $scsihwid);
4276
4277     return 1;
4278 }
4279
4280 sub qemu_add_pci_bridge {
4281     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4282
4283     my $bridges = {};
4284
4285     my $bridgeid;
4286
4287     print_pci_addr($device, $bridges, $arch, $machine_type);
4288
4289     while (my ($k, $v) = each %$bridges) {
4290         $bridgeid = $k;
4291     }
4292     return 1 if !defined($bridgeid) || $bridgeid < 1;
4293
4294     my $bridge = "pci.$bridgeid";
4295     my $devices_list = vm_devices_list($vmid);
4296
4297     if (!defined($devices_list->{$bridge})) {
4298         vm_deviceplug($storecfg, $conf, $vmid, $bridge, $arch, $machine_type);
4299     }
4300
4301     return 1;
4302 }
4303
4304 sub qemu_set_link_status {
4305     my ($vmid, $device, $up) = @_;
4306
4307     vm_mon_cmd($vmid, "set_link", name => $device,
4308                up => $up ? JSON::true : JSON::false);
4309 }
4310
4311 sub qemu_netdevadd {
4312     my ($vmid, $conf, $arch, $device, $deviceid) = @_;
4313
4314     my $netdev = print_netdev_full($vmid, $conf, $arch, $device, $deviceid, 1);
4315     my %options =  split(/[=,]/, $netdev);
4316
4317     vm_mon_cmd($vmid, "netdev_add",  %options);
4318     return 1;
4319 }
4320
4321 sub qemu_netdevdel {
4322     my ($vmid, $deviceid) = @_;
4323
4324     vm_mon_cmd($vmid, "netdev_del", id => $deviceid);
4325 }
4326
4327 sub qemu_usb_hotplug {
4328     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4329
4330     return if !$device;
4331
4332     # remove the old one first
4333     vm_deviceunplug($vmid, $conf, $deviceid);
4334
4335     # check if xhci controller is necessary and available
4336     if ($device->{usb3}) {
4337
4338         my $devicelist = vm_devices_list($vmid);
4339
4340         if (!$devicelist->{xhci}) {
4341             my $pciaddr = print_pci_addr("xhci", undef, $arch, $machine_type);
4342             qemu_deviceadd($vmid, "nec-usb-xhci,id=xhci$pciaddr");
4343         }
4344     }
4345     my $d = parse_usb_device($device->{host});
4346     $d->{usb3} = $device->{usb3};
4347
4348     # add the new one
4349     vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $d, $arch, $machine_type);
4350 }
4351
4352 sub qemu_cpu_hotplug {
4353     my ($vmid, $conf, $vcpus) = @_;
4354
4355     my $machine_type = PVE::QemuServer::get_current_qemu_machine($vmid);
4356
4357     my $sockets = 1;
4358     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
4359     $sockets = $conf->{sockets} if  $conf->{sockets};
4360     my $cores = $conf->{cores} || 1;
4361     my $maxcpus = $sockets * $cores;
4362
4363     $vcpus = $maxcpus if !$vcpus;
4364
4365     die "you can't add more vcpus than maxcpus\n"
4366         if $vcpus > $maxcpus;
4367
4368     my $currentvcpus = $conf->{vcpus} || $maxcpus;
4369
4370     if ($vcpus < $currentvcpus) {
4371
4372         if (qemu_machine_feature_enabled ($machine_type, undef, 2, 7)) {
4373
4374             for (my $i = $currentvcpus; $i > $vcpus; $i--) {
4375                 qemu_devicedel($vmid, "cpu$i");
4376                 my $retry = 0;
4377                 my $currentrunningvcpus = undef;
4378                 while (1) {
4379                     $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
4380                     last if scalar(@{$currentrunningvcpus}) == $i-1;
4381                     raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
4382                     $retry++;
4383                     sleep 1;
4384                 }
4385                 #update conf after each succesfull cpu unplug
4386                 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4387                 PVE::QemuConfig->write_config($vmid, $conf);
4388             }
4389         } else {
4390             die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
4391         }
4392
4393         return;
4394     }
4395
4396     my $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
4397     die "vcpus in running vm does not match its configuration\n"
4398         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
4399
4400     if (qemu_machine_feature_enabled ($machine_type, undef, 2, 7)) {
4401
4402         for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
4403             my $cpustr = print_cpu_device($conf, $i);
4404             qemu_deviceadd($vmid, $cpustr);
4405
4406             my $retry = 0;
4407             my $currentrunningvcpus = undef;
4408             while (1) {
4409                 $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
4410                 last if scalar(@{$currentrunningvcpus}) == $i;
4411                 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
4412                 sleep 1;
4413                 $retry++;
4414             }
4415             #update conf after each succesfull cpu hotplug
4416             $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4417             PVE::QemuConfig->write_config($vmid, $conf);
4418         }
4419     } else {
4420
4421         for (my $i = $currentvcpus; $i < $vcpus; $i++) {
4422             vm_mon_cmd($vmid, "cpu-add", id => int($i));
4423         }
4424     }
4425 }
4426
4427 sub qemu_block_set_io_throttle {
4428     my ($vmid, $deviceid,
4429         $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
4430         $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max,
4431         $bps_max_length, $bps_rd_max_length, $bps_wr_max_length,
4432         $iops_max_length, $iops_rd_max_length, $iops_wr_max_length) = @_;
4433
4434     return if !check_running($vmid) ;
4435
4436     vm_mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
4437         bps => int($bps),
4438         bps_rd => int($bps_rd),
4439         bps_wr => int($bps_wr),
4440         iops => int($iops),
4441         iops_rd => int($iops_rd),
4442         iops_wr => int($iops_wr),
4443         bps_max => int($bps_max),
4444         bps_rd_max => int($bps_rd_max),
4445         bps_wr_max => int($bps_wr_max),
4446         iops_max => int($iops_max),
4447         iops_rd_max => int($iops_rd_max),
4448         iops_wr_max => int($iops_wr_max),
4449         bps_max_length => int($bps_max_length),
4450         bps_rd_max_length => int($bps_rd_max_length),
4451         bps_wr_max_length => int($bps_wr_max_length),
4452         iops_max_length => int($iops_max_length),
4453         iops_rd_max_length => int($iops_rd_max_length),
4454         iops_wr_max_length => int($iops_wr_max_length),
4455     );
4456
4457 }
4458
4459 # old code, only used to shutdown old VM after update
4460 sub __read_avail {
4461     my ($fh, $timeout) = @_;
4462
4463     my $sel = new IO::Select;
4464     $sel->add($fh);
4465
4466     my $res = '';
4467     my $buf;
4468
4469     my @ready;
4470     while (scalar (@ready = $sel->can_read($timeout))) {
4471         my $count;
4472         if ($count = $fh->sysread($buf, 8192)) {
4473             if ($buf =~ /^(.*)\(qemu\) $/s) {
4474                 $res .= $1;
4475                 last;
4476             } else {
4477                 $res .= $buf;
4478             }
4479         } else {
4480             if (!defined($count)) {
4481                 die "$!\n";
4482             }
4483             last;
4484         }
4485     }
4486
4487     die "monitor read timeout\n" if !scalar(@ready);
4488
4489     return $res;
4490 }
4491
4492 sub qemu_block_resize {
4493     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4494
4495     my $running = check_running($vmid);
4496
4497     $size = 0 if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4498
4499     return if !$running;
4500
4501     vm_mon_cmd($vmid, "block_resize", device => $deviceid, size => int($size));
4502
4503 }
4504
4505 sub qemu_volume_snapshot {
4506     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4507
4508     my $running = check_running($vmid);
4509
4510     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4511         vm_mon_cmd($vmid, 'blockdev-snapshot-internal-sync', device => $deviceid, name => $snap);
4512     } else {
4513         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4514     }
4515 }
4516
4517 sub qemu_volume_snapshot_delete {
4518     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4519
4520     my $running = check_running($vmid);
4521
4522     if($running) {
4523
4524         $running = undef;
4525         my $conf = PVE::QemuConfig->load_config($vmid);
4526         foreach_drive($conf, sub {
4527             my ($ds, $drive) = @_;
4528             $running = 1 if $drive->{file} eq $volid;
4529         });
4530     }
4531
4532     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4533         vm_mon_cmd($vmid, 'blockdev-snapshot-delete-internal-sync', device => $deviceid, name => $snap);
4534     } else {
4535         PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4536     }
4537 }
4538
4539 sub set_migration_caps {
4540     my ($vmid) = @_;
4541
4542     my $cap_ref = [];
4543
4544     my $enabled_cap = {
4545         "auto-converge" => 1,
4546         "xbzrle" => 1,
4547         "x-rdma-pin-all" => 0,
4548         "zero-blocks" => 0,
4549         "compress" => 0
4550     };
4551
4552     my $supported_capabilities = vm_mon_cmd_nocheck($vmid, "query-migrate-capabilities");
4553
4554     for my $supported_capability (@$supported_capabilities) {
4555         push @$cap_ref, {
4556             capability => $supported_capability->{capability},
4557             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4558         };
4559     }
4560
4561     vm_mon_cmd_nocheck($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4562 }
4563
4564 my $fast_plug_option = {
4565     'lock' => 1,
4566     'name' => 1,
4567     'onboot' => 1,
4568     'shares' => 1,
4569     'startup' => 1,
4570     'description' => 1,
4571     'protection' => 1,
4572     'vmstatestorage' => 1,
4573 };
4574
4575 # hotplug changes in [PENDING]
4576 # $selection hash can be used to only apply specified options, for
4577 # example: { cores => 1 } (only apply changed 'cores')
4578 # $errors ref is used to return error messages
4579 sub vmconfig_hotplug_pending {
4580     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4581
4582     my $defaults = load_defaults();
4583     my ($arch, $machine_type) = get_basic_machine_info($conf, undef);
4584
4585     # commit values which do not have any impact on running VM first
4586     # Note: those option cannot raise errors, we we do not care about
4587     # $selection and always apply them.
4588
4589     my $add_error = sub {
4590         my ($opt, $msg) = @_;
4591         $errors->{$opt} = "hotplug problem - $msg";
4592     };
4593
4594     my $changes = 0;
4595     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4596         if ($fast_plug_option->{$opt}) {
4597             $conf->{$opt} = $conf->{pending}->{$opt};
4598             delete $conf->{pending}->{$opt};
4599             $changes = 1;
4600         }
4601     }
4602
4603     if ($changes) {
4604         PVE::QemuConfig->write_config($vmid, $conf);
4605         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4606     }
4607
4608     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4609
4610     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4611     while (my ($opt, $force) = each %$pending_delete_hash) {
4612         next if $selection && !$selection->{$opt};
4613         eval {
4614             if ($opt eq 'hotplug') {
4615                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
4616             } elsif ($opt eq 'tablet') {
4617                 die "skip\n" if !$hotplug_features->{usb};
4618                 if ($defaults->{tablet}) {
4619                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4620                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4621                         if $arch eq 'aarch64';
4622                 } else {
4623                     vm_deviceunplug($vmid, $conf, 'tablet');
4624                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4625                 }
4626             } elsif ($opt =~ m/^usb\d+/) {
4627                 die "skip\n";
4628                 # since we cannot reliably hot unplug usb devices
4629                 # we are disabling it
4630                 die "skip\n" if !$hotplug_features->{usb} || $conf->{$opt} =~ m/spice/i;
4631                 vm_deviceunplug($vmid, $conf, $opt);
4632             } elsif ($opt eq 'vcpus') {
4633                 die "skip\n" if !$hotplug_features->{cpu};
4634                 qemu_cpu_hotplug($vmid, $conf, undef);
4635             } elsif ($opt eq 'balloon') {
4636                 # enable balloon device is not hotpluggable
4637                 die "skip\n" if defined($conf->{balloon}) && $conf->{balloon} == 0;
4638                 # here we reset the ballooning value to memory
4639                 my $balloon = $conf->{memory} || $defaults->{memory};
4640                 vm_mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4641             } elsif ($fast_plug_option->{$opt}) {
4642                 # do nothing
4643             } elsif ($opt =~ m/^net(\d+)$/) {
4644                 die "skip\n" if !$hotplug_features->{network};
4645                 vm_deviceunplug($vmid, $conf, $opt);
4646             } elsif (is_valid_drivename($opt)) {
4647                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4648                 vm_deviceunplug($vmid, $conf, $opt);
4649                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4650             } elsif ($opt =~ m/^memory$/) {
4651                 die "skip\n" if !$hotplug_features->{memory};
4652                 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
4653             } elsif ($opt eq 'cpuunits') {
4654                 cgroups_write("cpu", $vmid, "cpu.shares", $defaults->{cpuunits});
4655             } elsif ($opt eq 'cpulimit') {
4656                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", -1);
4657             } else {
4658                 die "skip\n";
4659             }
4660         };
4661         if (my $err = $@) {
4662             &$add_error($opt, $err) if $err ne "skip\n";
4663         } else {
4664             # save new config if hotplug was successful
4665             delete $conf->{$opt};
4666             vmconfig_undelete_pending_option($conf, $opt);
4667             PVE::QemuConfig->write_config($vmid, $conf);
4668             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4669         }
4670     }
4671
4672     my $apply_pending_cloudinit;
4673     $apply_pending_cloudinit = sub {
4674         my ($key, $value) = @_;
4675         $apply_pending_cloudinit = sub {}; # once is enough
4676
4677         my @cloudinit_opts = keys %$confdesc_cloudinit;
4678         foreach my $opt (keys %{$conf->{pending}}) {
4679             next if !grep { $_ eq $opt } @cloudinit_opts;
4680             $conf->{$opt} = delete $conf->{pending}->{$opt};
4681         }
4682
4683         my $new_conf = { %$conf };
4684         $new_conf->{$key} = $value;
4685         PVE::QemuServer::Cloudinit::generate_cloudinitconfig($new_conf, $vmid);
4686     };
4687
4688     foreach my $opt (keys %{$conf->{pending}}) {
4689         next if $selection && !$selection->{$opt};
4690         my $value = $conf->{pending}->{$opt};
4691         eval {
4692             if ($opt eq 'hotplug') {
4693                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
4694             } elsif ($opt eq 'tablet') {
4695                 die "skip\n" if !$hotplug_features->{usb};
4696                 if ($value == 1) {
4697                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4698                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4699                         if $arch eq 'aarch64';
4700                 } elsif ($value == 0) {
4701                     vm_deviceunplug($vmid, $conf, 'tablet');
4702                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4703                 }
4704             } elsif ($opt =~ m/^usb\d+$/) {
4705                 die "skip\n";
4706                 # since we cannot reliably hot unplug usb devices
4707                 # we are disabling it
4708                 die "skip\n" if !$hotplug_features->{usb} || $value =~ m/spice/i;
4709                 my $d = eval { PVE::JSONSchema::parse_property_string($usbdesc->{format}, $value) };
4710                 die "skip\n" if !$d;
4711                 qemu_usb_hotplug($storecfg, $conf, $vmid, $opt, $d, $arch, $machine_type);
4712             } elsif ($opt eq 'vcpus') {
4713                 die "skip\n" if !$hotplug_features->{cpu};
4714                 qemu_cpu_hotplug($vmid, $conf, $value);
4715             } elsif ($opt eq 'balloon') {
4716                 # enable/disable balloning device is not hotpluggable
4717                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
4718                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
4719                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
4720
4721                 # allow manual ballooning if shares is set to zero
4722                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
4723                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
4724                     vm_mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4725                 }
4726             } elsif ($opt =~ m/^net(\d+)$/) {
4727                 # some changes can be done without hotplug
4728                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
4729                                     $vmid, $opt, $value, $arch, $machine_type);
4730             } elsif (is_valid_drivename($opt)) {
4731                 # some changes can be done without hotplug
4732                 my $drive = parse_drive($opt, $value);
4733                 if (drive_is_cloudinit($drive)) {
4734                     &$apply_pending_cloudinit($opt, $value);
4735                 }
4736                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
4737                                      $vmid, $opt, $value, 1, $arch, $machine_type);
4738             } elsif ($opt =~ m/^memory$/) { #dimms
4739                 die "skip\n" if !$hotplug_features->{memory};
4740                 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
4741             } elsif ($opt eq 'cpuunits') {
4742                 cgroups_write("cpu", $vmid, "cpu.shares", $conf->{pending}->{$opt});
4743             } elsif ($opt eq 'cpulimit') {
4744                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
4745                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", $cpulimit);
4746             } else {
4747                 die "skip\n";  # skip non-hot-pluggable options
4748             }
4749         };
4750         if (my $err = $@) {
4751             &$add_error($opt, $err) if $err ne "skip\n";
4752         } else {
4753             # save new config if hotplug was successful
4754             $conf->{$opt} = $value;
4755             delete $conf->{pending}->{$opt};
4756             PVE::QemuConfig->write_config($vmid, $conf);
4757             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4758         }
4759     }
4760 }
4761
4762 sub try_deallocate_drive {
4763     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
4764
4765     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
4766         my $volid = $drive->{file};
4767         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
4768             my $sid = PVE::Storage::parse_volume_id($volid);
4769             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
4770
4771             # check if the disk is really unused
4772             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
4773                 if is_volume_in_use($storecfg, $conf, $key, $volid);
4774             PVE::Storage::vdisk_free($storecfg, $volid);
4775             return 1;
4776         } else {
4777             # If vm is not owner of this disk remove from config
4778             return 1;
4779         }
4780     }
4781
4782     return undef;
4783 }
4784
4785 sub vmconfig_delete_or_detach_drive {
4786     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
4787
4788     my $drive = parse_drive($opt, $conf->{$opt});
4789
4790     my $rpcenv = PVE::RPCEnvironment::get();
4791     my $authuser = $rpcenv->get_user();
4792
4793     if ($force) {
4794         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
4795         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
4796     } else {
4797         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
4798     }
4799 }
4800
4801 sub vmconfig_apply_pending {
4802     my ($vmid, $conf, $storecfg) = @_;
4803
4804     # cold plug
4805
4806     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4807     while (my ($opt, $force) = each %$pending_delete_hash) {
4808         die "internal error" if $opt =~ m/^unused/;
4809         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4810         if (!defined($conf->{$opt})) {
4811             vmconfig_undelete_pending_option($conf, $opt);
4812             PVE::QemuConfig->write_config($vmid, $conf);
4813         } elsif (is_valid_drivename($opt)) {
4814             vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4815             vmconfig_undelete_pending_option($conf, $opt);
4816             delete $conf->{$opt};
4817             PVE::QemuConfig->write_config($vmid, $conf);
4818         } else {
4819             vmconfig_undelete_pending_option($conf, $opt);
4820             delete $conf->{$opt};
4821             PVE::QemuConfig->write_config($vmid, $conf);
4822         }
4823     }
4824
4825     $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4826
4827     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4828         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4829
4830         if (defined($conf->{$opt}) && ($conf->{$opt} eq $conf->{pending}->{$opt})) {
4831             # skip if nothing changed
4832         } elsif (is_valid_drivename($opt)) {
4833             vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
4834                 if defined($conf->{$opt});
4835             $conf->{$opt} = $conf->{pending}->{$opt};
4836         } else {
4837             $conf->{$opt} = $conf->{pending}->{$opt};
4838         }
4839
4840         delete $conf->{pending}->{$opt};
4841         PVE::QemuConfig->write_config($vmid, $conf);
4842     }
4843 }
4844
4845 my $safe_num_ne = sub {
4846     my ($a, $b) = @_;
4847
4848     return 0 if !defined($a) && !defined($b);
4849     return 1 if !defined($a);
4850     return 1 if !defined($b);
4851
4852     return $a != $b;
4853 };
4854
4855 my $safe_string_ne = sub {
4856     my ($a, $b) = @_;
4857
4858     return 0 if !defined($a) && !defined($b);
4859     return 1 if !defined($a);
4860     return 1 if !defined($b);
4861
4862     return $a ne $b;
4863 };
4864
4865 sub vmconfig_update_net {
4866     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
4867
4868     my $newnet = parse_net($value);
4869
4870     if ($conf->{$opt}) {
4871         my $oldnet = parse_net($conf->{$opt});
4872
4873         if (&$safe_string_ne($oldnet->{model}, $newnet->{model}) ||
4874             &$safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
4875             &$safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
4876             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
4877
4878             # for non online change, we try to hot-unplug
4879             die "skip\n" if !$hotplug;
4880             vm_deviceunplug($vmid, $conf, $opt);
4881         } else {
4882
4883             die "internal error" if $opt !~ m/net(\d+)/;
4884             my $iface = "tap${vmid}i$1";
4885
4886             if (&$safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
4887                 &$safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
4888                 &$safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
4889                 &$safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
4890                 PVE::Network::tap_unplug($iface);
4891                 PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
4892             } elsif (&$safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
4893                 # Rate can be applied on its own but any change above needs to
4894                 # include the rate in tap_plug since OVS resets everything.
4895                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
4896             }
4897
4898             if (&$safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
4899                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
4900             }
4901
4902             return 1;
4903         }
4904     }
4905
4906     if ($hotplug) {
4907         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet, $arch, $machine_type);
4908     } else {
4909         die "skip\n";
4910     }
4911 }
4912
4913 sub vmconfig_update_disk {
4914     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $force, $arch, $machine_type) = @_;
4915
4916     # fixme: do we need force?
4917
4918     my $drive = parse_drive($opt, $value);
4919
4920     if ($conf->{$opt}) {
4921
4922         if (my $old_drive = parse_drive($opt, $conf->{$opt}))  {
4923
4924             my $media = $drive->{media} || 'disk';
4925             my $oldmedia = $old_drive->{media} || 'disk';
4926             die "unable to change media type\n" if $media ne $oldmedia;
4927
4928             if (!drive_is_cdrom($old_drive)) {
4929
4930                 if ($drive->{file} ne $old_drive->{file}) {
4931
4932                     die "skip\n" if !$hotplug;
4933
4934                     # unplug and register as unused
4935                     vm_deviceunplug($vmid, $conf, $opt);
4936                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
4937
4938                 } else {
4939                     # update existing disk
4940
4941                     # skip non hotpluggable value
4942                     if (&$safe_string_ne($drive->{discard}, $old_drive->{discard}) ||
4943                         &$safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
4944                         &$safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
4945                         &$safe_string_ne($drive->{cache}, $old_drive->{cache})) {
4946                         die "skip\n";
4947                     }
4948
4949                     # apply throttle
4950                     if (&$safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
4951                         &$safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
4952                         &$safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
4953                         &$safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
4954                         &$safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
4955                         &$safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
4956                         &$safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
4957                         &$safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
4958                         &$safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
4959                         &$safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
4960                         &$safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
4961                         &$safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max}) ||
4962                         &$safe_num_ne($drive->{bps_max_length}, $old_drive->{bps_max_length}) ||
4963                         &$safe_num_ne($drive->{bps_rd_max_length}, $old_drive->{bps_rd_max_length}) ||
4964                         &$safe_num_ne($drive->{bps_wr_max_length}, $old_drive->{bps_wr_max_length}) ||
4965                         &$safe_num_ne($drive->{iops_max_length}, $old_drive->{iops_max_length}) ||
4966                         &$safe_num_ne($drive->{iops_rd_max_length}, $old_drive->{iops_rd_max_length}) ||
4967                         &$safe_num_ne($drive->{iops_wr_max_length}, $old_drive->{iops_wr_max_length})) {
4968
4969                         qemu_block_set_io_throttle($vmid,"drive-$opt",
4970                                                    ($drive->{mbps} || 0)*1024*1024,
4971                                                    ($drive->{mbps_rd} || 0)*1024*1024,
4972                                                    ($drive->{mbps_wr} || 0)*1024*1024,
4973                                                    $drive->{iops} || 0,
4974                                                    $drive->{iops_rd} || 0,
4975                                                    $drive->{iops_wr} || 0,
4976                                                    ($drive->{mbps_max} || 0)*1024*1024,
4977                                                    ($drive->{mbps_rd_max} || 0)*1024*1024,
4978                                                    ($drive->{mbps_wr_max} || 0)*1024*1024,
4979                                                    $drive->{iops_max} || 0,
4980                                                    $drive->{iops_rd_max} || 0,
4981                                                    $drive->{iops_wr_max} || 0,
4982                                                    $drive->{bps_max_length} || 1,
4983                                                    $drive->{bps_rd_max_length} || 1,
4984                                                    $drive->{bps_wr_max_length} || 1,
4985                                                    $drive->{iops_max_length} || 1,
4986                                                    $drive->{iops_rd_max_length} || 1,
4987                                                    $drive->{iops_wr_max_length} || 1);
4988
4989                     }
4990
4991                     return 1;
4992                 }
4993
4994             } else { # cdrom
4995
4996                 if ($drive->{file} eq 'none') {
4997                     vm_mon_cmd($vmid, "eject",force => JSON::true,device => "drive-$opt");
4998                     if (drive_is_cloudinit($old_drive)) {
4999                         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive);
5000                     }
5001                 } else {
5002                     my $path = get_iso_path($storecfg, $vmid, $drive->{file});
5003                     vm_mon_cmd($vmid, "eject", force => JSON::true,device => "drive-$opt"); # force eject if locked
5004                     vm_mon_cmd($vmid, "change", device => "drive-$opt",target => "$path") if $path;
5005                 }
5006
5007                 return 1;
5008             }
5009         }
5010     }
5011
5012     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
5013     # hotplug new disks
5014     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
5015     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive, $arch, $machine_type);
5016 }
5017
5018 sub vm_start {
5019     my ($storecfg, $vmid, $statefile, $skiplock, $migratedfrom, $paused,
5020         $forcemachine, $spice_ticket, $migration_network, $migration_type, $targetstorage) = @_;
5021
5022     PVE::QemuConfig->lock_config($vmid, sub {
5023         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
5024
5025         die "you can't start a vm if it's a template\n" if PVE::QemuConfig->is_template($conf);
5026
5027         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5028
5029         die "VM $vmid already running\n" if check_running($vmid, undef, $migratedfrom);
5030
5031         if (!$statefile && scalar(keys %{$conf->{pending}})) {
5032             vmconfig_apply_pending($vmid, $conf, $storecfg);
5033             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5034         }
5035
5036         PVE::QemuServer::Cloudinit::generate_cloudinitconfig($conf, $vmid);
5037
5038         my $defaults = load_defaults();
5039
5040         # set environment variable useful inside network script
5041         $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
5042
5043         my $local_volumes = {};
5044
5045         if ($targetstorage) {
5046             foreach_drive($conf, sub {
5047                 my ($ds, $drive) = @_;
5048
5049                 return if drive_is_cdrom($drive);
5050
5051                 my $volid = $drive->{file};
5052
5053                 return if !$volid;
5054
5055                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
5056
5057                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5058                 return if $scfg->{shared};
5059                 $local_volumes->{$ds} = [$volid, $storeid, $volname];
5060             });
5061
5062             my $format = undef;
5063
5064             foreach my $opt (sort keys %$local_volumes) {
5065
5066                 my ($volid, $storeid, $volname) = @{$local_volumes->{$opt}};
5067                 my $drive = parse_drive($opt, $conf->{$opt});
5068
5069                 #if remote storage is specified, use default format
5070                 if ($targetstorage && $targetstorage ne "1") {
5071                     $storeid = $targetstorage;
5072                     my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
5073                     $format = $defFormat;
5074                 } else {
5075                     #else we use same format than original
5076                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5077                     $format = qemu_img_format($scfg, $volid);
5078                 }
5079
5080                 my $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $format, undef, ($drive->{size}/1024));
5081                 my $newdrive = $drive;
5082                 $newdrive->{format} = $format;
5083                 $newdrive->{file} = $newvolid;
5084                 my $drivestr = PVE::QemuServer::print_drive($vmid, $newdrive);
5085                 $local_volumes->{$opt} = $drivestr;
5086                 #pass drive to conf for command line
5087                 $conf->{$opt} = $drivestr;
5088             }
5089         }
5090
5091         my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine);
5092
5093         my $migrate_port = 0;
5094         my $migrate_uri;
5095         if ($statefile) {
5096             if ($statefile eq 'tcp') {
5097                 my $localip = "localhost";
5098                 my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
5099                 my $nodename = PVE::INotify::nodename();
5100
5101                 if (!defined($migration_type)) {
5102                     if (defined($datacenterconf->{migration}->{type})) {
5103                         $migration_type = $datacenterconf->{migration}->{type};
5104                     } else {
5105                         $migration_type = 'secure';
5106                     }
5107                 }
5108
5109                 if ($migration_type eq 'insecure') {
5110                     my $migrate_network_addr = PVE::Cluster::get_local_migration_ip($migration_network);
5111                     if ($migrate_network_addr) {
5112                         $localip = $migrate_network_addr;
5113                     } else {
5114                         $localip = PVE::Cluster::remote_node_ip($nodename, 1);
5115                     }
5116
5117                     $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5118                 }
5119
5120                 my $pfamily = PVE::Tools::get_host_address_family($nodename);
5121                 $migrate_port = PVE::Tools::next_migrate_port($pfamily);
5122                 $migrate_uri = "tcp:${localip}:${migrate_port}";
5123                 push @$cmd, '-incoming', $migrate_uri;
5124                 push @$cmd, '-S';
5125
5126             } elsif ($statefile eq 'unix') {
5127                 # should be default for secure migrations as a ssh TCP forward
5128                 # tunnel is not deterministic reliable ready and fails regurarly
5129                 # to set up in time, so use UNIX socket forwards
5130                 my $socket_addr = "/run/qemu-server/$vmid.migrate";
5131                 unlink $socket_addr;
5132
5133                 $migrate_uri = "unix:$socket_addr";
5134
5135                 push @$cmd, '-incoming', $migrate_uri;
5136                 push @$cmd, '-S';
5137
5138             } else {
5139                 push @$cmd, '-loadstate', $statefile;
5140             }
5141         } elsif ($paused) {
5142             push @$cmd, '-S';
5143         }
5144
5145         # host pci devices
5146         for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
5147           my $d = parse_hostpci($conf->{"hostpci$i"});
5148           next if !$d;
5149           my $pcidevices = $d->{pciid};
5150           foreach my $pcidevice (@$pcidevices) {
5151                 my $pciid = $pcidevice->{id}.".".$pcidevice->{function};
5152
5153                 my $info = pci_device_info("0000:$pciid");
5154                 die "IOMMU not present\n" if !check_iommu_support();
5155                 die "no pci device info for device '$pciid'\n" if !$info;
5156                 die "can't unbind/bind pci group to vfio '$pciid'\n" if !pci_dev_group_bind_to_vfio($pciid);
5157                 die "can't reset pci device '$pciid'\n" if $info->{has_fl_reset} and !pci_dev_reset($info);
5158           }
5159         }
5160
5161         PVE::Storage::activate_volumes($storecfg, $vollist);
5162
5163         if (!check_running($vmid, 1)) {
5164             eval {
5165                 run_command(['/bin/systemctl', 'stop', "$vmid.scope"],
5166                     outfunc => sub {}, errfunc => sub {});
5167             };
5168         }
5169
5170         my $cpuunits = defined($conf->{cpuunits}) ? $conf->{cpuunits}
5171                                                   : $defaults->{cpuunits};
5172
5173         my $start_timeout = $conf->{hugepages} ? 300 : 30;
5174         my %run_params = (timeout => $statefile ? undef : $start_timeout, umask => 0077);
5175
5176         my %properties = (
5177             Slice => 'qemu.slice',
5178             KillMode => 'none',
5179             CPUShares => $cpuunits
5180         );
5181
5182         if (my $cpulimit = $conf->{cpulimit}) {
5183             $properties{CPUQuota} = int($cpulimit * 100);
5184         }
5185         $properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
5186
5187         my $run_qemu = sub {
5188             PVE::Tools::run_fork sub {
5189                 PVE::Systemd::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
5190                 run_command($cmd, %run_params);
5191             };
5192         };
5193
5194         if ($conf->{hugepages}) {
5195
5196             my $code = sub {
5197                 my $hugepages_topology = PVE::QemuServer::Memory::hugepages_topology($conf);
5198                 my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
5199
5200                 PVE::QemuServer::Memory::hugepages_mount();
5201                 PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
5202
5203                 eval { $run_qemu->() };
5204                 if (my $err = $@) {
5205                     PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology);
5206                     die $err;
5207                 }
5208
5209                 PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology);
5210             };
5211             eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
5212
5213         } else {
5214             eval { $run_qemu->() };
5215         }
5216
5217         if (my $err = $@) {
5218             # deactivate volumes if start fails
5219             eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
5220             die "start failed: $err";
5221         }
5222
5223         print "migration listens on $migrate_uri\n" if $migrate_uri;
5224
5225         if ($statefile && $statefile ne 'tcp' && $statefile ne 'unix')  {
5226             eval { vm_mon_cmd_nocheck($vmid, "cont"); };
5227             warn $@ if $@;
5228         }
5229
5230         #start nbd server for storage migration
5231         if ($targetstorage) {
5232             my $nodename = PVE::INotify::nodename();
5233             my $migrate_network_addr = PVE::Cluster::get_local_migration_ip($migration_network);
5234             my $localip = $migrate_network_addr ? $migrate_network_addr : PVE::Cluster::remote_node_ip($nodename, 1);
5235             my $pfamily = PVE::Tools::get_host_address_family($nodename);
5236             $migrate_port = PVE::Tools::next_migrate_port($pfamily);
5237
5238             vm_mon_cmd_nocheck($vmid, "nbd-server-start", addr => { type => 'inet', data => { host => "${localip}", port => "${migrate_port}" } } );
5239
5240             $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5241
5242             foreach my $opt (sort keys %$local_volumes) {
5243                 my $volid = $local_volumes->{$opt};
5244                 vm_mon_cmd_nocheck($vmid, "nbd-server-add", device => "drive-$opt", writable => JSON::true );
5245                 my $migrate_storage_uri = "nbd:${localip}:${migrate_port}:exportname=drive-$opt";
5246                 print "storage migration listens on $migrate_storage_uri volume:$volid\n";
5247             }
5248         }
5249
5250         if ($migratedfrom) {
5251             eval {
5252                 set_migration_caps($vmid);
5253             };
5254             warn $@ if $@;
5255
5256             if ($spice_port) {
5257                 print "spice listens on port $spice_port\n";
5258                 if ($spice_ticket) {
5259                     vm_mon_cmd_nocheck($vmid, "set_password", protocol => 'spice', password => $spice_ticket);
5260                     vm_mon_cmd_nocheck($vmid, "expire_password", protocol => 'spice', time => "+30");
5261                 }
5262             }
5263
5264         } else {
5265             vm_mon_cmd_nocheck($vmid, "balloon", value => $conf->{balloon}*1024*1024)
5266                 if !$statefile && $conf->{balloon};
5267
5268             foreach my $opt (keys %$conf) {
5269                 next if $opt !~  m/^net\d+$/;
5270                 my $nicconf = parse_net($conf->{$opt});
5271                 qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
5272             }
5273         }
5274
5275         vm_mon_cmd_nocheck($vmid, 'qom-set',
5276                     path => "machine/peripheral/balloon0",
5277                     property => "guest-stats-polling-interval",
5278                     value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
5279
5280     });
5281 }
5282
5283 sub vm_mon_cmd {
5284     my ($vmid, $execute, %params) = @_;
5285
5286     my $cmd = { execute => $execute, arguments => \%params };
5287     vm_qmp_command($vmid, $cmd);
5288 }
5289
5290 sub vm_mon_cmd_nocheck {
5291     my ($vmid, $execute, %params) = @_;
5292
5293     my $cmd = { execute => $execute, arguments => \%params };
5294     vm_qmp_command($vmid, $cmd, 1);
5295 }
5296
5297 sub vm_qmp_command {
5298     my ($vmid, $cmd, $nocheck) = @_;
5299
5300     my $res;
5301
5302     my $timeout;
5303     if ($cmd->{arguments} && $cmd->{arguments}->{timeout}) {
5304         $timeout = $cmd->{arguments}->{timeout};
5305         delete $cmd->{arguments}->{timeout};
5306     }
5307
5308     eval {
5309         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
5310         my $sname = qmp_socket($vmid);
5311         if (-e $sname) { # test if VM is reasonambe new and supports qmp/qga
5312             my $qmpclient = PVE::QMPClient->new();
5313
5314             $res = $qmpclient->cmd($vmid, $cmd, $timeout);
5315         } else {
5316             die "unable to open monitor socket\n";
5317         }
5318     };
5319     if (my $err = $@) {
5320         syslog("err", "VM $vmid qmp command failed - $err");
5321         die $err;
5322     }
5323
5324     return $res;
5325 }
5326
5327 sub vm_human_monitor_command {
5328     my ($vmid, $cmdline) = @_;
5329
5330     my $res;
5331
5332     my $cmd = {
5333         execute => 'human-monitor-command',
5334         arguments => { 'command-line' => $cmdline},
5335     };
5336
5337     return vm_qmp_command($vmid, $cmd);
5338 }
5339
5340 sub vm_commandline {
5341     my ($storecfg, $vmid) = @_;
5342
5343     my $conf = PVE::QemuConfig->load_config($vmid);
5344
5345     my $defaults = load_defaults();
5346
5347     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults);
5348
5349     return PVE::Tools::cmd2string($cmd);
5350 }
5351
5352 sub vm_reset {
5353     my ($vmid, $skiplock) = @_;
5354
5355     PVE::QemuConfig->lock_config($vmid, sub {
5356
5357         my $conf = PVE::QemuConfig->load_config($vmid);
5358
5359         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5360
5361         vm_mon_cmd($vmid, "system_reset");
5362     });
5363 }
5364
5365 sub get_vm_volumes {
5366     my ($conf) = @_;
5367
5368     my $vollist = [];
5369     foreach_volid($conf, sub {
5370         my ($volid, $attr) = @_;
5371
5372         return if $volid =~ m|^/|;
5373
5374         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5375         return if !$sid;
5376
5377         push @$vollist, $volid;
5378     });
5379
5380     return $vollist;
5381 }
5382
5383 sub vm_stop_cleanup {
5384     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
5385
5386     eval {
5387
5388         if (!$keepActive) {
5389             my $vollist = get_vm_volumes($conf);
5390             PVE::Storage::deactivate_volumes($storecfg, $vollist);
5391         }
5392
5393         foreach my $ext (qw(mon qmp pid vnc qga)) {
5394             unlink "/var/run/qemu-server/${vmid}.$ext";
5395         }
5396
5397         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
5398     };
5399     warn $@ if $@; # avoid errors - just warn
5400 }
5401
5402 # Note: use $nockeck to skip tests if VM configuration file exists.
5403 # We need that when migration VMs to other nodes (files already moved)
5404 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
5405 sub vm_stop {
5406     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
5407
5408     $force = 1 if !defined($force) && !$shutdown;
5409
5410     if ($migratedfrom){
5411         my $pid = check_running($vmid, $nocheck, $migratedfrom);
5412         kill 15, $pid if $pid;
5413         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
5414         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
5415         return;
5416     }
5417
5418     PVE::QemuConfig->lock_config($vmid, sub {
5419
5420         my $pid = check_running($vmid, $nocheck);
5421         return if !$pid;
5422
5423         my $conf;
5424         if (!$nocheck) {
5425             $conf = PVE::QemuConfig->load_config($vmid);
5426             PVE::QemuConfig->check_lock($conf) if !$skiplock;
5427             if (!defined($timeout) && $shutdown && $conf->{startup}) {
5428                 my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
5429                 $timeout = $opts->{down} if $opts->{down};
5430             }
5431         }
5432
5433         $timeout = 60 if !defined($timeout);
5434
5435         eval {
5436             if ($shutdown) {
5437                 if (defined($conf) && parse_guest_agent($conf)->{enabled}) {
5438                     vm_qmp_command($vmid, { execute => "guest-shutdown" }, $nocheck);
5439                 } else {
5440                     vm_qmp_command($vmid, { execute => "system_powerdown" }, $nocheck);
5441                 }
5442             } else {
5443                 vm_qmp_command($vmid, { execute => "quit" }, $nocheck);
5444             }
5445         };
5446         my $err = $@;
5447
5448         if (!$err) {
5449             my $count = 0;
5450             while (($count < $timeout) && check_running($vmid, $nocheck)) {
5451                 $count++;
5452                 sleep 1;
5453             }
5454
5455             if ($count >= $timeout) {
5456                 if ($force) {
5457                     warn "VM still running - terminating now with SIGTERM\n";
5458                     kill 15, $pid;
5459                 } else {
5460                     die "VM quit/powerdown failed - got timeout\n";
5461                 }
5462             } else {
5463                 vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5464                 return;
5465             }
5466         } else {
5467             if ($force) {
5468                 warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
5469                 kill 15, $pid;
5470             } else {
5471                 die "VM quit/powerdown failed\n";
5472             }
5473         }
5474
5475         # wait again
5476         $timeout = 10;
5477
5478         my $count = 0;
5479         while (($count < $timeout) && check_running($vmid, $nocheck)) {
5480             $count++;
5481             sleep 1;
5482         }
5483
5484         if ($count >= $timeout) {
5485             warn "VM still running - terminating now with SIGKILL\n";
5486             kill 9, $pid;
5487             sleep 1;
5488         }
5489
5490         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5491    });
5492 }
5493
5494 sub vm_suspend {
5495     my ($vmid, $skiplock) = @_;
5496
5497     PVE::QemuConfig->lock_config($vmid, sub {
5498
5499         my $conf = PVE::QemuConfig->load_config($vmid);
5500
5501         PVE::QemuConfig->check_lock($conf)
5502             if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
5503
5504         vm_mon_cmd($vmid, "stop");
5505     });
5506 }
5507
5508 sub vm_resume {
5509     my ($vmid, $skiplock, $nocheck) = @_;
5510
5511     PVE::QemuConfig->lock_config($vmid, sub {
5512
5513         my $res = vm_mon_cmd($vmid, 'query-status');
5514         my $resume_cmd = 'cont';
5515
5516         if ($res->{status} && $res->{status} eq 'suspended') {
5517             $resume_cmd = 'system_wakeup';
5518         }
5519
5520         if (!$nocheck) {
5521
5522             my $conf = PVE::QemuConfig->load_config($vmid);
5523
5524             PVE::QemuConfig->check_lock($conf)
5525                 if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
5526
5527             vm_mon_cmd($vmid, $resume_cmd);
5528
5529         } else {
5530             vm_mon_cmd_nocheck($vmid, $resume_cmd);
5531         }
5532     });
5533 }
5534
5535 sub vm_sendkey {
5536     my ($vmid, $skiplock, $key) = @_;
5537
5538     PVE::QemuConfig->lock_config($vmid, sub {
5539
5540         my $conf = PVE::QemuConfig->load_config($vmid);
5541
5542         # there is no qmp command, so we use the human monitor command
5543         vm_human_monitor_command($vmid, "sendkey $key");
5544     });
5545 }
5546
5547 sub vm_destroy {
5548     my ($storecfg, $vmid, $skiplock) = @_;
5549
5550     PVE::QemuConfig->lock_config($vmid, sub {
5551
5552         my $conf = PVE::QemuConfig->load_config($vmid);
5553
5554         if (!check_running($vmid)) {
5555             destroy_vm($storecfg, $vmid, undef, $skiplock);
5556         } else {
5557             die "VM $vmid is running - destroy failed\n";
5558         }
5559     });
5560 }
5561
5562 # pci helpers
5563
5564 sub file_write {
5565     my ($filename, $buf) = @_;
5566
5567     my $fh = IO::File->new($filename, "w");
5568     return undef if !$fh;
5569
5570     my $res = print $fh $buf;
5571
5572     $fh->close();
5573
5574     return $res;
5575 }
5576
5577 sub pci_device_info {
5578     my ($name) = @_;
5579
5580     my $res;
5581
5582     return undef if $name !~ m/^([a-f0-9]{4}):([a-f0-9]{2}):([a-f0-9]{2})\.([a-f0-9])$/;
5583     my ($domain, $bus, $slot, $func) = ($1, $2, $3, $4);
5584
5585     my $irq = file_read_firstline("$pcisysfs/devices/$name/irq");
5586     return undef if !defined($irq) || $irq !~ m/^\d+$/;
5587
5588     my $vendor = file_read_firstline("$pcisysfs/devices/$name/vendor");
5589     return undef if !defined($vendor) || $vendor !~ s/^0x//;
5590
5591     my $product = file_read_firstline("$pcisysfs/devices/$name/device");
5592     return undef if !defined($product) || $product !~ s/^0x//;
5593
5594     $res = {
5595         name => $name,
5596         vendor => $vendor,
5597         product => $product,
5598         domain => $domain,
5599         bus => $bus,
5600         slot => $slot,
5601         func => $func,
5602         irq => $irq,
5603         has_fl_reset => -f "$pcisysfs/devices/$name/reset" || 0,
5604     };
5605
5606     return $res;
5607 }
5608
5609 sub pci_dev_reset {
5610     my ($dev) = @_;
5611
5612     my $name = $dev->{name};
5613
5614     my $fn = "$pcisysfs/devices/$name/reset";
5615
5616     return file_write($fn, "1");
5617 }
5618
5619 sub pci_dev_bind_to_vfio {
5620     my ($dev) = @_;
5621
5622     my $name = $dev->{name};
5623
5624     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
5625
5626     if (!-d $vfio_basedir) {
5627         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
5628     }
5629     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
5630
5631     my $testdir = "$vfio_basedir/$name";
5632     return 1 if -d $testdir;
5633
5634     my $data = "$dev->{vendor} $dev->{product}";
5635     return undef if !file_write("$vfio_basedir/new_id", $data);
5636
5637     my $fn = "$pcisysfs/devices/$name/driver/unbind";
5638     if (!file_write($fn, $name)) {
5639         return undef if -f $fn;
5640     }
5641
5642     $fn = "$vfio_basedir/bind";
5643     if (! -d $testdir) {
5644         return undef if !file_write($fn, $name);
5645     }
5646
5647     return -d $testdir;
5648 }
5649
5650 sub pci_dev_group_bind_to_vfio {
5651     my ($pciid) = @_;
5652
5653     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
5654
5655     if (!-d $vfio_basedir) {
5656         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
5657     }
5658     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
5659
5660     # get IOMMU group devices
5661     opendir(my $D, "$pcisysfs/devices/0000:$pciid/iommu_group/devices/") || die "Cannot open iommu_group: $!\n";
5662       my @devs = grep /^0000:/, readdir($D);
5663     closedir($D);
5664
5665     foreach my $pciid (@devs) {
5666         $pciid =~ m/^([:\.\da-f]+)$/ or die "PCI ID $pciid not valid!\n";
5667
5668         # pci bridges, switches or root ports are not supported
5669         # they have a pci_bus subdirectory so skip them
5670         next if (-e "$pcisysfs/devices/$pciid/pci_bus");
5671
5672         my $info = pci_device_info($1);
5673         pci_dev_bind_to_vfio($info) || die "Cannot bind $pciid to vfio\n";
5674     }
5675
5676     return 1;
5677 }
5678
5679 # vzdump restore implementaion
5680
5681 sub tar_archive_read_firstfile {
5682     my $archive = shift;
5683
5684     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
5685
5686     # try to detect archive type first
5687     my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
5688         die "unable to open file '$archive'\n";
5689     my $firstfile = <$fh>;
5690     kill 15, $pid;
5691     close $fh;
5692
5693     die "ERROR: archive contaions no data\n" if !$firstfile;
5694     chomp $firstfile;
5695
5696     return $firstfile;
5697 }
5698
5699 sub tar_restore_cleanup {
5700     my ($storecfg, $statfile) = @_;
5701
5702     print STDERR "starting cleanup\n";
5703
5704     if (my $fd = IO::File->new($statfile, "r")) {
5705         while (defined(my $line = <$fd>)) {
5706             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5707                 my $volid = $2;
5708                 eval {
5709                     if ($volid =~ m|^/|) {
5710                         unlink $volid || die 'unlink failed\n';
5711                     } else {
5712                         PVE::Storage::vdisk_free($storecfg, $volid);
5713                     }
5714                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
5715                 };
5716                 print STDERR "unable to cleanup '$volid' - $@" if $@;
5717             } else {
5718                 print STDERR "unable to parse line in statfile - $line";
5719             }
5720         }
5721         $fd->close();
5722     }
5723 }
5724
5725 sub restore_archive {
5726     my ($archive, $vmid, $user, $opts) = @_;
5727
5728     my $format = $opts->{format};
5729     my $comp;
5730
5731     if ($archive =~ m/\.tgz$/ || $archive =~ m/\.tar\.gz$/) {
5732         $format = 'tar' if !$format;
5733         $comp = 'gzip';
5734     } elsif ($archive =~ m/\.tar$/) {
5735         $format = 'tar' if !$format;
5736     } elsif ($archive =~ m/.tar.lzo$/) {
5737         $format = 'tar' if !$format;
5738         $comp = 'lzop';
5739     } elsif ($archive =~ m/\.vma$/) {
5740         $format = 'vma' if !$format;
5741     } elsif ($archive =~ m/\.vma\.gz$/) {
5742         $format = 'vma' if !$format;
5743         $comp = 'gzip';
5744     } elsif ($archive =~ m/\.vma\.lzo$/) {
5745         $format = 'vma' if !$format;
5746         $comp = 'lzop';
5747     } else {
5748         $format = 'vma' if !$format; # default
5749     }
5750
5751     # try to detect archive format
5752     if ($format eq 'tar') {
5753         return restore_tar_archive($archive, $vmid, $user, $opts);
5754     } else {
5755         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
5756     }
5757 }
5758
5759 sub restore_update_config_line {
5760     my ($outfd, $cookie, $vmid, $map, $line, $unique) = @_;
5761
5762     return if $line =~ m/^\#qmdump\#/;
5763     return if $line =~ m/^\#vzdump\#/;
5764     return if $line =~ m/^lock:/;
5765     return if $line =~ m/^unused\d+:/;
5766     return if $line =~ m/^parent:/;
5767     return if $line =~ m/^template:/; # restored VM is never a template
5768
5769     my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
5770     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
5771         # try to convert old 1.X settings
5772         my ($id, $ind, $ethcfg) = ($1, $2, $3);
5773         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
5774             my ($model, $macaddr) = split(/\=/, $devconfig);
5775             $macaddr = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if !$macaddr || $unique;
5776             my $net = {
5777                 model => $model,
5778                 bridge => "vmbr$ind",
5779                 macaddr => $macaddr,
5780             };
5781             my $netstr = print_net($net);
5782
5783             print $outfd "net$cookie->{netcount}: $netstr\n";
5784             $cookie->{netcount}++;
5785         }
5786     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
5787         my ($id, $netstr) = ($1, $2);
5788         my $net = parse_net($netstr);
5789         $net->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if $net->{macaddr};
5790         $netstr = print_net($net);
5791         print $outfd "$id: $netstr\n";
5792     } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk)\d+):\s*(\S+)\s*$/) {
5793         my $virtdev = $1;
5794         my $value = $3;
5795         my $di = parse_drive($virtdev, $value);
5796         if (defined($di->{backup}) && !$di->{backup}) {
5797             print $outfd "#$line";
5798         } elsif ($map->{$virtdev}) {
5799             delete $di->{format}; # format can change on restore
5800             $di->{file} = $map->{$virtdev};
5801             $value = print_drive($vmid, $di);
5802             print $outfd "$virtdev: $value\n";
5803         } else {
5804             print $outfd $line;
5805         }
5806     } elsif (($line =~ m/^vmgenid: (.*)/)) {
5807         my $vmgenid = $1;
5808         if ($vmgenid ne '0') {
5809             # always generate a new vmgenid if there was a valid one setup
5810             $vmgenid = generate_uuid();
5811         }
5812         print $outfd "vmgenid: $vmgenid\n";
5813     } elsif (($line =~ m/^(smbios1: )(.*)/) && $unique) {
5814         my ($uuid, $uuid_str);
5815         UUID::generate($uuid);
5816         UUID::unparse($uuid, $uuid_str);
5817         my $smbios1 = parse_smbios1($2);
5818         $smbios1->{uuid} = $uuid_str;
5819         print $outfd $1.print_smbios1($smbios1)."\n";
5820     } else {
5821         print $outfd $line;
5822     }
5823 }
5824
5825 sub scan_volids {
5826     my ($cfg, $vmid) = @_;
5827
5828     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid);
5829
5830     my $volid_hash = {};
5831     foreach my $storeid (keys %$info) {
5832         foreach my $item (@{$info->{$storeid}}) {
5833             next if !($item->{volid} && $item->{size});
5834             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
5835             $volid_hash->{$item->{volid}} = $item;
5836         }
5837     }
5838
5839     return $volid_hash;
5840 }
5841
5842 sub is_volume_in_use {
5843     my ($storecfg, $conf, $skip_drive, $volid) = @_;
5844
5845     my $path = PVE::Storage::path($storecfg, $volid);
5846
5847     my $scan_config = sub {
5848         my ($cref, $snapname) = @_;
5849
5850         foreach my $key (keys %$cref) {
5851             my $value = $cref->{$key};
5852             if (is_valid_drivename($key)) {
5853                 next if $skip_drive && $key eq $skip_drive;
5854                 my $drive = parse_drive($key, $value);
5855                 next if !$drive || !$drive->{file} || drive_is_cdrom($drive);
5856                 return 1 if $volid eq $drive->{file};
5857                 if ($drive->{file} =~ m!^/!) {
5858                     return 1 if $drive->{file} eq $path;
5859                 } else {
5860                     my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file}, 1);
5861                     next if !$storeid;
5862                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid, 1);
5863                     next if !$scfg;
5864                     return 1 if $path eq PVE::Storage::path($storecfg, $drive->{file}, $snapname);
5865                 }
5866             }
5867         }
5868
5869         return 0;
5870     };
5871
5872     return 1 if &$scan_config($conf);
5873
5874     undef $skip_drive;
5875
5876     foreach my $snapname (keys %{$conf->{snapshots}}) {
5877         return 1 if &$scan_config($conf->{snapshots}->{$snapname}, $snapname);
5878     }
5879
5880     return 0;
5881 }
5882
5883 sub update_disksize {
5884     my ($vmid, $conf, $volid_hash) = @_;
5885
5886     my $changes;
5887     my $prefix = "VM $vmid:";
5888
5889     # used and unused disks
5890     my $referenced = {};
5891
5892     # Note: it is allowed to define multiple storages with same path (alias), so
5893     # we need to check both 'volid' and real 'path' (two different volid can point
5894     # to the same path).
5895
5896     my $referencedpath = {};
5897
5898     # update size info
5899     foreach my $opt (keys %$conf) {
5900         if (is_valid_drivename($opt)) {
5901             my $drive = parse_drive($opt, $conf->{$opt});
5902             my $volid = $drive->{file};
5903             next if !$volid;
5904
5905             $referenced->{$volid} = 1;
5906             if ($volid_hash->{$volid} &&
5907                 (my $path = $volid_hash->{$volid}->{path})) {
5908                 $referencedpath->{$path} = 1;
5909             }
5910
5911             next if drive_is_cdrom($drive);
5912             next if !$volid_hash->{$volid};
5913
5914             $drive->{size} = $volid_hash->{$volid}->{size};
5915             my $new = print_drive($vmid, $drive);
5916             if ($new ne $conf->{$opt}) {
5917                 $changes = 1;
5918                 $conf->{$opt} = $new;
5919                 print "$prefix update disk '$opt' information.\n";
5920             }
5921         }
5922     }
5923
5924     # remove 'unusedX' entry if volume is used
5925     foreach my $opt (keys %$conf) {
5926         next if $opt !~ m/^unused\d+$/;
5927         my $volid = $conf->{$opt};
5928         my $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
5929         if ($referenced->{$volid} || ($path && $referencedpath->{$path})) {
5930             print "$prefix remove entry '$opt', its volume '$volid' is in use.\n";
5931             $changes = 1;
5932             delete $conf->{$opt};
5933         }
5934
5935         $referenced->{$volid} = 1;
5936         $referencedpath->{$path} = 1 if $path;
5937     }
5938
5939     foreach my $volid (sort keys %$volid_hash) {
5940         next if $volid =~ m/vm-$vmid-state-/;
5941         next if $referenced->{$volid};
5942         my $path = $volid_hash->{$volid}->{path};
5943         next if !$path; # just to be sure
5944         next if $referencedpath->{$path};
5945         $changes = 1;
5946         my $key = PVE::QemuConfig->add_unused_volume($conf, $volid);
5947         print "$prefix add unreferenced volume '$volid' as '$key' to config.\n";
5948         $referencedpath->{$path} = 1; # avoid to add more than once (aliases)
5949     }
5950
5951     return $changes;
5952 }
5953
5954 sub rescan {
5955     my ($vmid, $nolock, $dryrun) = @_;
5956
5957     my $cfg = PVE::Storage::config();
5958
5959     # FIXME: Remove once our RBD plugin can handle CT and VM on a single storage
5960     # see: https://pve.proxmox.com/pipermail/pve-devel/2018-July/032900.html
5961     foreach my $stor (keys %{$cfg->{ids}}) {
5962         delete($cfg->{ids}->{$stor}) if ! $cfg->{ids}->{$stor}->{content}->{images};
5963     }
5964
5965     print "rescan volumes...\n";
5966     my $volid_hash = scan_volids($cfg, $vmid);
5967
5968     my $updatefn =  sub {
5969         my ($vmid) = @_;
5970
5971         my $conf = PVE::QemuConfig->load_config($vmid);
5972
5973         PVE::QemuConfig->check_lock($conf);
5974
5975         my $vm_volids = {};
5976         foreach my $volid (keys %$volid_hash) {
5977             my $info = $volid_hash->{$volid};
5978             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
5979         }
5980
5981         my $changes = update_disksize($vmid, $conf, $vm_volids);
5982
5983         PVE::QemuConfig->write_config($vmid, $conf) if $changes && !$dryrun;
5984     };
5985
5986     if (defined($vmid)) {
5987         if ($nolock) {
5988             &$updatefn($vmid);
5989         } else {
5990             PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
5991         }
5992     } else {
5993         my $vmlist = config_list();
5994         foreach my $vmid (keys %$vmlist) {
5995             if ($nolock) {
5996                 &$updatefn($vmid);
5997             } else {
5998                 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
5999             }
6000         }
6001     }
6002 }
6003
6004 sub restore_vma_archive {
6005     my ($archive, $vmid, $user, $opts, $comp) = @_;
6006
6007     my $readfrom = $archive;
6008
6009     my $cfg = PVE::Storage::config();
6010     my $commands = [];
6011     my $bwlimit = $opts->{bwlimit};
6012
6013     my $dbg_cmdstring = '';
6014     my $add_pipe = sub {
6015         my ($cmd) = @_;
6016         push @$commands, $cmd;
6017         $dbg_cmdstring .= ' | ' if length($dbg_cmdstring);
6018         $dbg_cmdstring .= PVE::Tools::cmd2string($cmd);
6019         $readfrom = '-';
6020     };
6021
6022     my $input = undef;
6023     if ($archive eq '-') {
6024         $input = '<&STDIN';
6025     } else {
6026         # If we use a backup from a PVE defined storage we also consider that
6027         # storage's rate limit:
6028         my (undef, $volid) = PVE::Storage::path_to_volume_id($cfg, $archive);
6029         if (defined($volid)) {
6030             my ($sid, undef) = PVE::Storage::parse_volume_id($volid);
6031             my $readlimit = PVE::Storage::get_bandwidth_limit('restore', [$sid], $bwlimit);
6032             if ($readlimit) {
6033                 print STDERR "applying read rate limit: $readlimit\n";
6034                 my $cstream = ['cstream', '-t', $readlimit*1024, '--', $readfrom];
6035                 $add_pipe->($cstream);
6036             }
6037         }
6038     }
6039
6040     if ($comp) {
6041         my $cmd;
6042         if ($comp eq 'gzip') {
6043             $cmd = ['zcat', $readfrom];
6044         } elsif ($comp eq 'lzop') {
6045             $cmd = ['lzop', '-d', '-c', $readfrom];
6046         } else {
6047             die "unknown compression method '$comp'\n";
6048         }
6049         $add_pipe->($cmd);
6050     }
6051
6052     my $tmpdir = "/var/tmp/vzdumptmp$$";
6053     rmtree $tmpdir;
6054
6055     # disable interrupts (always do cleanups)
6056     local $SIG{INT} =
6057         local $SIG{TERM} =
6058         local $SIG{QUIT} =
6059         local $SIG{HUP} = sub { warn "got interrupt - ignored\n"; };
6060
6061     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
6062     POSIX::mkfifo($mapfifo, 0600);
6063     my $fifofh;
6064
6065     my $openfifo = sub {
6066         open($fifofh, '>', $mapfifo) || die $!;
6067     };
6068
6069     $add_pipe->(['vma', 'extract', '-v', '-r', $mapfifo, $readfrom, $tmpdir]);
6070
6071     my $oldtimeout;
6072     my $timeout = 5;
6073
6074     my $devinfo = {};
6075
6076     my $rpcenv = PVE::RPCEnvironment::get();
6077
6078     my $conffile = PVE::QemuConfig->config_file($vmid);
6079     my $tmpfn = "$conffile.$$.tmp";
6080
6081     # Note: $oldconf is undef if VM does not exists
6082     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
6083     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
6084
6085     my %storage_limits;
6086
6087     my $print_devmap = sub {
6088         my $virtdev_hash = {};
6089
6090         my $cfgfn = "$tmpdir/qemu-server.conf";
6091
6092         # we can read the config - that is already extracted
6093         my $fh = IO::File->new($cfgfn, "r") ||
6094             "unable to read qemu-server.conf - $!\n";
6095
6096         my $fwcfgfn = "$tmpdir/qemu-server.fw";
6097         if (-f $fwcfgfn) {
6098             my $pve_firewall_dir = '/etc/pve/firewall';
6099             mkdir $pve_firewall_dir; # make sure the dir exists
6100             PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
6101         }
6102
6103         while (defined(my $line = <$fh>)) {
6104             if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
6105                 my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
6106                 die "archive does not contain data for drive '$virtdev'\n"
6107                     if !$devinfo->{$devname};
6108                 if (defined($opts->{storage})) {
6109                     $storeid = $opts->{storage} || 'local';
6110                 } elsif (!$storeid) {
6111                     $storeid = 'local';
6112                 }
6113                 $format = 'raw' if !$format;
6114                 $devinfo->{$devname}->{devname} = $devname;
6115                 $devinfo->{$devname}->{virtdev} = $virtdev;
6116                 $devinfo->{$devname}->{format} = $format;
6117                 $devinfo->{$devname}->{storeid} = $storeid;
6118
6119                 # check permission on storage
6120                 my $pool = $opts->{pool}; # todo: do we need that?
6121                 if ($user ne 'root@pam') {
6122                     $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
6123                 }
6124
6125                 $storage_limits{$storeid} = $bwlimit;
6126
6127                 $virtdev_hash->{$virtdev} = $devinfo->{$devname};
6128             }
6129         }
6130
6131         foreach my $key (keys %storage_limits) {
6132             my $limit = PVE::Storage::get_bandwidth_limit('restore', [$key], $bwlimit);
6133             next if !$limit;
6134             print STDERR "rate limit for storage $key: $limit KiB/s\n";
6135             $storage_limits{$key} = $limit * 1024;
6136         }
6137
6138         foreach my $devname (keys %$devinfo) {
6139             die "found no device mapping information for device '$devname'\n"
6140                 if !$devinfo->{$devname}->{virtdev};
6141         }
6142
6143         # create empty/temp config
6144         if ($oldconf) {
6145             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
6146             foreach_drive($oldconf, sub {
6147                 my ($ds, $drive) = @_;
6148
6149                 return if drive_is_cdrom($drive);
6150
6151                 my $volid = $drive->{file};
6152
6153                 return if !$volid || $volid =~ m|^/|;
6154
6155                 my ($path, $owner) = PVE::Storage::path($cfg, $volid);
6156                 return if !$path || !$owner || ($owner != $vmid);
6157
6158                 # Note: only delete disk we want to restore
6159                 # other volumes will become unused
6160                 if ($virtdev_hash->{$ds}) {
6161                     eval { PVE::Storage::vdisk_free($cfg, $volid); };
6162                     if (my $err = $@) {
6163                         warn $err;
6164                     }
6165                 }
6166             });
6167
6168             # delete vmstate files
6169             # since after the restore we have no snapshots anymore
6170             foreach my $snapname (keys %{$oldconf->{snapshots}}) {
6171                 my $snap = $oldconf->{snapshots}->{$snapname};
6172                 if ($snap->{vmstate}) {
6173                     eval { PVE::Storage::vdisk_free($cfg, $snap->{vmstate}); };
6174                     if (my $err = $@) {
6175                         warn $err;
6176                     }
6177                 }
6178             }
6179         }
6180
6181         my $map = {};
6182         foreach my $virtdev (sort keys %$virtdev_hash) {
6183             my $d = $virtdev_hash->{$virtdev};
6184             my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
6185             my $storeid = $d->{storeid};
6186             my $scfg = PVE::Storage::storage_config($cfg, $storeid);
6187
6188             my $map_opts = '';
6189             if (my $limit = $storage_limits{$storeid}) {
6190                 $map_opts .= "throttling.bps=$limit:throttling.group=$storeid:";
6191             }
6192
6193             # test if requested format is supported
6194             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($cfg, $storeid);
6195             my $supported = grep { $_ eq $d->{format} } @$validFormats;
6196             $d->{format} = $defFormat if !$supported;
6197
6198             my $volid = PVE::Storage::vdisk_alloc($cfg, $storeid, $vmid,
6199                                                   $d->{format}, undef, $alloc_size);
6200             print STDERR "new volume ID is '$volid'\n";
6201             $d->{volid} = $volid;
6202             my $path = PVE::Storage::path($cfg, $volid);
6203
6204             PVE::Storage::activate_volumes($cfg,[$volid]);
6205
6206             my $write_zeros = 1;
6207             if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
6208                 $write_zeros = 0;
6209             }
6210
6211             print $fifofh "${map_opts}format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
6212
6213             print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
6214             $map->{$virtdev} = $volid;
6215         }
6216
6217         $fh->seek(0, 0) || die "seek failed - $!\n";
6218
6219         my $outfd = new IO::File ($tmpfn, "w") ||
6220             die "unable to write config for VM $vmid\n";
6221
6222         my $cookie = { netcount => 0 };
6223         while (defined(my $line = <$fh>)) {
6224             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
6225         }
6226
6227         $fh->close();
6228         $outfd->close();
6229     };
6230
6231     eval {
6232         # enable interrupts
6233         local $SIG{INT} =
6234             local $SIG{TERM} =
6235             local $SIG{QUIT} =
6236             local $SIG{HUP} =
6237             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6238         local $SIG{ALRM} = sub { die "got timeout\n"; };
6239
6240         $oldtimeout = alarm($timeout);
6241
6242         my $parser = sub {
6243             my $line = shift;
6244
6245             print "$line\n";
6246
6247             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
6248                 my ($dev_id, $size, $devname) = ($1, $2, $3);
6249                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
6250             } elsif ($line =~ m/^CTIME: /) {
6251                 # we correctly received the vma config, so we can disable
6252                 # the timeout now for disk allocation (set to 10 minutes, so
6253                 # that we always timeout if something goes wrong)
6254                 alarm(600);
6255                 &$print_devmap();
6256                 print $fifofh "done\n";
6257                 my $tmp = $oldtimeout || 0;
6258                 $oldtimeout = undef;
6259                 alarm($tmp);
6260                 close($fifofh);
6261             }
6262         };
6263
6264         print "restore vma archive: $dbg_cmdstring\n";
6265         run_command($commands, input => $input, outfunc => $parser, afterfork => $openfifo);
6266     };
6267     my $err = $@;
6268
6269     alarm($oldtimeout) if $oldtimeout;
6270
6271     my $vollist = [];
6272     foreach my $devname (keys %$devinfo) {
6273         my $volid = $devinfo->{$devname}->{volid};
6274         push @$vollist, $volid if $volid;
6275     }
6276
6277     PVE::Storage::deactivate_volumes($cfg, $vollist);
6278
6279     unlink $mapfifo;
6280
6281     if ($err) {
6282         rmtree $tmpdir;
6283         unlink $tmpfn;
6284
6285         foreach my $devname (keys %$devinfo) {
6286             my $volid = $devinfo->{$devname}->{volid};
6287             next if !$volid;
6288             eval {
6289                 if ($volid =~ m|^/|) {
6290                     unlink $volid || die 'unlink failed\n';
6291                 } else {
6292                     PVE::Storage::vdisk_free($cfg, $volid);
6293                 }
6294                 print STDERR "temporary volume '$volid' sucessfuly removed\n";
6295             };
6296             print STDERR "unable to cleanup '$volid' - $@" if $@;
6297         }
6298         die $err;
6299     }
6300
6301     rmtree $tmpdir;
6302
6303     rename($tmpfn, $conffile) ||
6304         die "unable to commit configuration file '$conffile'\n";
6305
6306     PVE::Cluster::cfs_update(); # make sure we read new file
6307
6308     eval { rescan($vmid, 1); };
6309     warn $@ if $@;
6310 }
6311
6312 sub restore_tar_archive {
6313     my ($archive, $vmid, $user, $opts) = @_;
6314
6315     if ($archive ne '-') {
6316         my $firstfile = tar_archive_read_firstfile($archive);
6317         die "ERROR: file '$archive' dos not lock like a QemuServer vzdump backup\n"
6318             if $firstfile ne 'qemu-server.conf';
6319     }
6320
6321     my $storecfg = PVE::Storage::config();
6322
6323     # destroy existing data - keep empty config
6324     my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
6325     destroy_vm($storecfg, $vmid, 1) if -f $vmcfgfn;
6326
6327     my $tocmd = "/usr/lib/qemu-server/qmextract";
6328
6329     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
6330     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
6331     $tocmd .= ' --prealloc' if $opts->{prealloc};
6332     $tocmd .= ' --info' if $opts->{info};
6333
6334     # tar option "xf" does not autodetect compression when read from STDIN,
6335     # so we pipe to zcat
6336     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
6337         PVE::Tools::shellquote("--to-command=$tocmd");
6338
6339     my $tmpdir = "/var/tmp/vzdumptmp$$";
6340     mkpath $tmpdir;
6341
6342     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
6343     local $ENV{VZDUMP_VMID} = $vmid;
6344     local $ENV{VZDUMP_USER} = $user;
6345
6346     my $conffile = PVE::QemuConfig->config_file($vmid);
6347     my $tmpfn = "$conffile.$$.tmp";
6348
6349     # disable interrupts (always do cleanups)
6350     local $SIG{INT} =
6351         local $SIG{TERM} =
6352         local $SIG{QUIT} =
6353         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
6354
6355     eval {
6356         # enable interrupts
6357         local $SIG{INT} =
6358             local $SIG{TERM} =
6359             local $SIG{QUIT} =
6360             local $SIG{HUP} =
6361             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6362
6363         if ($archive eq '-') {
6364             print "extracting archive from STDIN\n";
6365             run_command($cmd, input => "<&STDIN");
6366         } else {
6367             print "extracting archive '$archive'\n";
6368             run_command($cmd);
6369         }
6370
6371         return if $opts->{info};
6372
6373         # read new mapping
6374         my $map = {};
6375         my $statfile = "$tmpdir/qmrestore.stat";
6376         if (my $fd = IO::File->new($statfile, "r")) {
6377             while (defined (my $line = <$fd>)) {
6378                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
6379                     $map->{$1} = $2 if $1;
6380                 } else {
6381                     print STDERR "unable to parse line in statfile - $line\n";
6382                 }
6383             }
6384             $fd->close();
6385         }
6386
6387         my $confsrc = "$tmpdir/qemu-server.conf";
6388
6389         my $srcfd = new IO::File($confsrc, "r") ||
6390             die "unable to open file '$confsrc'\n";
6391
6392         my $outfd = new IO::File ($tmpfn, "w") ||
6393             die "unable to write config for VM $vmid\n";
6394
6395         my $cookie = { netcount => 0 };
6396         while (defined (my $line = <$srcfd>)) {
6397             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
6398         }
6399
6400         $srcfd->close();
6401         $outfd->close();
6402     };
6403     my $err = $@;
6404
6405     if ($err) {
6406
6407         unlink $tmpfn;
6408
6409         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
6410
6411         die $err;
6412     }
6413
6414     rmtree $tmpdir;
6415
6416     rename $tmpfn, $conffile ||
6417         die "unable to commit configuration file '$conffile'\n";
6418
6419     PVE::Cluster::cfs_update(); # make sure we read new file
6420
6421     eval { rescan($vmid, 1); };
6422     warn $@ if $@;
6423 };
6424
6425 sub foreach_storage_used_by_vm {
6426     my ($conf, $func) = @_;
6427
6428     my $sidhash = {};
6429
6430     foreach_drive($conf, sub {
6431         my ($ds, $drive) = @_;
6432         return if drive_is_cdrom($drive);
6433
6434         my $volid = $drive->{file};
6435
6436         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
6437         $sidhash->{$sid} = $sid if $sid;
6438     });
6439
6440     foreach my $sid (sort keys %$sidhash) {
6441         &$func($sid);
6442     }
6443 }
6444
6445 sub do_snapshots_with_qemu {
6446     my ($storecfg, $volid) = @_;
6447
6448     my $storage_name = PVE::Storage::parse_volume_id($volid);
6449
6450     if ($qemu_snap_storage->{$storecfg->{ids}->{$storage_name}->{type}}
6451         && !$storecfg->{ids}->{$storage_name}->{krbd}){
6452         return 1;
6453     }
6454
6455     if ($volid =~ m/\.(qcow2|qed)$/){
6456         return 1;
6457     }
6458
6459     return undef;
6460 }
6461
6462 sub qga_check_running {
6463     my ($vmid, $nowarn) = @_;
6464
6465     eval { vm_mon_cmd($vmid, "guest-ping", timeout => 3); };
6466     if ($@) {
6467         warn "Qemu Guest Agent is not running - $@" if !$nowarn;
6468         return 0;
6469     }
6470     return 1;
6471 }
6472
6473 sub template_create {
6474     my ($vmid, $conf, $disk) = @_;
6475
6476     my $storecfg = PVE::Storage::config();
6477
6478     foreach_drive($conf, sub {
6479         my ($ds, $drive) = @_;
6480
6481         return if drive_is_cdrom($drive);
6482         return if $disk && $ds ne $disk;
6483
6484         my $volid = $drive->{file};
6485         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
6486
6487         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
6488         $drive->{file} = $voliddst;
6489         $conf->{$ds} = print_drive($vmid, $drive);
6490         PVE::QemuConfig->write_config($vmid, $conf);
6491     });
6492 }
6493
6494 sub qemu_img_convert {
6495     my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized) = @_;
6496
6497     my $storecfg = PVE::Storage::config();
6498     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
6499     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
6500
6501     if ($src_storeid && $dst_storeid) {
6502
6503         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
6504
6505         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
6506         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6507
6508         my $src_format = qemu_img_format($src_scfg, $src_volname);
6509         my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
6510
6511         my $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
6512         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6513
6514         my $cmd = [];
6515         push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
6516         push @$cmd, '-l', "snapshot.name=$snapname" if($snapname && $src_format eq "qcow2");
6517         push @$cmd, '-t', 'none' if $dst_scfg->{type} eq 'zfspool';
6518         push @$cmd, '-T', 'none' if $src_scfg->{type} eq 'zfspool';
6519         push @$cmd, '-f', $src_format, '-O', $dst_format, $src_path;
6520         if ($is_zero_initialized) {
6521             push @$cmd, "zeroinit:$dst_path";
6522         } else {
6523             push @$cmd, $dst_path;
6524         }
6525
6526         my $parser = sub {
6527             my $line = shift;
6528             if($line =~ m/\((\S+)\/100\%\)/){
6529                 my $percent = $1;
6530                 my $transferred = int($size * $percent / 100);
6531                 my $remaining = $size - $transferred;
6532
6533                 print "transferred: $transferred bytes remaining: $remaining bytes total: $size bytes progression: $percent %\n";
6534             }
6535
6536         };
6537
6538         eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
6539         my $err = $@;
6540         die "copy failed: $err" if $err;
6541     }
6542 }
6543
6544 sub qemu_img_format {
6545     my ($scfg, $volname) = @_;
6546
6547     if ($scfg->{path} && $volname =~ m/\.($QEMU_FORMAT_RE)$/) {
6548         return $1;
6549     } else {
6550         return "raw";
6551     }
6552 }
6553
6554 sub qemu_drive_mirror {
6555     my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $skipcomplete, $qga) = @_;
6556
6557     $jobs = {} if !$jobs;
6558
6559     my $qemu_target;
6560     my $format;
6561     $jobs->{"drive-$drive"} = {};
6562
6563     if ($dst_volid =~ /^nbd:/) {
6564         $qemu_target = $dst_volid;
6565         $format = "nbd";
6566     } else {
6567         my $storecfg = PVE::Storage::config();
6568         my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
6569
6570         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6571
6572         $format = qemu_img_format($dst_scfg, $dst_volname);
6573
6574         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6575
6576         $qemu_target = $is_zero_initialized ? "zeroinit:$dst_path" : $dst_path;
6577     }
6578
6579     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $qemu_target };
6580     $opts->{format} = $format if $format;
6581
6582     print "drive mirror is starting for drive-$drive\n";
6583
6584     eval { vm_mon_cmd($vmid, "drive-mirror", %$opts); }; #if a job already run for this device,it's throw an error
6585
6586     if (my $err = $@) {
6587         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6588         die "mirroring error: $err";
6589     }
6590
6591     qemu_drive_mirror_monitor ($vmid, $vmiddst, $jobs, $skipcomplete, $qga);
6592 }
6593
6594 sub qemu_drive_mirror_monitor {
6595     my ($vmid, $vmiddst, $jobs, $skipcomplete, $qga) = @_;
6596
6597     eval {
6598         my $err_complete = 0;
6599
6600         while (1) {
6601             die "storage migration timed out\n" if $err_complete > 300;
6602
6603             my $stats = vm_mon_cmd($vmid, "query-block-jobs");
6604
6605             my $running_mirror_jobs = {};
6606             foreach my $stat (@$stats) {
6607                 next if $stat->{type} ne 'mirror';
6608                 $running_mirror_jobs->{$stat->{device}} = $stat;
6609             }
6610
6611             my $readycounter = 0;
6612
6613             foreach my $job (keys %$jobs) {
6614
6615                 if(defined($jobs->{$job}->{complete}) && !defined($running_mirror_jobs->{$job})) {
6616                     print "$job : finished\n";
6617                     delete $jobs->{$job};
6618                     next;
6619                 }
6620
6621                 die "$job: mirroring has been cancelled\n" if !defined($running_mirror_jobs->{$job});
6622
6623                 my $busy = $running_mirror_jobs->{$job}->{busy};
6624                 my $ready = $running_mirror_jobs->{$job}->{ready};
6625                 if (my $total = $running_mirror_jobs->{$job}->{len}) {
6626                     my $transferred = $running_mirror_jobs->{$job}->{offset} || 0;
6627                     my $remaining = $total - $transferred;
6628                     my $percent = sprintf "%.2f", ($transferred * 100 / $total);
6629
6630                     print "$job: transferred: $transferred bytes remaining: $remaining bytes total: $total bytes progression: $percent % busy: $busy ready: $ready \n";
6631                 }
6632
6633                 $readycounter++ if $running_mirror_jobs->{$job}->{ready};
6634             }
6635
6636             last if scalar(keys %$jobs) == 0;
6637
6638             if ($readycounter == scalar(keys %$jobs)) {
6639                 print "all mirroring jobs are ready \n";
6640                 last if $skipcomplete; #do the complete later
6641
6642                 if ($vmiddst && $vmiddst != $vmid) {
6643                     my $agent_running = $qga && qga_check_running($vmid);
6644                     if ($agent_running) {
6645                         print "freeze filesystem\n";
6646                         eval { PVE::QemuServer::vm_mon_cmd($vmid, "guest-fsfreeze-freeze"); };
6647                     } else {
6648                         print "suspend vm\n";
6649                         eval { PVE::QemuServer::vm_suspend($vmid, 1); };
6650                     }
6651
6652                     # if we clone a disk for a new target vm, we don't switch the disk
6653                     PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs);
6654
6655                     if ($agent_running) {
6656                         print "unfreeze filesystem\n";
6657                         eval { PVE::QemuServer::vm_mon_cmd($vmid, "guest-fsfreeze-thaw"); };
6658                     } else {
6659                         print "resume vm\n";
6660                         eval {  PVE::QemuServer::vm_resume($vmid, 1, 1); };
6661                     }
6662
6663                     last;
6664                 } else {
6665
6666                     foreach my $job (keys %$jobs) {
6667                         # try to switch the disk if source and destination are on the same guest
6668                         print "$job: Completing block job...\n";
6669
6670                         eval { vm_mon_cmd($vmid, "block-job-complete", device => $job) };
6671                         if ($@ =~ m/cannot be completed/) {
6672                             print "$job: Block job cannot be completed, try again.\n";
6673                             $err_complete++;
6674                         }else {
6675                             print "$job: Completed successfully.\n";
6676                             $jobs->{$job}->{complete} = 1;
6677                         }
6678                     }
6679                 }
6680             }
6681             sleep 1;
6682         }
6683     };
6684     my $err = $@;
6685
6686     if ($err) {
6687         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6688         die "mirroring error: $err";
6689     }
6690
6691 }
6692
6693 sub qemu_blockjobs_cancel {
6694     my ($vmid, $jobs) = @_;
6695
6696     foreach my $job (keys %$jobs) {
6697         print "$job: Cancelling block job\n";
6698         eval { vm_mon_cmd($vmid, "block-job-cancel", device => $job); };
6699         $jobs->{$job}->{cancel} = 1;
6700     }
6701
6702     while (1) {
6703         my $stats = vm_mon_cmd($vmid, "query-block-jobs");
6704
6705         my $running_jobs = {};
6706         foreach my $stat (@$stats) {
6707             $running_jobs->{$stat->{device}} = $stat;
6708         }
6709
6710         foreach my $job (keys %$jobs) {
6711
6712             if (defined($jobs->{$job}->{cancel}) && !defined($running_jobs->{$job})) {
6713                 print "$job: Done.\n";
6714                 delete $jobs->{$job};
6715             }
6716         }
6717
6718         last if scalar(keys %$jobs) == 0;
6719
6720         sleep 1;
6721     }
6722 }
6723
6724 sub clone_disk {
6725     my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
6726         $newvmid, $storage, $format, $full, $newvollist, $jobs, $skipcomplete, $qga) = @_;
6727
6728     my $newvolid;
6729
6730     if (!$full) {
6731         print "create linked clone of drive $drivename ($drive->{file})\n";
6732         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
6733         push @$newvollist, $newvolid;
6734     } else {
6735
6736         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6737         $storeid = $storage if $storage;
6738
6739         my $dst_format = resolve_dst_disk_format($storecfg, $storeid, $volname, $format);
6740         my ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 3);
6741
6742         print "create full clone of drive $drivename ($drive->{file})\n";
6743         my $name = undef;
6744         if (drive_is_cloudinit($drive)) {
6745             $name = "vm-$newvmid-cloudinit";
6746             # cloudinit only supports raw and qcow2 atm:
6747             if ($dst_format eq 'qcow2') {
6748                 $name .= '.qcow2';
6749             } elsif ($dst_format ne 'raw') {
6750                 die "clone: unhandled format for cloudinit image\n";
6751             }
6752         }
6753         $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $newvmid, $dst_format, $name, ($size/1024));
6754         push @$newvollist, $newvolid;
6755
6756         PVE::Storage::activate_volumes($storecfg, [$newvolid]);
6757
6758         my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
6759         if (!$running || $snapname) {
6760             qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit);
6761         } else {
6762
6763             my $kvmver = get_running_qemu_version ($vmid);
6764             if (!qemu_machine_feature_enabled (undef, $kvmver, 2, 7)) {
6765                 die "drive-mirror with iothread requires qemu version 2.7 or higher\n"
6766                     if $drive->{iothread};
6767             }
6768
6769             qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid, $sparseinit, $jobs, $skipcomplete, $qga);
6770         }
6771     }
6772
6773     my ($size) = PVE::Storage::volume_size_info($storecfg, $newvolid, 3);
6774
6775     my $disk = $drive;
6776     $disk->{format} = undef;
6777     $disk->{file} = $newvolid;
6778     $disk->{size} = $size;
6779
6780     return $disk;
6781 }
6782
6783 # this only works if VM is running
6784 sub get_current_qemu_machine {
6785     my ($vmid) = @_;
6786
6787     my $cmd = { execute => 'query-machines', arguments => {} };
6788     my $res = vm_qmp_command($vmid, $cmd);
6789
6790     my ($current, $default);
6791     foreach my $e (@$res) {
6792         $default = $e->{name} if $e->{'is-default'};
6793         $current = $e->{name} if $e->{'is-current'};
6794     }
6795
6796     # fallback to the default machine if current is not supported by qemu
6797     return $current || $default || 'pc';
6798 }
6799
6800 sub get_running_qemu_version {
6801     my ($vmid) = @_;
6802     my $cmd = { execute => 'query-version', arguments => {} };
6803     my $res = vm_qmp_command($vmid, $cmd);
6804     return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
6805 }
6806
6807 sub qemu_machine_feature_enabled {
6808     my ($machine, $kvmver, $version_major, $version_minor) = @_;
6809
6810     my $current_major;
6811     my $current_minor;
6812
6813     if ($machine && $machine =~ m/^((?:pc(-i440fx|-q35)?|virt)-(\d+)\.(\d+))/) {
6814
6815         $current_major = $3;
6816         $current_minor = $4;
6817
6818     } elsif ($kvmver =~ m/^(\d+)\.(\d+)/) {
6819
6820         $current_major = $1;
6821         $current_minor = $2;
6822     }
6823
6824     return 1 if $current_major > $version_major ||
6825                 ($current_major == $version_major &&
6826                  $current_minor >= $version_minor);
6827 }
6828
6829 sub qemu_machine_pxe {
6830     my ($vmid, $conf, $machine) = @_;
6831
6832     $machine =  PVE::QemuServer::get_current_qemu_machine($vmid) if !$machine;
6833
6834     if ($conf->{machine} && $conf->{machine} =~ m/\.pxe$/) {
6835         $machine .= '.pxe';
6836     }
6837
6838     return $machine;
6839 }
6840
6841 sub qemu_use_old_bios_files {
6842     my ($machine_type) = @_;
6843
6844     return if !$machine_type;
6845
6846     my $use_old_bios_files = undef;
6847
6848     if ($machine_type =~ m/^(\S+)\.pxe$/) {
6849         $machine_type = $1;
6850         $use_old_bios_files = 1;
6851     } else {
6852         my $kvmver = kvm_user_version();
6853         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
6854         # load new efi bios files on migration. So this hack is required to allow
6855         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
6856         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
6857         $use_old_bios_files = !qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 4);
6858     }
6859
6860     return ($use_old_bios_files, $machine_type);
6861 }
6862
6863 sub create_efidisk($$$$$) {
6864     my ($storecfg, $storeid, $vmid, $fmt, $arch) = @_;
6865
6866     my (undef, $ovmf_vars) = get_ovmf_files($arch);
6867     die "EFI vars default image not found\n" if ! -f $ovmf_vars;
6868
6869     my $vars_size = PVE::Tools::convert_size(-s $ovmf_vars, 'b' => 'kb');
6870     my $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $fmt, undef, $vars_size);
6871     PVE::Storage::activate_volumes($storecfg, [$volid]);
6872
6873     my $path = PVE::Storage::path($storecfg, $volid);
6874     eval {
6875         run_command(['/usr/bin/qemu-img', 'convert', '-n', '-f', 'raw', '-O', $fmt, $ovmf_vars, $path]);
6876     };
6877     die "Copying EFI vars image failed: $@" if $@;
6878
6879     return ($volid, $vars_size);
6880 }
6881
6882 sub lspci {
6883
6884     my $devices = {};
6885
6886     dir_glob_foreach("$pcisysfs/devices", '[a-f0-9]{4}:([a-f0-9]{2}:[a-f0-9]{2})\.([0-9])', sub {
6887             my (undef, $id, $function) = @_;
6888             my $res = { id => $id, function => $function};
6889             push @{$devices->{$id}}, $res;
6890     });
6891
6892     # Entries should be sorted by functions.
6893     foreach my $id (keys %$devices) {
6894         my $dev = $devices->{$id};
6895         $devices->{$id} = [ sort { $a->{function} <=> $b->{function} } @$dev ];
6896     }
6897
6898     return $devices;
6899 }
6900
6901 sub vm_iothreads_list {
6902     my ($vmid) = @_;
6903
6904     my $res = vm_mon_cmd($vmid, 'query-iothreads');
6905
6906     my $iothreads = {};
6907     foreach my $iothread (@$res) {
6908         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
6909     }
6910
6911     return $iothreads;
6912 }
6913
6914 sub scsihw_infos {
6915     my ($conf, $drive) = @_;
6916
6917     my $maxdev = 0;
6918
6919     if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)) {
6920         $maxdev = 7;
6921     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
6922         $maxdev = 1;
6923     } else {
6924         $maxdev = 256;
6925     }
6926
6927     my $controller = int($drive->{index} / $maxdev);
6928     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single') ? "virtioscsi" : "scsihw";
6929
6930     return ($maxdev, $controller, $controller_prefix);
6931 }
6932
6933 sub add_hyperv_enlightenments {
6934     my ($cpuFlags, $winversion, $machine_type, $kvmver, $bios, $gpu_passthrough) = @_;
6935
6936     return if $winversion < 6;
6937     return if $bios && $bios eq 'ovmf' && $winversion < 8;
6938
6939     push @$cpuFlags , 'hv_vendor_id=proxmox' if $gpu_passthrough;
6940
6941     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
6942         push @$cpuFlags , 'hv_spinlocks=0x1fff';
6943         push @$cpuFlags , 'hv_vapic';
6944         push @$cpuFlags , 'hv_time';
6945     } else {
6946         push @$cpuFlags , 'hv_spinlocks=0xffff';
6947     }
6948
6949     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 6)) {
6950         push @$cpuFlags , 'hv_reset';
6951         push @$cpuFlags , 'hv_vpindex';
6952         push @$cpuFlags , 'hv_runtime';
6953     }
6954
6955     if ($winversion >= 7) {
6956         push @$cpuFlags , 'hv_relaxed';
6957
6958         if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 12)) {
6959             push @$cpuFlags , 'hv_synic';
6960             push @$cpuFlags , 'hv_stimer';
6961         }
6962     }
6963 }
6964
6965 sub windows_version {
6966     my ($ostype) = @_;
6967
6968     return 0 if !$ostype;
6969
6970     my $winversion = 0;
6971
6972     if($ostype eq 'wxp' || $ostype eq 'w2k3' || $ostype eq 'w2k') {
6973         $winversion = 5;
6974     } elsif($ostype eq 'w2k8' || $ostype eq 'wvista') {
6975         $winversion = 6;
6976     } elsif ($ostype =~ m/^win(\d+)$/) {
6977         $winversion = $1;
6978     }
6979
6980     return $winversion;
6981 }
6982
6983 sub resolve_dst_disk_format {
6984         my ($storecfg, $storeid, $src_volname, $format) = @_;
6985         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
6986
6987         if (!$format) {
6988             # if no target format is specified, use the source disk format as hint
6989             if ($src_volname) {
6990                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6991                 $format = qemu_img_format($scfg, $src_volname);
6992             } else {
6993                 return $defFormat;
6994             }
6995         }
6996
6997         # test if requested format is supported - else use default
6998         my $supported = grep { $_ eq $format } @$validFormats;
6999         $format = $defFormat if !$supported;
7000         return $format;
7001 }
7002
7003 sub resolve_first_disk {
7004     my $conf = shift;
7005     my @disks = PVE::QemuServer::valid_drive_names();
7006     my $firstdisk;
7007     foreach my $ds (reverse @disks) {
7008         next if !$conf->{$ds};
7009         my $disk = PVE::QemuServer::parse_drive($ds, $conf->{$ds});
7010         next if PVE::QemuServer::drive_is_cdrom($disk);
7011         $firstdisk = $ds;
7012     }
7013     return $firstdisk;
7014 }
7015
7016 sub generate_uuid {
7017     my ($uuid, $uuid_str);
7018     UUID::generate($uuid);
7019     UUID::unparse($uuid, $uuid_str);
7020     return $uuid_str;
7021 }
7022
7023 sub generate_smbios1_uuid {
7024     return "uuid=".generate_uuid();
7025 }
7026
7027 sub nbd_stop {
7028     my ($vmid) = @_;
7029
7030     vm_mon_cmd($vmid, 'nbd-server-stop');
7031 }
7032
7033 # bash completion helper
7034
7035 sub complete_backup_archives {
7036     my ($cmdname, $pname, $cvalue) = @_;
7037
7038     my $cfg = PVE::Storage::config();
7039
7040     my $storeid;
7041
7042     if ($cvalue =~ m/^([^:]+):/) {
7043         $storeid = $1;
7044     }
7045
7046     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
7047
7048     my $res = [];
7049     foreach my $id (keys %$data) {
7050         foreach my $item (@{$data->{$id}}) {
7051             next if $item->{format} !~ m/^vma\.(gz|lzo)$/;
7052             push @$res, $item->{volid} if defined($item->{volid});
7053         }
7054     }
7055
7056     return $res;
7057 }
7058
7059 my $complete_vmid_full = sub {
7060     my ($running) = @_;
7061
7062     my $idlist = vmstatus();
7063
7064     my $res = [];
7065
7066     foreach my $id (keys %$idlist) {
7067         my $d = $idlist->{$id};
7068         if (defined($running)) {
7069             next if $d->{template};
7070             next if $running && $d->{status} ne 'running';
7071             next if !$running && $d->{status} eq 'running';
7072         }
7073         push @$res, $id;
7074
7075     }
7076     return $res;
7077 };
7078
7079 sub complete_vmid {
7080     return &$complete_vmid_full();
7081 }
7082
7083 sub complete_vmid_stopped {
7084     return &$complete_vmid_full(0);
7085 }
7086
7087 sub complete_vmid_running {
7088     return &$complete_vmid_full(1);
7089 }
7090
7091 sub complete_storage {
7092
7093     my $cfg = PVE::Storage::config();
7094     my $ids = $cfg->{ids};
7095
7096     my $res = [];
7097     foreach my $sid (keys %$ids) {
7098         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
7099         next if !$ids->{$sid}->{content}->{images};
7100         push @$res, $sid;
7101     }
7102
7103     return $res;
7104 }
7105
7106 1;