PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5 use POSIX;
   6 use IO::Handle;
   7 use IO::Select;
   8 use IO::File;
   9 use IO::Dir;
  10 use IO::Socket::UNIX;
  11 use File::Basename;
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use Digest::SHA;
  16 use Fcntl ':flock';
  17 use Cwd 'abs_path';
  18 use IPC::Open3;
  19 use JSON;
  20 use Fcntl;
  21 use PVE::SafeSyslog;
  22 use Storable qw(dclone);
  23 use PVE::Exception qw(raise raise_param_exc);
  24 use PVE::Storage;
  25 use PVE::Tools qw(run_command lock_file lock_file_full file_read_firstline dir_glob_foreach $IPV6RE);
  26 use PVE::JSONSchema qw(get_standard_option);
  27 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file cfs_lock_file);
  28 use PVE::INotify;
  29 use PVE::ProcFSTools;
  30 use PVE::QemuConfig;
  31 use PVE::QMPClient;
  32 use PVE::RPCEnvironment;
  33 use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr);
  34 use PVE::QemuServer::Memory;
  35 use PVE::QemuServer::USB qw(parse_usb_device);
  36 use PVE::QemuServer::Cloudinit;
  37 use PVE::Systemd;
  38 use Time::HiRes qw(gettimeofday);
  39 use File::Copy qw(copy);
  40 use URI::Escape;
  41
  42 my $EDK2_FW_BASE = '/usr/share/pve-edk2-firmware/';
  43 my $OVMF_CODE = "$EDK2_FW_BASE/OVMF_CODE.fd";
  44 my $OVMF_VARS = "$EDK2_FW_BASE/OVMF_VARS.fd";
  45
  46 my $qemu_snap_storage = {rbd => 1, sheepdog => 1};
  47
  48 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  49
  50 my $QEMU_FORMAT_RE = qr/raw|cow|qcow|qcow2|qed|vmdk|cloop/;
  51
  52 # Note about locking: we use flock on the config file protect
  53 # against concurent actions.
  54 # Aditionaly, we have a 'lock' setting in the config file. This
  55 # can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
  56 # allowed when such lock is set. But you can ignore this kind of
  57 # lock with the --skiplock flag.
  58
  59 cfs_register_file('/qemu-server/',
  60                   \&parse_vm_config,
  61                   \&write_vm_config);
  62
  63 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
  64     description => "Some command save/restore state from this location.",
  65     type => 'string',
  66     maxLength => 128,
  67     optional => 1,
  68 });
  69
  70 PVE::JSONSchema::register_standard_option('pve-snapshot-name', {
  71     description => "The name of the snapshot.",
  72     type => 'string', format => 'pve-configid',
  73     maxLength => 40,
  74 });
  75
  76 PVE::JSONSchema::register_standard_option('pve-qm-image-format', {
  77     type => 'string',
  78     enum => [qw(raw cow qcow qed qcow2 vmdk cloop)],
  79     description => "The drive's backing file's data format.",
  80     optional => 1,
  81 });
  82
  83 PVE::JSONSchema::register_standard_option('pve-qemu-machine', {
  84         description => "Specifies the Qemu machine type.",
  85         type => 'string',
  86         pattern => '(pc|pc(-i440fx)?-\d+\.\d+(\.pxe)?|q35|pc-q35-\d+\.\d+(\.pxe)?)',
  87         maxLength => 40,
  88         optional => 1,
  89 });
  90
  91 #no warnings 'redefine';
  92
  93 sub cgroups_write {
  94    my ($controller, $vmid, $option, $value) = @_;
  95
  96    my $path = "/sys/fs/cgroup/$controller/qemu.slice/$vmid.scope/$option";
  97    PVE::ProcFSTools::write_proc_entry($path, $value);
  98
  99 }
 100
 101 my $nodename = PVE::INotify::nodename();
 102
 103 mkdir "/etc/pve/nodes/$nodename";
 104 my $confdir = "/etc/pve/nodes/$nodename/qemu-server";
 105 mkdir $confdir;
 106
 107 my $var_run_tmpdir = "/var/run/qemu-server";
 108 mkdir $var_run_tmpdir;
 109
 110 my $lock_dir = "/var/lock/qemu-server";
 111 mkdir $lock_dir;
 112
 113 my $pcisysfs = "/sys/bus/pci";
 114
 115 my $cpu_vendor_list = {
 116     # Intel CPUs
 117     486 => 'GenuineIntel',
 118     pentium => 'GenuineIntel',
 119     pentium2  => 'GenuineIntel',
 120     pentium3  => 'GenuineIntel',
 121     coreduo => 'GenuineIntel',
 122     core2duo => 'GenuineIntel',
 123     Conroe  => 'GenuineIntel',
 124     Penryn  => 'GenuineIntel',
 125     Nehalem  => 'GenuineIntel',
 126     'Nehalem-IBRS'  => 'GenuineIntel',
 127     Westmere => 'GenuineIntel',
 128     'Westmere-IBRS' => 'GenuineIntel',
 129     SandyBridge => 'GenuineIntel',
 130     'SandyBridge-IBRS' => 'GenuineIntel',
 131     IvyBridge => 'GenuineIntel',
 132     'IvyBridge-IBRS' => 'GenuineIntel',
 133     Haswell => 'GenuineIntel',
 134     'Haswell-IBRS' => 'GenuineIntel',
 135     'Haswell-noTSX' => 'GenuineIntel',
 136     'Haswell-noTSX-IBRS' => 'GenuineIntel',
 137     Broadwell => 'GenuineIntel',
 138     'Broadwell-IBRS' => 'GenuineIntel',
 139     'Broadwell-noTSX' => 'GenuineIntel',
 140     'Broadwell-noTSX-IBRS' => 'GenuineIntel',
 141     'Skylake-Client' => 'GenuineIntel',
 142     'Skylake-Client-IBRS' => 'GenuineIntel',
 143     'Skylake-Server' => 'GenuineIntel',
 144     'Skylake-Server-IBRS' => 'GenuineIntel',
 145
 146     # AMD CPUs
 147     athlon => 'AuthenticAMD',
 148     phenom  => 'AuthenticAMD',
 149     Opteron_G1  => 'AuthenticAMD',
 150     Opteron_G2  => 'AuthenticAMD',
 151     Opteron_G3  => 'AuthenticAMD',
 152     Opteron_G4  => 'AuthenticAMD',
 153     Opteron_G5  => 'AuthenticAMD',
 154     EPYC => 'AuthenticAMD',
 155     'EPYC-IBPB' => 'AuthenticAMD',
 156
 157     # generic types, use vendor from host node
 158     host => 'default',
 159     kvm32 => 'default',
 160     kvm64 => 'default',
 161     qemu32 => 'default',
 162     qemu64 => 'default',
 163     max => 'default',
 164 };
 165
 166 my $cpu_flag = qr/[+-](pcid|spec-ctrl|ibpb|ssbd|virt-ssbd|amd-ssbd|amd-no-ssb|pdpe1gb)/;
 167
 168 my $cpu_fmt = {
 169     cputype => {
 170         description => "Emulated CPU type.",
 171         type => 'string',
 172         enum => [ sort { "\L$a" cmp "\L$b" } keys %$cpu_vendor_list ],
 173         default => 'kvm64',
 174         default_key => 1,
 175     },
 176     hidden => {
 177         description => "Do not identify as a KVM virtual machine.",
 178         type => 'boolean',
 179         optional => 1,
 180         default => 0
 181     },
 182     flags => {
 183         description => "List of additional CPU flags separated by ';'."
 184                      . " Use '+FLAG' to enable, '-FLAG' to disable a flag."
 185                      . " Currently supported flags: 'pcid', 'spec-ctrl', 'ibpb', 'ssbd', 'virt-ssbd', 'amd-ssbd', 'amd-no-ssb', 'pdpe1gb'.",
 186         format_description => '+FLAG[;-FLAG...]',
 187         type => 'string',
 188         pattern => qr/$cpu_flag(;$cpu_flag)*/,
 189         optional => 1,
 190     },
 191 };
 192
 193 my $watchdog_fmt = {
 194     model => {
 195         default_key => 1,
 196         type => 'string',
 197         enum => [qw(i6300esb ib700)],
 198         description => "Watchdog type to emulate.",
 199         default => 'i6300esb',
 200         optional => 1,
 201     },
 202     action => {
 203         type => 'string',
 204         enum => [qw(reset shutdown poweroff pause debug none)],
 205         description => "The action to perform if after activation the guest fails to poll the watchdog in time.",
 206         optional => 1,
 207     },
 208 };
 209 PVE::JSONSchema::register_format('pve-qm-watchdog', $watchdog_fmt);
 210
 211 my $agent_fmt = {
 212     enabled => {
 213         description => "Enable/disable Qemu GuestAgent.",
 214         type => 'boolean',
 215         default => 0,
 216         default_key => 1,
 217     },
 218     fstrim_cloned_disks => {
 219         description => "Run fstrim after cloning/moving a disk.",
 220         type => 'boolean',
 221         optional => 1,
 222         default => 0
 223     },
 224 };
 225
 226 my $confdesc = {
 227     onboot => {
 228         optional => 1,
 229         type => 'boolean',
 230         description => "Specifies whether a VM will be started during system bootup.",
 231         default => 0,
 232     },
 233     autostart => {
 234         optional => 1,
 235         type => 'boolean',
 236         description => "Automatic restart after crash (currently ignored).",
 237         default => 0,
 238     },
 239     hotplug => {
 240         optional => 1,
 241         type => 'string', format => 'pve-hotplug-features',
 242         description => "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
 243         default => 'network,disk,usb',
 244     },
 245     reboot => {
 246         optional => 1,
 247         type => 'boolean',
 248         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 249         default => 1,
 250     },
 251     lock => {
 252         optional => 1,
 253         type => 'string',
 254         description => "Lock/unlock the VM.",
 255         enum => [qw(migrate backup snapshot rollback)],
 256     },
 257     cpulimit => {
 258         optional => 1,
 259         type => 'number',
 260         description => "Limit of CPU usage.",
 261         verbose_description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
 262         minimum => 0,
 263         maximum => 128,
 264         default => 0,
 265     },
 266     cpuunits => {
 267         optional => 1,
 268         type => 'integer',
 269         description => "CPU weight for a VM.",
 270         verbose_description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.",
 271         minimum => 2,
 272         maximum => 262144,
 273         default => 1024,
 274     },
 275     memory => {
 276         optional => 1,
 277         type => 'integer',
 278         description => "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
 279         minimum => 16,
 280         default => 512,
 281     },
 282     balloon => {
 283         optional => 1,
 284         type => 'integer',
 285         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 286         minimum => 0,
 287     },
 288     shares => {
 289         optional => 1,
 290         type => 'integer',
 291         description => "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning. Auto-ballooning is done by pvestatd.",
 292         minimum => 0,
 293         maximum => 50000,
 294         default => 1000,
 295     },
 296     keyboard => {
 297         optional => 1,
 298         type => 'string',
 299         description => "Keybord layout for vnc server. Default is read from the '/etc/pve/datacenter.cfg' configuration file.".
 300                        "It should not be necessary to set it.",
 301         enum => PVE::Tools::kvmkeymaplist(),
 302         default => undef,
 303     },
 304     name => {
 305         optional => 1,
 306         type => 'string', format => 'dns-name',
 307         description => "Set a name for the VM. Only used on the configuration web interface.",
 308     },
 309     scsihw => {
 310         optional => 1,
 311         type => 'string',
 312         description => "SCSI controller model",
 313         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 314         default => 'lsi',
 315     },
 316     description => {
 317         optional => 1,
 318         type => 'string',
 319         description => "Description for the VM. Only used on the configuration web interface. This is saved as comment inside the configuration file.",
 320     },
 321     ostype => {
 322         optional => 1,
 323         type => 'string',
 324         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 l24 l26 solaris)],
 325         description => "Specify guest operating system.",
 326         verbose_description => <<EODESC,
 327 Specify guest operating system. This is used to enable special
 328 optimization/features for specific operating systems:
 329
 330 [horizontal]
 331 other;; unspecified OS
 332 wxp;; Microsoft Windows XP
 333 w2k;; Microsoft Windows 2000
 334 w2k3;; Microsoft Windows 2003
 335 w2k8;; Microsoft Windows 2008
 336 wvista;; Microsoft Windows Vista
 337 win7;; Microsoft Windows 7
 338 win8;; Microsoft Windows 8/2012/2012r2
 339 win10;; Microsoft Windows 10/2016
 340 l24;; Linux 2.4 Kernel
 341 l26;; Linux 2.6/3.X Kernel
 342 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
 343 EODESC
 344     },
 345     boot => {
 346         optional => 1,
 347         type => 'string',
 348         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n).",
 349         pattern => '[acdn]{1,4}',
 350         default => 'cdn',
 351     },
 352     bootdisk => {
 353         optional => 1,
 354         type => 'string', format => 'pve-qm-bootdisk',
 355         description => "Enable booting from specified disk.",
 356         pattern => '(ide|sata|scsi|virtio)\d+',
 357     },
 358     smp => {
 359         optional => 1,
 360         type => 'integer',
 361         description => "The number of CPUs. Please use option -sockets instead.",
 362         minimum => 1,
 363         default => 1,
 364     },
 365     sockets => {
 366         optional => 1,
 367         type => 'integer',
 368         description => "The number of CPU sockets.",
 369         minimum => 1,
 370         default => 1,
 371     },
 372     cores => {
 373         optional => 1,
 374         type => 'integer',
 375         description => "The number of cores per socket.",
 376         minimum => 1,
 377         default => 1,
 378     },
 379     numa => {
 380         optional => 1,
 381         type => 'boolean',
 382         description => "Enable/disable NUMA.",
 383         default => 0,
 384     },
 385     hugepages => {
 386         optional => 1,
 387         type => 'string',
 388         description => "Enable/disable hugepages memory.",
 389         enum => [qw(any 2 1024)],
 390     },
 391     vcpus => {
 392         optional => 1,
 393         type => 'integer',
 394         description => "Number of hotplugged vcpus.",
 395         minimum => 1,
 396         default => 0,
 397     },
 398     acpi => {
 399         optional => 1,
 400         type => 'boolean',
 401         description => "Enable/disable ACPI.",
 402         default => 1,
 403     },
 404     agent => {
 405         optional => 1,
 406         description => "Enable/disable Qemu GuestAgent and its properties.",
 407         type => 'string',
 408         format => $agent_fmt,
 409     },
 410     kvm => {
 411         optional => 1,
 412         type => 'boolean',
 413         description => "Enable/disable KVM hardware virtualization.",
 414         default => 1,
 415     },
 416     tdf => {
 417         optional => 1,
 418         type => 'boolean',
 419         description => "Enable/disable time drift fix.",
 420         default => 0,
 421     },
 422     localtime => {
 423         optional => 1,
 424         type => 'boolean',
 425         description => "Set the real time clock to local time. This is enabled by default if ostype indicates a Microsoft OS.",
 426     },
 427     freeze => {
 428         optional => 1,
 429         type => 'boolean',
 430         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 431     },
 432     vga => {
 433         optional => 1,
 434         type => 'string',
 435         description => "Select the VGA type.",
 436         verbose_description => "Select the VGA type. If you want to use high resolution" .
 437             " modes (>= 1280x1024x16) then you should use the options " .
 438             "'std' or 'vmware'. Default is 'std' for win8/win7/w2k8, and " .
 439             "'cirrus' for other OS types. The 'qxl' option enables the SPICE " .
 440             "display sever. For win* OS you can select how many independent " .
 441             "displays you want, Linux guests can add displays them self. " .
 442             "You can also run without any graphic card, using a serial device" .
 443             " as terminal.",
 444         enum => [qw(cirrus qxl qxl2 qxl3 qxl4 serial0 serial1 serial2 serial3 std virtio vmware)],
 445     },
 446     watchdog => {
 447         optional => 1,
 448         type => 'string', format => 'pve-qm-watchdog',
 449         description => "Create a virtual hardware watchdog device.",
 450         verbose_description => "Create a virtual hardware watchdog device. Once enabled" .
 451             " (by a guest action), the watchdog must be periodically polled " .
 452             "by an agent inside the guest or else the watchdog will reset " .
 453             "the guest (or execute the respective action specified)",
 454     },
 455     startdate => {
 456         optional => 1,
 457         type => 'string',
 458         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 459         description => "Set the initial date of the real time clock. Valid format for date are: 'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 460         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 461         default => 'now',
 462     },
 463     startup =>  get_standard_option('pve-startup-order'),
 464     template => {
 465         optional => 1,
 466         type => 'boolean',
 467         description => "Enable/disable Template.",
 468         default => 0,
 469     },
 470     args => {
 471         optional => 1,
 472         type => 'string',
 473         description => "Arbitrary arguments passed to kvm.",
 474         verbose_description => <<EODESCR,
 475 Arbitrary arguments passed to kvm, for example:
 476
 477 args: -no-reboot -no-hpet
 478
 479 NOTE: this option is for experts only.
 480 EODESCR
 481     },
 482     tablet => {
 483         optional => 1,
 484         type => 'boolean',
 485         default => 1,
 486         description => "Enable/disable the USB tablet device.",
 487         verbose_description => "Enable/disable the USB tablet device. This device is " .
 488             "usually needed to allow absolute mouse positioning with VNC. " .
 489             "Else the mouse runs out of sync with normal VNC clients. " .
 490             "If you're running lots of console-only guests on one host, " .
 491             "you may consider disabling this to save some context switches. " .
 492             "This is turned off by default if you use spice (-vga=qxl).",
 493     },
 494     migrate_speed => {
 495         optional => 1,
 496         type => 'integer',
 497         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 498         minimum => 0,
 499         default => 0,
 500     },
 501     migrate_downtime => {
 502         optional => 1,
 503         type => 'number',
 504         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 505         minimum => 0,
 506         default => 0.1,
 507     },
 508     cdrom => {
 509         optional => 1,
 510         type => 'string', format => 'pve-qm-ide',
 511         typetext => '<volume>',
 512         description => "This is an alias for option -ide2",
 513     },
 514     cpu => {
 515         optional => 1,
 516         description => "Emulated CPU type.",
 517         type => 'string',
 518         format => $cpu_fmt,
 519     },
 520     parent => get_standard_option('pve-snapshot-name', {
 521         optional => 1,
 522         description => "Parent snapshot name. This is used internally, and should not be modified.",
 523     }),
 524     snaptime => {
 525         optional => 1,
 526         description => "Timestamp for snapshots.",
 527         type => 'integer',
 528         minimum => 0,
 529     },
 530     vmstate => {
 531         optional => 1,
 532         type => 'string', format => 'pve-volume-id',
 533         description => "Reference to a volume which stores the VM state. This is used internally for snapshots.",
 534     },
 535     vmstatestorage => get_standard_option('pve-storage-id', {
 536         description => "Default storage for VM state volumes/files.",
 537         optional => 1,
 538     }),
 539     runningmachine => get_standard_option('pve-qemu-machine', {
 540         description => "Specifies the Qemu machine type of the running vm. This is used internally for snapshots.",
 541     }),
 542     machine => get_standard_option('pve-qemu-machine'),
 543     smbios1 => {
 544         description => "Specify SMBIOS type 1 fields.",
 545         type => 'string', format => 'pve-qm-smbios1',
 546         maxLength => 256,
 547         optional => 1,
 548     },
 549     protection => {
 550         optional => 1,
 551         type => 'boolean',
 552         description => "Sets the protection flag of the VM. This will disable the remove VM and remove disk operations.",
 553         default => 0,
 554     },
 555     bios => {
 556         optional => 1,
 557         type => 'string',
 558         enum => [ qw(seabios ovmf) ],
 559         description => "Select BIOS implementation.",
 560         default => 'seabios',
 561     },
 562     vmgenid => {
 563         type => 'string',
 564         pattern => '(?:[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}|[01])',
 565         format_description => 'UUID',
 566         description => "Set VM Generation ID. Use '1' to autogenerate on create or update, pass '0' to disable explicitly.",
 567         verbose_description => "The VM generation ID (vmgenid) device exposes a".
 568             " 128-bit integer value identifier to the guest OS. This allows to".
 569             " notify the guest operating system when the virtual machine is".
 570             " executed with a different configuration (e.g. snapshot execution".
 571             " or creation from a template). The guest operating system notices".
 572             " the change, and is then able to react as appropriate by marking".
 573             " its copies of distributed databases as dirty, re-initializing its".
 574             " random number generator, etc.\n".
 575             "Note that auto-creation only works when done throug API/CLI create".
 576             " or update methods, but not when manually editing the config file.",
 577         default => "1 (autogenerated)",
 578         optional => 1,
 579     },
 580 };
 581
 582 my $confdesc_cloudinit = {
 583     citype => {
 584         optional => 1,
 585         type => 'string',
 586         description => 'Specifies the cloud-init configuration format. The default depends on the configured operating system type (`ostype`. We use the `nocloud` format for Linux, and `configdrive2` for windows.',
 587         enum => ['configdrive2', 'nocloud'],
 588     },
 589     ciuser => {
 590         optional => 1,
 591         type => 'string',
 592         description => "cloud-init: User name to change ssh keys and password for instead of the image's configured default user.",
 593     },
 594     cipassword => {
 595         optional => 1,
 596         type => 'string',
 597         description => 'cloud-init: Password to assign the user. Using this is generally not recommended. Use ssh keys instead. Also note that older cloud-init versions do not support hashed passwords.',
 598     },
 599     searchdomain => {
 600         optional => 1,
 601         type => 'string',
 602         description => "cloud-init: Sets DNS search domains for a container. Create will automatically use the setting from the host if neither searchdomain nor nameserver are set.",
 603     },
 604     nameserver => {
 605         optional => 1,
 606         type => 'string', format => 'address-list',
 607         description => "cloud-init: Sets DNS server IP address for a container. Create will automatically use the setting from the host if neither searchdomain nor nameserver are set.",
 608     },
 609     sshkeys => {
 610         optional => 1,
 611         type => 'string',
 612         format => 'urlencoded',
 613         description => "cloud-init: Setup public SSH keys (one key per line, OpenSSH format).",
 614     },
 615 };
 616
 617 # what about other qemu settings ?
 618 #cpu => 'string',
 619 #machine => 'string',
 620 #fda => 'file',
 621 #fdb => 'file',
 622 #mtdblock => 'file',
 623 #sd => 'file',
 624 #pflash => 'file',
 625 #snapshot => 'bool',
 626 #bootp => 'file',
 627 ##tftp => 'dir',
 628 ##smb => 'dir',
 629 #kernel => 'file',
 630 #append => 'string',
 631 #initrd => 'file',
 632 ##soundhw => 'string',
 633
 634 while (my ($k, $v) = each %$confdesc) {
 635     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 636 }
 637
 638 my $MAX_IDE_DISKS = 4;
 639 my $MAX_SCSI_DISKS = 14;
 640 my $MAX_VIRTIO_DISKS = 16;
 641 my $MAX_SATA_DISKS = 6;
 642 my $MAX_USB_DEVICES = 5;
 643 my $MAX_NETS = 32;
 644 my $MAX_UNUSED_DISKS = 8;
 645 my $MAX_HOSTPCI_DEVICES = 4;
 646 my $MAX_SERIAL_PORTS = 4;
 647 my $MAX_PARALLEL_PORTS = 3;
 648 my $MAX_NUMA = 8;
 649
 650 my $numa_fmt = {
 651     cpus => {
 652         type => "string",
 653         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 654         description => "CPUs accessing this NUMA node.",
 655         format_description => "id[-id];...",
 656     },
 657     memory => {
 658         type => "number",
 659         description => "Amount of memory this NUMA node provides.",
 660         optional => 1,
 661     },
 662     hostnodes => {
 663         type => "string",
 664         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 665         description => "Host NUMA nodes to use.",
 666         format_description => "id[-id];...",
 667         optional => 1,
 668     },
 669     policy => {
 670         type => 'string',
 671         enum => [qw(preferred bind interleave)],
 672         description => "NUMA allocation policy.",
 673         optional => 1,
 674     },
 675 };
 676 PVE::JSONSchema::register_format('pve-qm-numanode', $numa_fmt);
 677 my $numadesc = {
 678     optional => 1,
 679     type => 'string', format => $numa_fmt,
 680     description => "NUMA topology.",
 681 };
 682 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 683
 684 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 685     $confdesc->{"numa$i"} = $numadesc;
 686 }
 687
 688 my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000',  'pcnet',  'virtio',
 689                       'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3',
 690                       'e1000-82540em', 'e1000-82544gc', 'e1000-82545em'];
 691 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 692
 693 my $net_fmt_bridge_descr = <<__EOD__;
 694 Bridge to attach the network device to. The Proxmox VE standard bridge
 695 is called 'vmbr0'.
 696
 697 If you do not specify a bridge, we create a kvm user (NATed) network
 698 device, which provides DHCP and DNS services. The following addresses
 699 are used:
 700
 701  10.0.2.2   Gateway
 702  10.0.2.3   DNS Server
 703  10.0.2.4   SMB Server
 704
 705 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 706 __EOD__
 707
 708 my $net_fmt = {
 709     macaddr => {
 710         type => 'string',
 711         pattern => qr/[0-9a-f]{2}(?::[0-9a-f]{2}){5}/i,
 712         description => "MAC address. That address must be unique withing your network. This is automatically generated if not specified.",
 713         format_description => "XX:XX:XX:XX:XX:XX",
 714         optional => 1,
 715     },
 716     model => {
 717         type => 'string',
 718         description => "Network Card Model. The 'virtio' model provides the best performance with very low CPU overhead. If your guest does not support this driver, it is usually best to use 'e1000'.",
 719         enum => $nic_model_list,
 720         default_key => 1,
 721     },
 722     (map { $_ => { keyAlias => 'model', alias => 'macaddr' }} @$nic_model_list),
 723     bridge => {
 724         type => 'string',
 725         description => $net_fmt_bridge_descr,
 726         format_description => 'bridge',
 727         optional => 1,
 728     },
 729     queues => {
 730         type => 'integer',
 731         minimum => 0, maximum => 16,
 732         description => 'Number of packet queues to be used on the device.',
 733         optional => 1,
 734     },
 735     rate => {
 736         type => 'number',
 737         minimum => 0,
 738         description => "Rate limit in mbps (megabytes per second) as floating point number.",
 739         optional => 1,
 740     },
 741     tag => {
 742         type => 'integer',
 743         minimum => 1, maximum => 4094,
 744         description => 'VLAN tag to apply to packets on this interface.',
 745         optional => 1,
 746     },
 747     trunks => {
 748         type => 'string',
 749         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 750         description => 'VLAN trunks to pass through this interface.',
 751         format_description => 'vlanid[;vlanid...]',
 752         optional => 1,
 753     },
 754     firewall => {
 755         type => 'boolean',
 756         description => 'Whether this interface should be protected by the firewall.',
 757         optional => 1,
 758     },
 759     link_down => {
 760         type => 'boolean',
 761         description => 'Whether this interface should be disconnected (like pulling the plug).',
 762         optional => 1,
 763     },
 764 };
 765
 766 my $netdesc = {
 767     optional => 1,
 768     type => 'string', format => $net_fmt,
 769     description => "Specify network devices.",
 770 };
 771
 772 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 773
 774 my $ipconfig_fmt = {
 775     ip => {
 776         type => 'string',
 777         format => 'pve-ipv4-config',
 778         format_description => 'IPv4Format/CIDR',
 779         description => 'IPv4 address in CIDR format.',
 780         optional => 1,
 781         default => 'dhcp',
 782     },
 783     gw => {
 784         type => 'string',
 785         format => 'ipv4',
 786         format_description => 'GatewayIPv4',
 787         description => 'Default gateway for IPv4 traffic.',
 788         optional => 1,
 789         requires => 'ip',
 790     },
 791     ip6 => {
 792         type => 'string',
 793         format => 'pve-ipv6-config',
 794         format_description => 'IPv6Format/CIDR',
 795         description => 'IPv6 address in CIDR format.',
 796         optional => 1,
 797         default => 'dhcp',
 798     },
 799     gw6 => {
 800         type => 'string',
 801         format => 'ipv6',
 802         format_description => 'GatewayIPv6',
 803         description => 'Default gateway for IPv6 traffic.',
 804         optional => 1,
 805         requires => 'ip6',
 806     },
 807 };
 808 PVE::JSONSchema::register_format('pve-qm-ipconfig', $ipconfig_fmt);
 809 my $ipconfigdesc = {
 810     optional => 1,
 811     type => 'string', format => 'pve-qm-ipconfig',
 812     description => <<'EODESCR',
 813 cloud-init: Specify IP addresses and gateways for the corresponding interface.
 814
 815 IP addresses use CIDR notation, gateways are optional but need an IP of the same type specified.
 816
 817 The special string 'dhcp' can be used for IP addresses to use DHCP, in which case no explicit gateway should be provided.
 818 For IPv6 the special string 'auto' can be used to use stateless autoconfiguration.
 819
 820 If cloud-init is enabled and neither an IPv4 nor an IPv6 address is specified, it defaults to using dhcp on IPv4.
 821 EODESCR
 822 };
 823 PVE::JSONSchema::register_standard_option("pve-qm-ipconfig", $netdesc);
 824
 825 for (my $i = 0; $i < $MAX_NETS; $i++)  {
 826     $confdesc->{"net$i"} = $netdesc;
 827     $confdesc_cloudinit->{"ipconfig$i"} = $ipconfigdesc;
 828 }
 829
 830 foreach my $key (keys %$confdesc_cloudinit) {
 831     $confdesc->{$key} = $confdesc_cloudinit->{$key};
 832 }
 833
 834 PVE::JSONSchema::register_format('pve-volume-id-or-qm-path', \&verify_volume_id_or_qm_path);
 835 sub verify_volume_id_or_qm_path {
 836     my ($volid, $noerr) = @_;
 837
 838     if ($volid eq 'none' || $volid eq 'cdrom' || $volid =~ m|^/|) {
 839         return $volid;
 840     }
 841
 842     # if its neither 'none' nor 'cdrom' nor a path, check if its a volume-id
 843     $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') };
 844     if ($@) {
 845         return undef if $noerr;
 846         die $@;
 847     }
 848     return $volid;
 849 }
 850
 851 my $drivename_hash;
 852
 853 my %drivedesc_base = (
 854     volume => { alias => 'file' },
 855     file => {
 856         type => 'string',
 857         format => 'pve-volume-id-or-qm-path',
 858         default_key => 1,
 859         format_description => 'volume',
 860         description => "The drive's backing volume.",
 861     },
 862     media => {
 863         type => 'string',
 864         enum => [qw(cdrom disk)],
 865         description => "The drive's media type.",
 866         default => 'disk',
 867         optional => 1
 868     },
 869     cyls => {
 870         type => 'integer',
 871         description => "Force the drive's physical geometry to have a specific cylinder count.",
 872         optional => 1
 873     },
 874     heads => {
 875         type => 'integer',
 876         description => "Force the drive's physical geometry to have a specific head count.",
 877         optional => 1
 878     },
 879     secs => {
 880         type => 'integer',
 881         description => "Force the drive's physical geometry to have a specific sector count.",
 882         optional => 1
 883     },
 884     trans => {
 885         type => 'string',
 886         enum => [qw(none lba auto)],
 887         description => "Force disk geometry bios translation mode.",
 888         optional => 1,
 889     },
 890     snapshot => {
 891         type => 'boolean',
 892         description => "Controls qemu's snapshot mode feature."
 893             . " If activated, changes made to the disk are temporary and will"
 894             . " be discarded when the VM is shutdown.",
 895         optional => 1,
 896     },
 897     cache => {
 898         type => 'string',
 899         enum => [qw(none writethrough writeback unsafe directsync)],
 900         description => "The drive's cache mode",
 901         optional => 1,
 902     },
 903     format => get_standard_option('pve-qm-image-format'),
 904     size => {
 905         type => 'string',
 906         format => 'disk-size',
 907         format_description => 'DiskSize',
 908         description => "Disk size. This is purely informational and has no effect.",
 909         optional => 1,
 910     },
 911     backup => {
 912         type => 'boolean',
 913         description => "Whether the drive should be included when making backups.",
 914         optional => 1,
 915     },
 916     replicate => {
 917         type => 'boolean',
 918         description => 'Whether the drive should considered for replication jobs.',
 919         optional => 1,
 920         default => 1,
 921     },
 922     rerror => {
 923         type => 'string',
 924         enum => [qw(ignore report stop)],
 925         description => 'Read error action.',
 926         optional => 1,
 927     },
 928     werror => {
 929         type => 'string',
 930         enum => [qw(enospc ignore report stop)],
 931         description => 'Write error action.',
 932         optional => 1,
 933     },
 934     aio => {
 935         type => 'string',
 936         enum => [qw(native threads)],
 937         description => 'AIO type to use.',
 938         optional => 1,
 939     },
 940     discard => {
 941         type => 'string',
 942         enum => [qw(ignore on)],
 943         description => 'Controls whether to pass discard/trim requests to the underlying storage.',
 944         optional => 1,
 945     },
 946     detect_zeroes => {
 947         type => 'boolean',
 948         description => 'Controls whether to detect and try to optimize writes of zeroes.',
 949         optional => 1,
 950     },
 951     serial => {
 952         type => 'string',
 953         format => 'urlencoded',
 954         format_description => 'serial',
 955         maxLength => 20*3, # *3 since it's %xx url enoded
 956         description => "The drive's reported serial number, url-encoded, up to 20 bytes long.",
 957         optional => 1,
 958     },
 959     shared => {
 960         type => 'boolean',
 961         description => 'Mark this locally-managed volume as available on all nodes',
 962         verbose_description => "Mark this locally-managed volume as available on all nodes.\n\nWARNING: This option does not share the volume automatically, it assumes it is shared already!",
 963         optional => 1,
 964         default => 0,
 965     }
 966 );
 967
 968 my %iothread_fmt = ( iothread => {
 969         type => 'boolean',
 970         description => "Whether to use iothreads for this drive",
 971         optional => 1,
 972 });
 973
 974 my %model_fmt = (
 975     model => {
 976         type => 'string',
 977         format => 'urlencoded',
 978         format_description => 'model',
 979         maxLength => 40*3, # *3 since it's %xx url enoded
 980         description => "The drive's reported model name, url-encoded, up to 40 bytes long.",
 981         optional => 1,
 982     },
 983 );
 984
 985 my %queues_fmt = (
 986     queues => {
 987         type => 'integer',
 988         description => "Number of queues.",
 989         minimum => 2,
 990         optional => 1
 991     }
 992 );
 993
 994 my %scsiblock_fmt = (
 995     scsiblock => {
 996         type => 'boolean',
 997         description => "whether to use scsi-block for full passthrough of host block device\n\nWARNING: can lead to I/O errors in combination with low memory or high memory fragmentation on host",
 998         optional => 1,
 999         default => 0,
1000     },
1001 );
1002
1003 my $add_throttle_desc = sub {
1004     my ($key, $type, $what, $unit, $longunit, $minimum) = @_;
1005     my $d = {
1006         type => $type,
1007         format_description => $unit,
1008         description => "Maximum $what in $longunit.",
1009         optional => 1,
1010     };
1011     $d->{minimum} = $minimum if defined($minimum);
1012     $drivedesc_base{$key} = $d;
1013 };
1014 # throughput: (leaky bucket)
1015 $add_throttle_desc->('bps',     'integer', 'r/w speed',   'bps',  'bytes per second');
1016 $add_throttle_desc->('bps_rd',  'integer', 'read speed',  'bps',  'bytes per second');
1017 $add_throttle_desc->('bps_wr',  'integer', 'write speed', 'bps',  'bytes per second');
1018 $add_throttle_desc->('mbps',    'number',  'r/w speed',   'mbps', 'megabytes per second');
1019 $add_throttle_desc->('mbps_rd', 'number',  'read speed',  'mbps', 'megabytes per second');
1020 $add_throttle_desc->('mbps_wr', 'number',  'write speed', 'mbps', 'megabytes per second');
1021 $add_throttle_desc->('iops',    'integer', 'r/w I/O',     'iops', 'operations per second');
1022 $add_throttle_desc->('iops_rd', 'integer', 'read I/O',    'iops', 'operations per second');
1023 $add_throttle_desc->('iops_wr', 'integer', 'write I/O',   'iops', 'operations per second');
1024
1025 # pools: (pool of IO before throttling starts taking effect)
1026 $add_throttle_desc->('mbps_max',    'number',  'unthrottled r/w pool',       'mbps', 'megabytes per second');
1027 $add_throttle_desc->('mbps_rd_max', 'number',  'unthrottled read pool',      'mbps', 'megabytes per second');
1028 $add_throttle_desc->('mbps_wr_max', 'number',  'unthrottled write pool',     'mbps', 'megabytes per second');
1029 $add_throttle_desc->('iops_max',    'integer', 'unthrottled r/w I/O pool',   'iops', 'operations per second');
1030 $add_throttle_desc->('iops_rd_max', 'integer', 'unthrottled read I/O pool',  'iops', 'operations per second');
1031 $add_throttle_desc->('iops_wr_max', 'integer', 'unthrottled write I/O pool', 'iops', 'operations per second');
1032
1033 # burst lengths
1034 $add_throttle_desc->('bps_max_length',     'integer', 'length of I/O bursts',       'seconds', 'seconds', 1);
1035 $add_throttle_desc->('bps_rd_max_length',  'integer', 'length of read I/O bursts',  'seconds', 'seconds', 1);
1036 $add_throttle_desc->('bps_wr_max_length',  'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
1037 $add_throttle_desc->('iops_max_length',    'integer', 'length of I/O bursts',       'seconds', 'seconds', 1);
1038 $add_throttle_desc->('iops_rd_max_length', 'integer', 'length of read I/O bursts',  'seconds', 'seconds', 1);
1039 $add_throttle_desc->('iops_wr_max_length', 'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
1040
1041 # legacy support
1042 $drivedesc_base{'bps_rd_length'} = { alias => 'bps_rd_max_length' };
1043 $drivedesc_base{'bps_wr_length'} = { alias => 'bps_wr_max_length' };
1044 $drivedesc_base{'iops_rd_length'} = { alias => 'iops_rd_max_length' };
1045 $drivedesc_base{'iops_wr_length'} = { alias => 'iops_wr_max_length' };
1046
1047 my $ide_fmt = {
1048     %drivedesc_base,
1049     %model_fmt,
1050 };
1051 PVE::JSONSchema::register_format("pve-qm-ide", $ide_fmt);
1052
1053 my $idedesc = {
1054     optional => 1,
1055     type => 'string', format => $ide_fmt,
1056     description => "Use volume as IDE hard disk or CD-ROM (n is 0 to " .($MAX_IDE_DISKS -1) . ").",
1057 };
1058 PVE::JSONSchema::register_standard_option("pve-qm-ide", $idedesc);
1059
1060 my $scsi_fmt = {
1061     %drivedesc_base,
1062     %iothread_fmt,
1063     %queues_fmt,
1064     %scsiblock_fmt,
1065 };
1066 my $scsidesc = {
1067     optional => 1,
1068     type => 'string', format => $scsi_fmt,
1069     description => "Use volume as SCSI hard disk or CD-ROM (n is 0 to " . ($MAX_SCSI_DISKS - 1) . ").",
1070 };
1071 PVE::JSONSchema::register_standard_option("pve-qm-scsi", $scsidesc);
1072
1073 my $sata_fmt = {
1074     %drivedesc_base,
1075 };
1076 my $satadesc = {
1077     optional => 1,
1078     type => 'string', format => $sata_fmt,
1079     description => "Use volume as SATA hard disk or CD-ROM (n is 0 to " . ($MAX_SATA_DISKS - 1). ").",
1080 };
1081 PVE::JSONSchema::register_standard_option("pve-qm-sata", $satadesc);
1082
1083 my $virtio_fmt = {
1084     %drivedesc_base,
1085     %iothread_fmt,
1086 };
1087 my $virtiodesc = {
1088     optional => 1,
1089     type => 'string', format => $virtio_fmt,
1090     description => "Use volume as VIRTIO hard disk (n is 0 to " . ($MAX_VIRTIO_DISKS - 1) . ").",
1091 };
1092 PVE::JSONSchema::register_standard_option("pve-qm-virtio", $virtiodesc);
1093
1094 my $alldrive_fmt = {
1095     %drivedesc_base,
1096     %iothread_fmt,
1097     %model_fmt,
1098     %queues_fmt,
1099     %scsiblock_fmt,
1100 };
1101
1102 my $efidisk_fmt = {
1103     volume => { alias => 'file' },
1104     file => {
1105         type => 'string',
1106         format => 'pve-volume-id-or-qm-path',
1107         default_key => 1,
1108         format_description => 'volume',
1109         description => "The drive's backing volume.",
1110     },
1111     format => get_standard_option('pve-qm-image-format'),
1112     size => {
1113         type => 'string',
1114         format => 'disk-size',
1115         format_description => 'DiskSize',
1116         description => "Disk size. This is purely informational and has no effect.",
1117         optional => 1,
1118     },
1119 };
1120
1121 my $efidisk_desc = {
1122     optional => 1,
1123     type => 'string', format => $efidisk_fmt,
1124     description => "Configure a Disk for storing EFI vars",
1125 };
1126
1127 PVE::JSONSchema::register_standard_option("pve-qm-efidisk", $efidisk_desc);
1128
1129 my $usb_fmt = {
1130     host => {
1131         default_key => 1,
1132         type => 'string', format => 'pve-qm-usb-device',
1133         format_description => 'HOSTUSBDEVICE|spice',
1134         description => <<EODESCR,
1135 The Host USB device or port or the value 'spice'. HOSTUSBDEVICE syntax is:
1136
1137  'bus-port(.port)*' (decimal numbers) or
1138  'vendor_id:product_id' (hexadeciaml numbers) or
1139  'spice'
1140
1141 You can use the 'lsusb -t' command to list existing usb devices.
1142
1143 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1144
1145 The value 'spice' can be used to add a usb redirection devices for spice.
1146 EODESCR
1147     },
1148     usb3 => {
1149         optional => 1,
1150         type => 'boolean',
1151         description => "Specifies whether if given host option is a USB3 device or port (this does currently not work reliably with spice redirection and is then ignored).",
1152         default => 0,
1153     },
1154 };
1155
1156 my $usbdesc = {
1157     optional => 1,
1158     type => 'string', format => $usb_fmt,
1159     description => "Configure an USB device (n is 0 to 4).",
1160 };
1161 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
1162
1163 # NOTE: the match-groups of this regex are used in parse_hostpci
1164 my $PCIRE = qr/([a-f0-9]{2}:[a-f0-9]{2})(?:\.([a-f0-9]))?/;
1165 my $hostpci_fmt = {
1166     host => {
1167         default_key => 1,
1168         type => 'string',
1169         pattern => qr/$PCIRE(;$PCIRE)*/,
1170         format_description => 'HOSTPCIID[;HOSTPCIID2...]',
1171         description => <<EODESCR,
1172 Host PCI device pass through. The PCI ID of a host's PCI device or a list
1173 of PCI virtual functions of the host. HOSTPCIID syntax is:
1174
1175 'bus:dev.func' (hexadecimal numbers)
1176
1177 You can us the 'lspci' command to list existing PCI devices.
1178 EODESCR
1179     },
1180     rombar => {
1181         type => 'boolean',
1182         description =>  "Specify whether or not the device's ROM will be visible in the guest's memory map.",
1183         optional => 1,
1184         default => 1,
1185     },
1186     romfile => {
1187         type => 'string',
1188         pattern => '[^,;]+',
1189         format_description => 'string',
1190         description => "Custom pci device rom filename (must be located in /usr/share/kvm/).",
1191         optional => 1,
1192     },
1193     pcie => {
1194         type => 'boolean',
1195         description =>  "Choose the PCI-express bus (needs the 'q35' machine model).",
1196         optional => 1,
1197         default => 0,
1198     },
1199     'x-vga' => {
1200         type => 'boolean',
1201         description =>  "Enable vfio-vga device support.",
1202         optional => 1,
1203         default => 0,
1204     },
1205 };
1206 PVE::JSONSchema::register_format('pve-qm-hostpci', $hostpci_fmt);
1207
1208 my $hostpcidesc = {
1209         optional => 1,
1210         type => 'string', format => 'pve-qm-hostpci',
1211         description => "Map host PCI devices into guest.",
1212         verbose_description =>  <<EODESCR,
1213 Map host PCI devices into guest.
1214
1215 NOTE: This option allows direct access to host hardware. So it is no longer
1216 possible to migrate such machines - use with special care.
1217
1218 CAUTION: Experimental! User reported problems with this option.
1219 EODESCR
1220 };
1221 PVE::JSONSchema::register_standard_option("pve-qm-hostpci", $hostpcidesc);
1222
1223 my $serialdesc = {
1224         optional => 1,
1225         type => 'string',
1226         pattern => '(/dev/.+|socket)',
1227         description =>  "Create a serial device inside the VM (n is 0 to 3)",
1228         verbose_description =>  <<EODESCR,
1229 Create a serial device inside the VM (n is 0 to 3), and pass through a
1230 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1231 host side (use 'qm terminal' to open a terminal connection).
1232
1233 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines - use with special care.
1234
1235 CAUTION: Experimental! User reported problems with this option.
1236 EODESCR
1237 };
1238
1239 my $paralleldesc= {
1240         optional => 1,
1241         type => 'string',
1242         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
1243         description =>  "Map host parallel devices (n is 0 to 2).",
1244         verbose_description =>  <<EODESCR,
1245 Map host parallel devices (n is 0 to 2).
1246
1247 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1248
1249 CAUTION: Experimental! User reported problems with this option.
1250 EODESCR
1251 };
1252
1253 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
1254     $confdesc->{"parallel$i"} = $paralleldesc;
1255 }
1256
1257 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
1258     $confdesc->{"serial$i"} = $serialdesc;
1259 }
1260
1261 for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
1262     $confdesc->{"hostpci$i"} = $hostpcidesc;
1263 }
1264
1265 for (my $i = 0; $i < $MAX_IDE_DISKS; $i++)  {
1266     $drivename_hash->{"ide$i"} = 1;
1267     $confdesc->{"ide$i"} = $idedesc;
1268 }
1269
1270 for (my $i = 0; $i < $MAX_SATA_DISKS; $i++)  {
1271     $drivename_hash->{"sata$i"} = 1;
1272     $confdesc->{"sata$i"} = $satadesc;
1273 }
1274
1275 for (my $i = 0; $i < $MAX_SCSI_DISKS; $i++)  {
1276     $drivename_hash->{"scsi$i"} = 1;
1277     $confdesc->{"scsi$i"} = $scsidesc ;
1278 }
1279
1280 for (my $i = 0; $i < $MAX_VIRTIO_DISKS; $i++)  {
1281     $drivename_hash->{"virtio$i"} = 1;
1282     $confdesc->{"virtio$i"} = $virtiodesc;
1283 }
1284
1285 $drivename_hash->{efidisk0} = 1;
1286 $confdesc->{efidisk0} = $efidisk_desc;
1287
1288 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
1289     $confdesc->{"usb$i"} = $usbdesc;
1290 }
1291
1292 my $unuseddesc = {
1293     optional => 1,
1294     type => 'string', format => 'pve-volume-id',
1295     description => "Reference to unused volumes. This is used internally, and should not be modified manually.",
1296 };
1297
1298 for (my $i = 0; $i < $MAX_UNUSED_DISKS; $i++)  {
1299     $confdesc->{"unused$i"} = $unuseddesc;
1300 }
1301
1302 my $kvm_api_version = 0;
1303
1304 sub kvm_version {
1305
1306     return $kvm_api_version if $kvm_api_version;
1307
1308     my $fh = IO::File->new("</dev/kvm") ||
1309         return 0;
1310
1311     if (my $v = $fh->ioctl(KVM_GET_API_VERSION(), 0)) {
1312         $kvm_api_version = $v;
1313     }
1314
1315     $fh->close();
1316
1317     return  $kvm_api_version;
1318 }
1319
1320 my $kvm_user_version;
1321
1322 sub kvm_user_version {
1323
1324     return $kvm_user_version if $kvm_user_version;
1325
1326     $kvm_user_version = 'unknown';
1327
1328     my $code = sub {
1329         my $line = shift;
1330         if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1331             $kvm_user_version = $2;
1332         }
1333     };
1334
1335     eval { run_command("kvm -version", outfunc => $code); };
1336     warn $@ if $@;
1337
1338     return $kvm_user_version;
1339
1340 }
1341
1342 my $kernel_has_vhost_net = -c '/dev/vhost-net';
1343
1344 sub valid_drive_names {
1345     # order is important - used to autoselect boot disk
1346     return ((map { "ide$_" } (0 .. ($MAX_IDE_DISKS - 1))),
1347             (map { "scsi$_" } (0 .. ($MAX_SCSI_DISKS - 1))),
1348             (map { "virtio$_" } (0 .. ($MAX_VIRTIO_DISKS - 1))),
1349             (map { "sata$_" } (0 .. ($MAX_SATA_DISKS - 1))),
1350             'efidisk0');
1351 }
1352
1353 sub is_valid_drivename {
1354     my $dev = shift;
1355
1356     return defined($drivename_hash->{$dev});
1357 }
1358
1359 sub option_exists {
1360     my $key = shift;
1361     return defined($confdesc->{$key});
1362 }
1363
1364 sub nic_models {
1365     return $nic_model_list;
1366 }
1367
1368 sub os_list_description {
1369
1370     return {
1371         other => 'Other',
1372         wxp => 'Windows XP',
1373         w2k => 'Windows 2000',
1374         w2k3 =>, 'Windows 2003',
1375         w2k8 => 'Windows 2008',
1376         wvista => 'Windows Vista',
1377         win7 => 'Windows 7',
1378         win8 => 'Windows 8/2012',
1379         win10 => 'Windows 10/2016',
1380         l24 => 'Linux 2.4',
1381         l26 => 'Linux 2.6',
1382     };
1383 }
1384
1385 my $cdrom_path;
1386
1387 sub get_cdrom_path {
1388
1389     return  $cdrom_path if $cdrom_path;
1390
1391     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
1392     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
1393     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
1394 }
1395
1396 sub get_iso_path {
1397     my ($storecfg, $vmid, $cdrom) = @_;
1398
1399     if ($cdrom eq 'cdrom') {
1400         return get_cdrom_path();
1401     } elsif ($cdrom eq 'none') {
1402         return '';
1403     } elsif ($cdrom =~ m|^/|) {
1404         return $cdrom;
1405     } else {
1406         return PVE::Storage::path($storecfg, $cdrom);
1407     }
1408 }
1409
1410 # try to convert old style file names to volume IDs
1411 sub filename_to_volume_id {
1412     my ($vmid, $file, $media) = @_;
1413
1414      if (!($file eq 'none' || $file eq 'cdrom' ||
1415           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1416
1417         return undef if $file =~ m|/|;
1418
1419         if ($media && $media eq 'cdrom') {
1420             $file = "local:iso/$file";
1421         } else {
1422             $file = "local:$vmid/$file";
1423         }
1424     }
1425
1426     return $file;
1427 }
1428
1429 sub verify_media_type {
1430     my ($opt, $vtype, $media) = @_;
1431
1432     return if !$media;
1433
1434     my $etype;
1435     if ($media eq 'disk') {
1436         $etype = 'images';
1437     } elsif ($media eq 'cdrom') {
1438         $etype = 'iso';
1439     } else {
1440         die "internal error";
1441     }
1442
1443     return if ($vtype eq $etype);
1444
1445     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1446 }
1447
1448 sub cleanup_drive_path {
1449     my ($opt, $storecfg, $drive) = @_;
1450
1451     # try to convert filesystem paths to volume IDs
1452
1453     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1454         ($drive->{file} !~ m|^/dev/.+|) &&
1455         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1456         ($drive->{file} !~ m/^\d+$/)) {
1457         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1458         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"}) if !$vtype;
1459         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1460         verify_media_type($opt, $vtype, $drive->{media});
1461         $drive->{file} = $volid;
1462     }
1463
1464     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1465 }
1466
1467 sub parse_hotplug_features {
1468     my ($data) = @_;
1469
1470     my $res = {};
1471
1472     return $res if $data eq '0';
1473
1474     $data = $confdesc->{hotplug}->{default} if $data eq '1';
1475
1476     foreach my $feature (PVE::Tools::split_list($data)) {
1477         if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
1478             $res->{$1} = 1;
1479         } else {
1480             die "invalid hotplug feature '$feature'\n";
1481         }
1482     }
1483     return $res;
1484 }
1485
1486 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1487 sub pve_verify_hotplug_features {
1488     my ($value, $noerr) = @_;
1489
1490     return $value if parse_hotplug_features($value);
1491
1492     return undef if $noerr;
1493
1494     die "unable to parse hotplug option\n";
1495 }
1496
1497 # ideX = [volume=]volume-id[,media=d][,cyls=c,heads=h,secs=s[,trans=t]]
1498 #        [,snapshot=on|off][,cache=on|off][,format=f][,backup=yes|no]
1499 #        [,rerror=ignore|report|stop][,werror=enospc|ignore|report|stop]
1500 #        [,aio=native|threads][,discard=ignore|on][,detect_zeroes=on|off]
1501 #        [,iothread=on][,serial=serial][,model=model]
1502
1503 sub parse_drive {
1504     my ($key, $data) = @_;
1505
1506     my ($interface, $index);
1507
1508     if ($key =~ m/^([^\d]+)(\d+)$/) {
1509         $interface = $1;
1510         $index = $2;
1511     } else {
1512         return undef;
1513     }
1514
1515     my $desc = $key =~ /^unused\d+$/ ? $alldrive_fmt
1516                                      : $confdesc->{$key}->{format};
1517     if (!$desc) {
1518         warn "invalid drive key: $key\n";
1519         return undef;
1520     }
1521     my $res = eval { PVE::JSONSchema::parse_property_string($desc, $data) };
1522     return undef if !$res;
1523     $res->{interface} = $interface;
1524     $res->{index} = $index;
1525
1526     my $error = 0;
1527     foreach my $opt (qw(bps bps_rd bps_wr)) {
1528         if (my $bps = defined(delete $res->{$opt})) {
1529             if (defined($res->{"m$opt"})) {
1530                 warn "both $opt and m$opt specified\n";
1531                 ++$error;
1532                 next;
1533             }
1534             $res->{"m$opt"} = sprintf("%.3f", $bps / (1024*1024.0));
1535         }
1536     }
1537
1538     # can't use the schema's 'requires' because of the mbps* => bps* "transforming aliases"
1539     for my $requirement (
1540         [mbps_max => 'mbps'],
1541         [mbps_rd_max => 'mbps_rd'],
1542         [mbps_wr_max => 'mbps_wr'],
1543         [miops_max => 'miops'],
1544         [miops_rd_max => 'miops_rd'],
1545         [miops_wr_max => 'miops_wr'],
1546         [bps_max_length => 'mbps_max'],
1547         [bps_rd_max_length => 'mbps_rd_max'],
1548         [bps_wr_max_length => 'mbps_wr_max'],
1549         [iops_max_length => 'iops_max'],
1550         [iops_rd_max_length => 'iops_rd_max'],
1551         [iops_wr_max_length => 'iops_wr_max']) {
1552         my ($option, $requires) = @$requirement;
1553         if ($res->{$option} && !$res->{$requires}) {
1554             warn "$option requires $requires\n";
1555             ++$error;
1556         }
1557     }
1558
1559     return undef if $error;
1560
1561     return undef if $res->{mbps_rd} && $res->{mbps};
1562     return undef if $res->{mbps_wr} && $res->{mbps};
1563     return undef if $res->{iops_rd} && $res->{iops};
1564     return undef if $res->{iops_wr} && $res->{iops};
1565
1566     if ($res->{media} && ($res->{media} eq 'cdrom')) {
1567         return undef if $res->{snapshot} || $res->{trans} || $res->{format};
1568         return undef if $res->{heads} || $res->{secs} || $res->{cyls};
1569         return undef if $res->{interface} eq 'virtio';
1570     }
1571
1572     if (my $size = $res->{size}) {
1573         return undef if !defined($res->{size} = PVE::JSONSchema::parse_size($size));
1574     }
1575
1576     return $res;
1577 }
1578
1579 sub print_drive {
1580     my ($vmid, $drive) = @_;
1581     my $data = { %$drive };
1582     delete $data->{$_} for qw(index interface);
1583     return PVE::JSONSchema::print_property_string($data, $alldrive_fmt);
1584 }
1585
1586 sub scsi_inquiry {
1587     my($fh, $noerr) = @_;
1588
1589     my $SG_IO = 0x2285;
1590     my $SG_GET_VERSION_NUM = 0x2282;
1591
1592     my $versionbuf = "\x00" x 8;
1593     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1594     if (!$ret) {
1595         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1596         return undef;
1597     }
1598     my $version = unpack("I", $versionbuf);
1599     if ($version < 30000) {
1600         die "scsi generic interface too old\n"  if !$noerr;
1601         return undef;
1602     }
1603
1604     my $buf = "\x00" x 36;
1605     my $sensebuf = "\x00" x 8;
1606     my $cmd = pack("C x3 C x1", 0x12, 36);
1607
1608     # see /usr/include/scsi/sg.h
1609     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1610
1611     my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1612                       length($sensebuf), 0, length($buf), $buf,
1613                       $cmd, $sensebuf, 6000);
1614
1615     $ret = ioctl($fh, $SG_IO, $packet);
1616     if (!$ret) {
1617         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1618         return undef;
1619     }
1620
1621     my @res = unpack($sg_io_hdr_t, $packet);
1622     if ($res[17] || $res[18]) {
1623         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1624         return undef;
1625     }
1626
1627     my $res = {};
1628     (my $byte0, my $byte1, $res->{vendor},
1629      $res->{product}, $res->{revision}) = unpack("C C x6 A8 A16 A4", $buf);
1630
1631     $res->{removable} = $byte1 & 128 ? 1 : 0;
1632     $res->{type} = $byte0 & 31;
1633
1634     return $res;
1635 }
1636
1637 sub path_is_scsi {
1638     my ($path) = @_;
1639
1640     my $fh = IO::File->new("+<$path") || return undef;
1641     my $res = scsi_inquiry($fh, 1);
1642     close($fh);
1643
1644     return $res;
1645 }
1646
1647 sub machine_type_is_q35 {
1648     my ($conf) = @_;
1649
1650     return $conf->{machine} && ($conf->{machine} =~ m/q35/) ? 1 : 0;
1651 }
1652
1653 sub print_tabletdevice_full {
1654     my ($conf) = @_;
1655
1656     my $q35 = machine_type_is_q35($conf);
1657
1658     # we use uhci for old VMs because tablet driver was buggy in older qemu
1659     my $usbbus = $q35 ? "ehci" : "uhci";
1660
1661     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1662 }
1663
1664 sub print_drivedevice_full {
1665     my ($storecfg, $conf, $vmid, $drive, $bridges) = @_;
1666
1667     my $device = '';
1668     my $maxdev = 0;
1669
1670     if ($drive->{interface} eq 'virtio') {
1671         my $pciaddr = print_pci_addr("$drive->{interface}$drive->{index}", $bridges);
1672         $device = "virtio-blk-pci,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}$pciaddr";
1673         $device .= ",iothread=iothread-$drive->{interface}$drive->{index}" if $drive->{iothread};
1674     } elsif ($drive->{interface} eq 'scsi') {
1675
1676         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1677         my $unit = $drive->{index} % $maxdev;
1678         my $devicetype = 'hd';
1679         my $path = '';
1680         if (drive_is_cdrom($drive)) {
1681             $devicetype = 'cd';
1682         } else {
1683             if ($drive->{file} =~ m|^/|) {
1684                 $path = $drive->{file};
1685                 if (my $info = path_is_scsi($path)) {
1686                     if ($info->{type} == 0 && $drive->{scsiblock}) {
1687                         $devicetype = 'block';
1688                     } elsif ($info->{type} == 1) { # tape
1689                         $devicetype = 'generic';
1690                     }
1691                 }
1692             } else {
1693                  $path = PVE::Storage::path($storecfg, $drive->{file});
1694             }
1695
1696             if($path =~ m/^iscsi\:\/\//){
1697                 $devicetype = 'generic';
1698             }
1699         }
1700
1701         if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)){
1702             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1703         } else {
1704             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0,lun=$drive->{index},drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1705         }
1706
1707     } elsif ($drive->{interface} eq 'ide'){
1708         $maxdev = 2;
1709         my $controller = int($drive->{index} / $maxdev);
1710         my $unit = $drive->{index} % $maxdev;
1711         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1712
1713         $device = "ide-$devicetype,bus=ide.$controller,unit=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1714         if ($devicetype eq 'hd' && (my $model = $drive->{model})) {
1715             $model = URI::Escape::uri_unescape($model);
1716             $device .= ",model=$model";
1717         }
1718     } elsif ($drive->{interface} eq 'sata'){
1719         my $controller = int($drive->{index} / $MAX_SATA_DISKS);
1720         my $unit = $drive->{index} % $MAX_SATA_DISKS;
1721         $device = "ide-drive,bus=ahci$controller.$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1722     } elsif ($drive->{interface} eq 'usb') {
1723         die "implement me";
1724         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1725     } else {
1726         die "unsupported interface type";
1727     }
1728
1729     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1730
1731     if (my $serial = $drive->{serial}) {
1732         $serial = URI::Escape::uri_unescape($serial);
1733         $device .= ",serial=$serial";
1734     }
1735
1736
1737     return $device;
1738 }
1739
1740 sub get_initiator_name {
1741     my $initiator;
1742
1743     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return undef;
1744     while (defined(my $line = <$fh>)) {
1745         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1746         $initiator = $1;
1747         last;
1748     }
1749     $fh->close();
1750
1751     return $initiator;
1752 }
1753
1754 sub print_drive_full {
1755     my ($storecfg, $vmid, $drive) = @_;
1756
1757     my $path;
1758     my $volid = $drive->{file};
1759     my $format;
1760
1761     if (drive_is_cdrom($drive)) {
1762         $path = get_iso_path($storecfg, $vmid, $volid);
1763     } else {
1764         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1765         if ($storeid) {
1766             $path = PVE::Storage::path($storecfg, $volid);
1767             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
1768             $format = qemu_img_format($scfg, $volname);
1769         } else {
1770             $path = $volid;
1771             $format = "raw";
1772         }
1773    }
1774
1775     my $opts = '';
1776     my @qemu_drive_options = qw(heads secs cyls trans media format cache rerror werror aio discard);
1777     foreach my $o (@qemu_drive_options) {
1778         $opts .= ",$o=$drive->{$o}" if defined($drive->{$o});
1779     }
1780
1781     # snapshot only accepts on|off
1782     if (defined($drive->{snapshot})) {
1783         my $v = $drive->{snapshot} ? 'on' : 'off';
1784         $opts .= ",snapshot=$v";
1785     }
1786
1787     foreach my $type (['', '-total'], [_rd => '-read'], [_wr => '-write']) {
1788         my ($dir, $qmpname) = @$type;
1789         if (my $v = $drive->{"mbps$dir"}) {
1790             $opts .= ",throttling.bps$qmpname=".int($v*1024*1024);
1791         }
1792         if (my $v = $drive->{"mbps${dir}_max"}) {
1793             $opts .= ",throttling.bps$qmpname-max=".int($v*1024*1024);
1794         }
1795         if (my $v = $drive->{"bps${dir}_max_length"}) {
1796             $opts .= ",throttling.bps$qmpname-max-length=$v";
1797         }
1798         if (my $v = $drive->{"iops${dir}"}) {
1799             $opts .= ",throttling.iops$qmpname=$v";
1800         }
1801         if (my $v = $drive->{"iops${dir}_max"}) {
1802             $opts .= ",throttling.iops$qmpname-max=$v";
1803         }
1804         if (my $v = $drive->{"iops${dir}_max_length"}) {
1805             $opts .= ",throttling.iops$qmpname-max-length=$v";
1806         }
1807     }
1808
1809     $opts .= ",format=$format" if $format && !$drive->{format};
1810
1811     my $cache_direct = 0;
1812
1813     if (my $cache = $drive->{cache}) {
1814         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1815     } elsif (!drive_is_cdrom($drive)) {
1816         $opts .= ",cache=none";
1817         $cache_direct = 1;
1818     }
1819
1820     # aio native works only with O_DIRECT
1821     if (!$drive->{aio}) {
1822         if($cache_direct) {
1823             $opts .= ",aio=native";
1824         } else {
1825             $opts .= ",aio=threads";
1826         }
1827     }
1828
1829     if (!drive_is_cdrom($drive)) {
1830         my $detectzeroes;
1831         if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
1832             $detectzeroes = 'off';
1833         } elsif ($drive->{discard}) {
1834             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
1835         } else {
1836             # This used to be our default with discard not being specified:
1837             $detectzeroes = 'on';
1838         }
1839         $opts .= ",detect-zeroes=$detectzeroes" if $detectzeroes;
1840     }
1841
1842     my $pathinfo = $path ? "file=$path," : '';
1843
1844     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1845 }
1846
1847 sub print_netdevice_full {
1848     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files) = @_;
1849
1850     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
1851
1852     my $device = $net->{model};
1853     if ($net->{model} eq 'virtio') {
1854          $device = 'virtio-net-pci';
1855      };
1856
1857     my $pciaddr = print_pci_addr("$netid", $bridges);
1858     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1859     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
1860         #Consider we have N queues, the number of vectors needed is 2*N + 2 (plus one config interrupt and control vq)
1861         my $vectors = $net->{queues} * 2 + 2;
1862         $tmpstr .= ",vectors=$vectors,mq=on";
1863     }
1864     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
1865
1866     if ($use_old_bios_files) {
1867         my $romfile;
1868         if ($device eq 'virtio-net-pci') {
1869             $romfile = 'pxe-virtio.rom';
1870         } elsif ($device eq 'e1000') {
1871             $romfile = 'pxe-e1000.rom';
1872         } elsif ($device eq 'ne2k') {
1873             $romfile = 'pxe-ne2k_pci.rom';
1874         } elsif ($device eq 'pcnet') {
1875             $romfile = 'pxe-pcnet.rom';
1876         } elsif ($device eq 'rtl8139') {
1877             $romfile = 'pxe-rtl8139.rom';
1878         }
1879         $tmpstr .= ",romfile=$romfile" if $romfile;
1880     }
1881
1882     return $tmpstr;
1883 }
1884
1885 sub print_netdev_full {
1886     my ($vmid, $conf, $net, $netid, $hotplug) = @_;
1887
1888     my $i = '';
1889     if ($netid =~ m/^net(\d+)$/) {
1890         $i = int($1);
1891     }
1892
1893     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
1894
1895     my $ifname = "tap${vmid}i$i";
1896
1897     # kvm uses TUNSETIFF ioctl, and that limits ifname length
1898     die "interface name '$ifname' is too long (max 15 character)\n"
1899         if length($ifname) >= 16;
1900
1901     my $vhostparam = '';
1902     $vhostparam = ',vhost=on' if $kernel_has_vhost_net && $net->{model} eq 'virtio';
1903
1904     my $vmname = $conf->{name} || "vm$vmid";
1905
1906     my $netdev = "";
1907     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
1908
1909     if ($net->{bridge}) {
1910         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script,downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1911     } else {
1912         $netdev = "type=user,id=$netid,hostname=$vmname";
1913     }
1914
1915     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
1916
1917     return $netdev;
1918 }
1919
1920
1921 sub print_cpu_device {
1922     my ($conf, $id) = @_;
1923
1924     my $kvm = $conf->{kvm} // 1;
1925     my $cpu = $kvm ? "kvm64" : "qemu64";
1926     if (my $cputype = $conf->{cpu}) {
1927         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
1928             or die "Cannot parse cpu description: $cputype\n";
1929         $cpu = $cpuconf->{cputype};
1930     }
1931
1932     my $cores = $conf->{cores} || 1;
1933
1934     my $current_core = ($id - 1) % $cores;
1935     my $current_socket = int(($id - 1 - $current_core)/$cores);
1936
1937     return "$cpu-x86_64-cpu,id=cpu$id,socket-id=$current_socket,core-id=$current_core,thread-id=0";
1938 }
1939
1940 sub drive_is_cloudinit {
1941     my ($drive) = @_;
1942     return $drive->{file} =~ m@[:/]vm-\d+-cloudinit(?:\.$QEMU_FORMAT_RE)?$@;
1943 }
1944
1945 sub drive_is_cdrom {
1946     my ($drive, $exclude_cloudinit) = @_;
1947
1948     return 0 if $exclude_cloudinit && drive_is_cloudinit($drive);
1949
1950     return $drive && $drive->{media} && ($drive->{media} eq 'cdrom');
1951
1952 }
1953
1954 sub parse_number_sets {
1955     my ($set) = @_;
1956     my $res = [];
1957     foreach my $part (split(/;/, $set)) {
1958         if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
1959             die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
1960             push @$res, [ $1, $2 ];
1961         } else {
1962             die "invalid range: $part\n";
1963         }
1964     }
1965     return $res;
1966 }
1967
1968 sub parse_numa {
1969     my ($data) = @_;
1970
1971     my $res = PVE::JSONSchema::parse_property_string($numa_fmt, $data);
1972     $res->{cpus} = parse_number_sets($res->{cpus}) if defined($res->{cpus});
1973     $res->{hostnodes} = parse_number_sets($res->{hostnodes}) if defined($res->{hostnodes});
1974     return $res;
1975 }
1976
1977 sub parse_hostpci {
1978     my ($value) = @_;
1979
1980     return undef if !$value;
1981
1982     my $res = PVE::JSONSchema::parse_property_string($hostpci_fmt, $value);
1983
1984     my @idlist = split(/;/, $res->{host});
1985     delete $res->{host};
1986     foreach my $id (@idlist) {
1987         if ($id =~ /^$PCIRE$/) {
1988             if (defined($2)) {
1989                 push @{$res->{pciid}}, { id => $1, function => $2 };
1990             } else {
1991                 my $pcidevices = lspci($1);
1992                 $res->{pciid} = $pcidevices->{$1};
1993             }
1994         } else {
1995             # should have been caught by parse_property_string already
1996             die "failed to parse PCI id: $id\n";
1997         }
1998     }
1999     return $res;
2000 }
2001
2002 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
2003 sub parse_net {
2004     my ($data) = @_;
2005
2006     my $res = eval { PVE::JSONSchema::parse_property_string($net_fmt, $data) };
2007     if ($@) {
2008         warn $@;
2009         return undef;
2010     }
2011     if (!defined($res->{macaddr})) {
2012         my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
2013         $res->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix});
2014     }
2015     return $res;
2016 }
2017
2018 # ipconfigX ip=cidr,gw=ip,ip6=cidr,gw6=ip
2019 sub parse_ipconfig {
2020     my ($data) = @_;
2021
2022     my $res = eval { PVE::JSONSchema::parse_property_string($ipconfig_fmt, $data) };
2023     if ($@) {
2024         warn $@;
2025         return undef;
2026     }
2027
2028     if ($res->{gw} && !$res->{ip}) {
2029         warn 'gateway specified without specifying an IP address';
2030         return undef;
2031     }
2032     if ($res->{gw6} && !$res->{ip6}) {
2033         warn 'IPv6 gateway specified without specifying an IPv6 address';
2034         return undef;
2035     }
2036     if ($res->{gw} && $res->{ip} eq 'dhcp') {
2037         warn 'gateway specified together with DHCP';
2038         return undef;
2039     }
2040     if ($res->{gw6} && $res->{ip6} !~ /^$IPV6RE/) {
2041         # gw6 + auto/dhcp
2042         warn "IPv6 gateway specified together with $res->{ip6} address";
2043         return undef;
2044     }
2045
2046     if (!$res->{ip} && !$res->{ip6}) {
2047         return { ip => 'dhcp', ip6 => 'dhcp' };
2048     }
2049
2050     return $res;
2051 }
2052
2053 sub print_net {
2054     my $net = shift;
2055
2056     return PVE::JSONSchema::print_property_string($net, $net_fmt);
2057 }
2058
2059 sub add_random_macs {
2060     my ($settings) = @_;
2061
2062     foreach my $opt (keys %$settings) {
2063         next if $opt !~ m/^net(\d+)$/;
2064         my $net = parse_net($settings->{$opt});
2065         next if !$net;
2066         $settings->{$opt} = print_net($net);
2067     }
2068 }
2069
2070 sub vm_is_volid_owner {
2071     my ($storecfg, $vmid, $volid) = @_;
2072
2073     if ($volid !~  m|^/|) {
2074         my ($path, $owner);
2075         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
2076         if ($owner && ($owner == $vmid)) {
2077             return 1;
2078         }
2079     }
2080
2081     return undef;
2082 }
2083
2084 sub split_flagged_list {
2085     my $text = shift || '';
2086     $text =~ s/[,;]/ /g;
2087     $text =~ s/^\s+//;
2088     return { map { /^(!?)(.*)$/ && ($2, $1) } ($text =~ /\S+/g) };
2089 }
2090
2091 sub join_flagged_list {
2092     my ($how, $lst) = @_;
2093     join $how, map { $lst->{$_} . $_ } keys %$lst;
2094 }
2095
2096 sub vmconfig_delete_pending_option {
2097     my ($conf, $key, $force) = @_;
2098
2099     delete $conf->{pending}->{$key};
2100     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
2101     $pending_delete_hash->{$key} = $force ? '!' : '';
2102     $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
2103 }
2104
2105 sub vmconfig_undelete_pending_option {
2106     my ($conf, $key) = @_;
2107
2108     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
2109     delete $pending_delete_hash->{$key};
2110
2111     if (%$pending_delete_hash) {
2112         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
2113     } else {
2114         delete $conf->{pending}->{delete};
2115     }
2116 }
2117
2118 sub vmconfig_register_unused_drive {
2119     my ($storecfg, $vmid, $conf, $drive) = @_;
2120
2121     if (drive_is_cloudinit($drive)) {
2122         eval { PVE::Storage::vdisk_free($storecfg, $drive->{file}) };
2123         warn $@ if $@;
2124     } elsif (!drive_is_cdrom($drive)) {
2125         my $volid = $drive->{file};
2126         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
2127             PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
2128         }
2129     }
2130 }
2131
2132 sub vmconfig_cleanup_pending {
2133     my ($conf) = @_;
2134
2135     # remove pending changes when nothing changed
2136     my $changes;
2137     foreach my $opt (keys %{$conf->{pending}}) {
2138         if (defined($conf->{$opt}) && ($conf->{pending}->{$opt} eq  $conf->{$opt})) {
2139             $changes = 1;
2140             delete $conf->{pending}->{$opt};
2141         }
2142     }
2143
2144     my $current_delete_hash = split_flagged_list($conf->{pending}->{delete});
2145     my $pending_delete_hash = {};
2146     while (my ($opt, $force) = each %$current_delete_hash) {
2147         if (defined($conf->{$opt})) {
2148             $pending_delete_hash->{$opt} = $force;
2149         } else {
2150             $changes = 1;
2151         }
2152     }
2153
2154     if (%$pending_delete_hash) {
2155         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
2156     } else {
2157         delete $conf->{pending}->{delete};
2158     }
2159
2160     return $changes;
2161 }
2162
2163 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str]
2164 my $smbios1_fmt = {
2165     uuid => {
2166         type => 'string',
2167         pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
2168         format_description => 'UUID',
2169         description => "Set SMBIOS1 UUID.",
2170         optional => 1,
2171     },
2172     version => {
2173         type => 'string',
2174         pattern => '\S+',
2175         format_description => 'string',
2176         description => "Set SMBIOS1 version.",
2177         optional => 1,
2178     },
2179     serial => {
2180         type => 'string',
2181         pattern => '\S+',
2182         format_description => 'string',
2183         description => "Set SMBIOS1 serial number.",
2184         optional => 1,
2185     },
2186     manufacturer => {
2187         type => 'string',
2188         pattern => '\S+',
2189         format_description => 'string',
2190         description => "Set SMBIOS1 manufacturer.",
2191         optional => 1,
2192     },
2193     product => {
2194         type => 'string',
2195         pattern => '\S+',
2196         format_description => 'string',
2197         description => "Set SMBIOS1 product ID.",
2198         optional => 1,
2199     },
2200     sku => {
2201         type => 'string',
2202         pattern => '\S+',
2203         format_description => 'string',
2204         description => "Set SMBIOS1 SKU string.",
2205         optional => 1,
2206     },
2207     family => {
2208         type => 'string',
2209         pattern => '\S+',
2210         format_description => 'string',
2211         description => "Set SMBIOS1 family string.",
2212         optional => 1,
2213     },
2214 };
2215
2216 sub parse_smbios1 {
2217     my ($data) = @_;
2218
2219     my $res = eval { PVE::JSONSchema::parse_property_string($smbios1_fmt, $data) };
2220     warn $@ if $@;
2221     return $res;
2222 }
2223
2224 sub print_smbios1 {
2225     my ($smbios1) = @_;
2226     return PVE::JSONSchema::print_property_string($smbios1, $smbios1_fmt);
2227 }
2228
2229 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_fmt);
2230
2231 PVE::JSONSchema::register_format('pve-qm-bootdisk', \&verify_bootdisk);
2232 sub verify_bootdisk {
2233     my ($value, $noerr) = @_;
2234
2235     return $value if is_valid_drivename($value);
2236
2237     return undef if $noerr;
2238
2239     die "invalid boot disk '$value'\n";
2240 }
2241
2242 sub parse_watchdog {
2243     my ($value) = @_;
2244
2245     return undef if !$value;
2246
2247     my $res = eval { PVE::JSONSchema::parse_property_string($watchdog_fmt, $value) };
2248     warn $@ if $@;
2249     return $res;
2250 }
2251
2252 sub parse_guest_agent {
2253     my ($value) = @_;
2254
2255     return {} if !defined($value->{agent});
2256
2257     my $res = eval { PVE::JSONSchema::parse_property_string($agent_fmt, $value->{agent}) };
2258     warn $@ if $@;
2259
2260     # if the agent is disabled ignore the other potentially set properties
2261     return {} if !$res->{enabled};
2262     return $res;
2263 }
2264
2265 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
2266 sub verify_usb_device {
2267     my ($value, $noerr) = @_;
2268
2269     return $value if parse_usb_device($value);
2270
2271     return undef if $noerr;
2272
2273     die "unable to parse usb device\n";
2274 }
2275
2276 # add JSON properties for create and set function
2277 sub json_config_properties {
2278     my $prop = shift;
2279
2280     foreach my $opt (keys %$confdesc) {
2281         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate' || $opt eq 'runningmachine';
2282         $prop->{$opt} = $confdesc->{$opt};
2283     }
2284
2285     return $prop;
2286 }
2287
2288 # return copy of $confdesc_cloudinit to generate documentation
2289 sub cloudinit_config_properties {
2290
2291     return dclone($confdesc_cloudinit);
2292 }
2293
2294 sub check_type {
2295     my ($key, $value) = @_;
2296
2297     die "unknown setting '$key'\n" if !$confdesc->{$key};
2298
2299     my $type = $confdesc->{$key}->{type};
2300
2301     if (!defined($value)) {
2302         die "got undefined value\n";
2303     }
2304
2305     if ($value =~ m/[\n\r]/) {
2306         die "property contains a line feed\n";
2307     }
2308
2309     if ($type eq 'boolean') {
2310         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
2311         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
2312         die "type check ('boolean') failed - got '$value'\n";
2313     } elsif ($type eq 'integer') {
2314         return int($1) if $value =~ m/^(\d+)$/;
2315         die "type check ('integer') failed - got '$value'\n";
2316     } elsif ($type eq 'number') {
2317         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
2318         die "type check ('number') failed - got '$value'\n";
2319     } elsif ($type eq 'string') {
2320         if (my $fmt = $confdesc->{$key}->{format}) {
2321             PVE::JSONSchema::check_format($fmt, $value);
2322             return $value;
2323         }
2324         $value =~ s/^\"(.*)\"$/$1/;
2325         return $value;
2326     } else {
2327         die "internal error"
2328     }
2329 }
2330
2331 sub check_iommu_support{
2332     #fixme : need to check IOMMU support
2333     #http://www.linux-kvm.org/page/How_to_assign_devices_with_VT-d_in_KVM
2334
2335     my $iommu=1;
2336     return $iommu;
2337
2338 }
2339
2340 sub touch_config {
2341     my ($vmid) = @_;
2342
2343     my $conf = PVE::QemuConfig->config_file($vmid);
2344     utime undef, undef, $conf;
2345 }
2346
2347 sub destroy_vm {
2348     my ($storecfg, $vmid, $keep_empty_config, $skiplock) = @_;
2349
2350     my $conffile = PVE::QemuConfig->config_file($vmid);
2351
2352     my $conf = PVE::QemuConfig->load_config($vmid);
2353
2354     PVE::QemuConfig->check_lock($conf) if !$skiplock;
2355
2356     if ($conf->{template}) {
2357         # check if any base image is still used by a linked clone
2358         foreach_drive($conf, sub {
2359                 my ($ds, $drive) = @_;
2360
2361                 return if drive_is_cdrom($drive);
2362
2363                 my $volid = $drive->{file};
2364
2365                 return if !$volid || $volid =~ m|^/|;
2366
2367                 die "base volume '$volid' is still in use by linked cloned\n"
2368                     if PVE::Storage::volume_is_base_and_used($storecfg, $volid);
2369
2370         });
2371     }
2372
2373     # only remove disks owned by this VM
2374     foreach_drive($conf, sub {
2375         my ($ds, $drive) = @_;
2376
2377         return if drive_is_cdrom($drive, 1);
2378
2379         my $volid = $drive->{file};
2380
2381         return if !$volid || $volid =~ m|^/|;
2382
2383         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
2384         return if !$path || !$owner || ($owner != $vmid);
2385
2386         eval {
2387             PVE::Storage::vdisk_free($storecfg, $volid);
2388         };
2389         warn "Could not remove disk '$volid', check manually: $@" if $@;
2390
2391     });
2392
2393     if ($keep_empty_config) {
2394         PVE::Tools::file_set_contents($conffile, "memory: 128\n");
2395     } else {
2396         unlink $conffile;
2397     }
2398
2399     # also remove unused disk
2400     eval {
2401         my $dl = PVE::Storage::vdisk_list($storecfg, undef, $vmid);
2402
2403         eval {
2404             PVE::Storage::foreach_volid($dl, sub {
2405                 my ($volid, $sid, $volname, $d) = @_;
2406                 PVE::Storage::vdisk_free($storecfg, $volid);
2407             });
2408         };
2409         warn $@ if $@;
2410
2411     };
2412     warn $@ if $@;
2413 }
2414
2415 sub parse_vm_config {
2416     my ($filename, $raw) = @_;
2417
2418     return undef if !defined($raw);
2419
2420     my $res = {
2421         digest => Digest::SHA::sha1_hex($raw),
2422         snapshots => {},
2423         pending => {},
2424     };
2425
2426     $filename =~ m|/qemu-server/(\d+)\.conf$|
2427         || die "got strange filename '$filename'";
2428
2429     my $vmid = $1;
2430
2431     my $conf = $res;
2432     my $descr;
2433     my $section = '';
2434
2435     my @lines = split(/\n/, $raw);
2436     foreach my $line (@lines) {
2437         next if $line =~ m/^\s*$/;
2438
2439         if ($line =~ m/^\[PENDING\]\s*$/i) {
2440             $section = 'pending';
2441             if (defined($descr)) {
2442                 $descr =~ s/\s+$//;
2443                 $conf->{description} = $descr;
2444             }
2445             $descr = undef;
2446             $conf = $res->{$section} = {};
2447             next;
2448
2449         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2450             $section = $1;
2451             if (defined($descr)) {
2452                 $descr =~ s/\s+$//;
2453                 $conf->{description} = $descr;
2454             }
2455             $descr = undef;
2456             $conf = $res->{snapshots}->{$section} = {};
2457             next;
2458         }
2459
2460         if ($line =~ m/^\#(.*)\s*$/) {
2461             $descr = '' if !defined($descr);
2462             $descr .= PVE::Tools::decode_text($1) . "\n";
2463             next;
2464         }
2465
2466         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2467             $descr = '' if !defined($descr);
2468             $descr .= PVE::Tools::decode_text($2);
2469         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2470             $conf->{snapstate} = $1;
2471         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2472             my $key = $1;
2473             my $value = $2;
2474             $conf->{$key} = $value;
2475         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2476             my $value = $1;
2477             if ($section eq 'pending') {
2478                 $conf->{delete} = $value; # we parse this later
2479             } else {
2480                 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
2481             }
2482         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(.+?)\s*$/) {
2483             my $key = $1;
2484             my $value = $2;
2485             eval { $value = check_type($key, $value); };
2486             if ($@) {
2487                 warn "vm $vmid - unable to parse value of '$key' - $@";
2488             } else {
2489                 $key = 'ide2' if $key eq 'cdrom';
2490                 my $fmt = $confdesc->{$key}->{format};
2491                 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2492                     my $v = parse_drive($key, $value);
2493                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2494                         $v->{file} = $volid;
2495                         $value = print_drive($vmid, $v);
2496                     } else {
2497                         warn "vm $vmid - unable to parse value of '$key'\n";
2498                         next;
2499                     }
2500                 }
2501
2502                 $conf->{$key} = $value;
2503             }
2504         }
2505     }
2506
2507     if (defined($descr)) {
2508         $descr =~ s/\s+$//;
2509         $conf->{description} = $descr;
2510     }
2511     delete $res->{snapstate}; # just to be sure
2512
2513     return $res;
2514 }
2515
2516 sub write_vm_config {
2517     my ($filename, $conf) = @_;
2518
2519     delete $conf->{snapstate}; # just to be sure
2520
2521     if ($conf->{cdrom}) {
2522         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2523         $conf->{ide2} = $conf->{cdrom};
2524         delete $conf->{cdrom};
2525     }
2526
2527     # we do not use 'smp' any longer
2528     if ($conf->{sockets}) {
2529         delete $conf->{smp};
2530     } elsif ($conf->{smp}) {
2531         $conf->{sockets} = $conf->{smp};
2532         delete $conf->{cores};
2533         delete $conf->{smp};
2534     }
2535
2536     my $used_volids = {};
2537
2538     my $cleanup_config = sub {
2539         my ($cref, $pending, $snapname) = @_;
2540
2541         foreach my $key (keys %$cref) {
2542             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2543                 $key eq 'snapstate' || $key eq 'pending';
2544             my $value = $cref->{$key};
2545             if ($key eq 'delete') {
2546                 die "propertry 'delete' is only allowed in [PENDING]\n"
2547                     if !$pending;
2548                 # fixme: check syntax?
2549                 next;
2550             }
2551             eval { $value = check_type($key, $value); };
2552             die "unable to parse value of '$key' - $@" if $@;
2553
2554             $cref->{$key} = $value;
2555
2556             if (!$snapname && is_valid_drivename($key)) {
2557                 my $drive = parse_drive($key, $value);
2558                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2559             }
2560         }
2561     };
2562
2563     &$cleanup_config($conf);
2564
2565     &$cleanup_config($conf->{pending}, 1);
2566
2567     foreach my $snapname (keys %{$conf->{snapshots}}) {
2568         die "internal error" if $snapname eq 'pending';
2569         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2570     }
2571
2572     # remove 'unusedX' settings if we re-add a volume
2573     foreach my $key (keys %$conf) {
2574         my $value = $conf->{$key};
2575         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2576             delete $conf->{$key};
2577         }
2578     }
2579
2580     my $generate_raw_config = sub {
2581         my ($conf, $pending) = @_;
2582
2583         my $raw = '';
2584
2585         # add description as comment to top of file
2586         if (defined(my $descr = $conf->{description})) {
2587             if ($descr) {
2588                 foreach my $cl (split(/\n/, $descr)) {
2589                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2590                 }
2591             } else {
2592                 $raw .= "#\n" if $pending;
2593             }
2594         }
2595
2596         foreach my $key (sort keys %$conf) {
2597             next if $key eq 'digest' || $key eq 'description' || $key eq 'pending' || $key eq 'snapshots';
2598             $raw .= "$key: $conf->{$key}\n";
2599         }
2600         return $raw;
2601     };
2602
2603     my $raw = &$generate_raw_config($conf);
2604
2605     if (scalar(keys %{$conf->{pending}})){
2606         $raw .= "\n[PENDING]\n";
2607         $raw .= &$generate_raw_config($conf->{pending}, 1);
2608     }
2609
2610     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2611         $raw .= "\n[$snapname]\n";
2612         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2613     }
2614
2615     return $raw;
2616 }
2617
2618 sub load_defaults {
2619
2620     my $res = {};
2621
2622     # we use static defaults from our JSON schema configuration
2623     foreach my $key (keys %$confdesc) {
2624         if (defined(my $default = $confdesc->{$key}->{default})) {
2625             $res->{$key} = $default;
2626         }
2627     }
2628
2629     return $res;
2630 }
2631
2632 sub config_list {
2633     my $vmlist = PVE::Cluster::get_vmlist();
2634     my $res = {};
2635     return $res if !$vmlist || !$vmlist->{ids};
2636     my $ids = $vmlist->{ids};
2637
2638     foreach my $vmid (keys %$ids) {
2639         my $d = $ids->{$vmid};
2640         next if !$d->{node} || $d->{node} ne $nodename;
2641         next if !$d->{type} || $d->{type} ne 'qemu';
2642         $res->{$vmid}->{exists} = 1;
2643     }
2644     return $res;
2645 }
2646
2647 # test if VM uses local resources (to prevent migration)
2648 sub check_local_resources {
2649     my ($conf, $noerr) = @_;
2650
2651     my $loc_res = 0;
2652
2653     $loc_res = 1 if $conf->{hostusb}; # old syntax
2654     $loc_res = 1 if $conf->{hostpci}; # old syntax
2655
2656     foreach my $k (keys %$conf) {
2657         next if $k =~ m/^usb/ && ($conf->{$k} eq 'spice');
2658         # sockets are safe: they will recreated be on the target side post-migrate
2659         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2660         $loc_res = 1 if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2661     }
2662
2663     die "VM uses local resources\n" if $loc_res && !$noerr;
2664
2665     return $loc_res;
2666 }
2667
2668 # check if used storages are available on all nodes (use by migrate)
2669 sub check_storage_availability {
2670     my ($storecfg, $conf, $node) = @_;
2671
2672     foreach_drive($conf, sub {
2673         my ($ds, $drive) = @_;
2674
2675         my $volid = $drive->{file};
2676         return if !$volid;
2677
2678         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2679         return if !$sid;
2680
2681         # check if storage is available on both nodes
2682         my $scfg = PVE::Storage::storage_check_node($storecfg, $sid);
2683         PVE::Storage::storage_check_node($storecfg, $sid, $node);
2684    });
2685 }
2686
2687 # list nodes where all VM images are available (used by has_feature API)
2688 sub shared_nodes {
2689     my ($conf, $storecfg) = @_;
2690
2691     my $nodelist = PVE::Cluster::get_nodelist();
2692     my $nodehash = { map { $_ => 1 } @$nodelist };
2693     my $nodename = PVE::INotify::nodename();
2694
2695     foreach_drive($conf, sub {
2696         my ($ds, $drive) = @_;
2697
2698         my $volid = $drive->{file};
2699         return if !$volid;
2700
2701         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2702         if ($storeid) {
2703             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2704             if ($scfg->{disable}) {
2705                 $nodehash = {};
2706             } elsif (my $avail = $scfg->{nodes}) {
2707                 foreach my $node (keys %$nodehash) {
2708                     delete $nodehash->{$node} if !$avail->{$node};
2709                 }
2710             } elsif (!$scfg->{shared}) {
2711                 foreach my $node (keys %$nodehash) {
2712                     delete $nodehash->{$node} if $node ne $nodename
2713                 }
2714             }
2715         }
2716     });
2717
2718     return $nodehash
2719 }
2720
2721 sub check_cmdline {
2722     my ($pidfile, $pid) = @_;
2723
2724     my $fh = IO::File->new("/proc/$pid/cmdline", "r");
2725     if (defined($fh)) {
2726         my $line = <$fh>;
2727         $fh->close;
2728         return undef if !$line;
2729         my @param = split(/\0/, $line);
2730
2731         my $cmd = $param[0];
2732         return if !$cmd || ($cmd !~ m|kvm$| && $cmd !~ m|qemu-system-x86_64$|);
2733
2734         for (my $i = 0; $i < scalar (@param); $i++) {
2735             my $p = $param[$i];
2736             next if !$p;
2737             if (($p eq '-pidfile') || ($p eq '--pidfile')) {
2738                 my $p = $param[$i+1];
2739                 return 1 if $p && ($p eq $pidfile);
2740                 return undef;
2741             }
2742         }
2743     }
2744     return undef;
2745 }
2746
2747 sub check_running {
2748     my ($vmid, $nocheck, $node) = @_;
2749
2750     my $filename = PVE::QemuConfig->config_file($vmid, $node);
2751
2752     die "unable to find configuration file for VM $vmid - no such machine\n"
2753         if !$nocheck && ! -f $filename;
2754
2755     my $pidfile = pidfile_name($vmid);
2756
2757     if (my $fd = IO::File->new("<$pidfile")) {
2758         my $st = stat($fd);
2759         my $line = <$fd>;
2760         close($fd);
2761
2762         my $mtime = $st->mtime;
2763         if ($mtime > time()) {
2764             warn "file '$filename' modified in future\n";
2765         }
2766
2767         if ($line =~ m/^(\d+)$/) {
2768             my $pid = $1;
2769             if (check_cmdline($pidfile, $pid)) {
2770                 if (my $pinfo = PVE::ProcFSTools::check_process_running($pid)) {
2771                     return $pid;
2772                 }
2773             }
2774         }
2775     }
2776
2777     return undef;
2778 }
2779
2780 sub vzlist {
2781
2782     my $vzlist = config_list();
2783
2784     my $fd = IO::Dir->new($var_run_tmpdir) || return $vzlist;
2785
2786     while (defined(my $de = $fd->read)) {
2787         next if $de !~ m/^(\d+)\.pid$/;
2788         my $vmid = $1;
2789         next if !defined($vzlist->{$vmid});
2790         if (my $pid = check_running($vmid)) {
2791             $vzlist->{$vmid}->{pid} = $pid;
2792         }
2793     }
2794
2795     return $vzlist;
2796 }
2797
2798 sub disksize {
2799     my ($storecfg, $conf) = @_;
2800
2801     my $bootdisk = $conf->{bootdisk};
2802     return undef if !$bootdisk;
2803     return undef if !is_valid_drivename($bootdisk);
2804
2805     return undef if !$conf->{$bootdisk};
2806
2807     my $drive = parse_drive($bootdisk, $conf->{$bootdisk});
2808     return undef if !defined($drive);
2809
2810     return undef if drive_is_cdrom($drive);
2811
2812     my $volid = $drive->{file};
2813     return undef if !$volid;
2814
2815     return $drive->{size};
2816 }
2817
2818 our $vmstatus_return_properties = {
2819     vmid => get_standard_option('pve-vmid'),
2820     status => {
2821         description => "Qemu process status.",
2822         type => 'string',
2823         enum => ['stopped', 'running'],
2824     },
2825     maxmem => {
2826         description => "Maximum memory in bytes.",
2827         type => 'integer',
2828         optional => 1,
2829         renderer => 'bytes',
2830     },
2831     maxdisk => {
2832         description => "Root disk size in bytes.",
2833         type => 'integer',
2834         optional => 1,
2835         renderer => 'bytes',
2836     },
2837     name => {
2838         description => "VM name.",
2839         type => 'string',
2840         optional => 1,
2841     },
2842     qmpstatus => {
2843         description => "Qemu QMP agent status.",
2844         type => 'string',
2845         optional => 1,
2846     },
2847     pid => {
2848         description => "PID of running qemu process.",
2849         type => 'integer',
2850         optional => 1,
2851     },
2852     uptime => {
2853         description => "Uptime.",
2854         type => 'integer',
2855         optional => 1,
2856         renderer => 'duration',
2857     },
2858     cpus => {
2859         description => "Maximum usable CPUs.",
2860         type => 'number',
2861         optional => 1,
2862     },
2863 };
2864
2865 my $last_proc_pid_stat;
2866
2867 # get VM status information
2868 # This must be fast and should not block ($full == false)
2869 # We only query KVM using QMP if $full == true (this can be slow)
2870 sub vmstatus {
2871     my ($opt_vmid, $full) = @_;
2872
2873     my $res = {};
2874
2875     my $storecfg = PVE::Storage::config();
2876
2877     my $list = vzlist();
2878     my $defaults = load_defaults();
2879
2880     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
2881
2882     my $cpucount = $cpuinfo->{cpus} || 1;
2883
2884     foreach my $vmid (keys %$list) {
2885         next if $opt_vmid && ($vmid ne $opt_vmid);
2886
2887         my $cfspath = PVE::QemuConfig->cfs_config_path($vmid);
2888         my $conf = PVE::Cluster::cfs_read_file($cfspath) || {};
2889
2890         my $d = { vmid => $vmid };
2891         $d->{pid} = $list->{$vmid}->{pid};
2892
2893         # fixme: better status?
2894         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
2895
2896         my $size = disksize($storecfg, $conf);
2897         if (defined($size)) {
2898             $d->{disk} = 0; # no info available
2899             $d->{maxdisk} = $size;
2900         } else {
2901             $d->{disk} = 0;
2902             $d->{maxdisk} = 0;
2903         }
2904
2905         $d->{cpus} = ($conf->{sockets} || $defaults->{sockets})
2906             * ($conf->{cores} || $defaults->{cores});
2907         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
2908         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
2909
2910         $d->{name} = $conf->{name} || "VM $vmid";
2911         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024)
2912             : $defaults->{memory}*(1024*1024);
2913
2914         if ($conf->{balloon}) {
2915             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
2916             $d->{shares} = defined($conf->{shares}) ? $conf->{shares}
2917                 : $defaults->{shares};
2918         }
2919
2920         $d->{uptime} = 0;
2921         $d->{cpu} = 0;
2922         $d->{mem} = 0;
2923
2924         $d->{netout} = 0;
2925         $d->{netin} = 0;
2926
2927         $d->{diskread} = 0;
2928         $d->{diskwrite} = 0;
2929
2930         $d->{template} = PVE::QemuConfig->is_template($conf);
2931
2932         $d->{serial} = 1 if conf_has_serial($conf);
2933
2934         $res->{$vmid} = $d;
2935     }
2936
2937     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
2938     foreach my $dev (keys %$netdev) {
2939         next if $dev !~ m/^tap([1-9]\d*)i/;
2940         my $vmid = $1;
2941         my $d = $res->{$vmid};
2942         next if !$d;
2943
2944         $d->{netout} += $netdev->{$dev}->{receive};
2945         $d->{netin} += $netdev->{$dev}->{transmit};
2946
2947         if ($full) {
2948             $d->{nics}->{$dev}->{netout} = $netdev->{$dev}->{receive};
2949             $d->{nics}->{$dev}->{netin} = $netdev->{$dev}->{transmit};
2950         }
2951
2952     }
2953
2954     my $ctime = gettimeofday;
2955
2956     foreach my $vmid (keys %$list) {
2957
2958         my $d = $res->{$vmid};
2959         my $pid = $d->{pid};
2960         next if !$pid;
2961
2962         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
2963         next if !$pstat; # not running
2964
2965         my $used = $pstat->{utime} + $pstat->{stime};
2966
2967         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
2968
2969         if ($pstat->{vsize}) {
2970             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
2971         }
2972
2973         my $old = $last_proc_pid_stat->{$pid};
2974         if (!$old) {
2975             $last_proc_pid_stat->{$pid} = {
2976                 time => $ctime,
2977                 used => $used,
2978                 cpu => 0,
2979             };
2980             next;
2981         }
2982
2983         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
2984
2985         if ($dtime > 1000) {
2986             my $dutime = $used -  $old->{used};
2987
2988             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
2989             $last_proc_pid_stat->{$pid} = {
2990                 time => $ctime,
2991                 used => $used,
2992                 cpu => $d->{cpu},
2993             };
2994         } else {
2995             $d->{cpu} = $old->{cpu};
2996         }
2997     }
2998
2999     return $res if !$full;
3000
3001     my $qmpclient = PVE::QMPClient->new();
3002
3003     my $ballooncb = sub {
3004         my ($vmid, $resp) = @_;
3005
3006         my $info = $resp->{'return'};
3007         return if !$info->{max_mem};
3008
3009         my $d = $res->{$vmid};
3010
3011         # use memory assigned to VM
3012         $d->{maxmem} = $info->{max_mem};
3013         $d->{balloon} = $info->{actual};
3014
3015         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
3016             $d->{mem} = $info->{total_mem} - $info->{free_mem};
3017             $d->{freemem} = $info->{free_mem};
3018         }
3019
3020         $d->{ballooninfo} = $info;
3021     };
3022
3023     my $blockstatscb = sub {
3024         my ($vmid, $resp) = @_;
3025         my $data = $resp->{'return'} || [];
3026         my $totalrdbytes = 0;
3027         my $totalwrbytes = 0;
3028
3029         for my $blockstat (@$data) {
3030             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
3031             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
3032
3033             $blockstat->{device} =~ s/drive-//;
3034             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
3035         }
3036         $res->{$vmid}->{diskread} = $totalrdbytes;
3037         $res->{$vmid}->{diskwrite} = $totalwrbytes;
3038     };
3039
3040     my $statuscb = sub {
3041         my ($vmid, $resp) = @_;
3042
3043         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
3044         # this fails if ballon driver is not loaded, so this must be
3045         # the last commnand (following command are aborted if this fails).
3046         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
3047
3048         my $status = 'unknown';
3049         if (!defined($status = $resp->{'return'}->{status})) {
3050             warn "unable to get VM status\n";
3051             return;
3052         }
3053
3054         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
3055     };
3056
3057     foreach my $vmid (keys %$list) {
3058         next if $opt_vmid && ($vmid ne $opt_vmid);
3059         next if !$res->{$vmid}->{pid}; # not running
3060         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
3061     }
3062
3063     $qmpclient->queue_execute(undef, 2);
3064
3065     foreach my $vmid (keys %$list) {
3066         next if $opt_vmid && ($vmid ne $opt_vmid);
3067         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
3068     }
3069
3070     return $res;
3071 }
3072
3073 sub foreach_drive {
3074     my ($conf, $func, @param) = @_;
3075
3076     foreach my $ds (valid_drive_names()) {
3077         next if !defined($conf->{$ds});
3078
3079         my $drive = parse_drive($ds, $conf->{$ds});
3080         next if !$drive;
3081
3082         &$func($ds, $drive, @param);
3083     }
3084 }
3085
3086 sub foreach_volid {
3087     my ($conf, $func, @param) = @_;
3088
3089     my $volhash = {};
3090
3091     my $test_volid = sub {
3092         my ($volid, $is_cdrom, $replicate, $shared, $snapname) = @_;
3093
3094         return if !$volid;
3095
3096         $volhash->{$volid}->{cdrom} //= 1;
3097         $volhash->{$volid}->{cdrom} = 0 if !$is_cdrom;
3098
3099         $volhash->{$volid}->{replicate} //= 0;
3100         $volhash->{$volid}->{replicate} = 1 if $replicate;
3101
3102         $volhash->{$volid}->{shared} //= 0;
3103         $volhash->{$volid}->{shared} = 1 if $shared;
3104
3105         $volhash->{$volid}->{referenced_in_config} //= 0;
3106         $volhash->{$volid}->{referenced_in_config} = 1 if !defined($snapname);
3107
3108         $volhash->{$volid}->{referenced_in_snapshot}->{$snapname} = 1
3109             if defined($snapname);
3110     };
3111
3112     foreach_drive($conf, sub {
3113         my ($ds, $drive) = @_;
3114         $test_volid->($drive->{file}, drive_is_cdrom($drive), $drive->{replicate} // 1, $drive->{shared}, undef);
3115     });
3116
3117     foreach my $snapname (keys %{$conf->{snapshots}}) {
3118         my $snap = $conf->{snapshots}->{$snapname};
3119         $test_volid->($snap->{vmstate}, 0, 1, $snapname);
3120         foreach_drive($snap, sub {
3121             my ($ds, $drive) = @_;
3122             $test_volid->($drive->{file}, drive_is_cdrom($drive), $drive->{replicate} // 1, $drive->{shared}, $snapname);
3123         });
3124     }
3125
3126     foreach my $volid (keys %$volhash) {
3127         &$func($volid, $volhash->{$volid}, @param);
3128     }
3129 }
3130
3131 sub conf_has_serial {
3132     my ($conf) = @_;
3133
3134     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3135         if ($conf->{"serial$i"}) {
3136             return 1;
3137         }
3138     }
3139
3140     return 0;
3141 }
3142
3143 sub vga_conf_has_spice {
3144     my ($vga) = @_;
3145
3146     return 0 if !$vga || $vga !~ m/^qxl([234])?$/;
3147
3148     return $1 || 1;
3149 }
3150
3151 sub config_to_command {
3152     my ($storecfg, $vmid, $conf, $defaults, $forcemachine) = @_;
3153
3154     my $cmd = [];
3155     my $globalFlags = [];
3156     my $machineFlags = [];
3157     my $rtcFlags = [];
3158     my $cpuFlags = [];
3159     my $devices = [];
3160     my $pciaddr = '';
3161     my $bridges = {};
3162     my $kvmver = kvm_user_version();
3163     my $vernum = 0; # unknown
3164     my $ostype = $conf->{ostype};
3165     my $winversion = windows_version($ostype);
3166     my $kvm = $conf->{kvm} // 1;
3167
3168     die "KVM virtualisation configured, but not available. Either disable in VM configuration or enable in BIOS.\n" if (!$cpuinfo->{hvm} && $kvm);
3169
3170     if ($kvmver =~ m/^(\d+)\.(\d+)$/) {
3171         $vernum = $1*1000000+$2*1000;
3172     } elsif ($kvmver =~ m/^(\d+)\.(\d+)\.(\d+)$/) {
3173         $vernum = $1*1000000+$2*1000+$3;
3174     }
3175
3176     die "detected old qemu-kvm binary ($kvmver)\n" if $vernum < 15000;
3177
3178     my $have_ovz = -f '/proc/vz/vestat';
3179
3180     my $q35 = machine_type_is_q35($conf);
3181     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
3182     my $machine_type = $forcemachine || $conf->{machine};
3183     my $use_old_bios_files = undef;
3184     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3185
3186     my $cpuunits = defined($conf->{cpuunits}) ?
3187             $conf->{cpuunits} : $defaults->{cpuunits};
3188
3189     push @$cmd, '/usr/bin/kvm';
3190
3191     push @$cmd, '-id', $vmid;
3192
3193     my $vmname = $conf->{name} || "vm$vmid";
3194
3195     push @$cmd, '-name', $vmname;
3196
3197     my $use_virtio = 0;
3198
3199     my $qmpsocket = qmp_socket($vmid);
3200     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server,nowait";
3201     push @$cmd, '-mon', "chardev=qmp,mode=control";
3202
3203     if (qemu_machine_feature_enabled($machine_type, $kvmver, 2, 12)) {
3204         my $eventsocket = qmp_socket($vmid, 0, 'event');
3205         push @$cmd, '-chardev', "socket,id=qmp-event,path=$eventsocket,server,nowait";
3206         push @$cmd, '-mon', "chardev=qmp-event,mode=control";
3207     }
3208
3209     push @$cmd, '-pidfile' , pidfile_name($vmid);
3210
3211     push @$cmd, '-daemonize';
3212
3213     if ($conf->{smbios1}) {
3214         push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
3215     }
3216
3217     if ($conf->{vmgenid}) {
3218         push @$devices, '-device', 'vmgenid,guid='.$conf->{vmgenid};
3219     }
3220
3221     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3222         die "uefi base image not found\n" if ! -f $OVMF_CODE;
3223
3224         my $path;
3225         my $format;
3226         if (my $efidisk = $conf->{efidisk0}) {
3227             my $d = PVE::JSONSchema::parse_property_string($efidisk_fmt, $efidisk);
3228             my ($storeid, $volname) = PVE::Storage::parse_volume_id($d->{file}, 1);
3229             $format = $d->{format};
3230             if ($storeid) {
3231                 $path = PVE::Storage::path($storecfg, $d->{file});
3232                 if (!defined($format)) {
3233                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
3234                     $format = qemu_img_format($scfg, $volname);
3235                 }
3236             } else {
3237                 $path = $d->{file};
3238                 die "efidisk format must be specified\n"
3239                     if !defined($format);
3240             }
3241         } else {
3242             warn "no efidisk configured! Using temporary efivars disk.\n";
3243             $path = "/tmp/$vmid-ovmf.fd";
3244             PVE::Tools::file_copy($OVMF_VARS, $path, -s $OVMF_VARS);
3245             $format = 'raw';
3246         }
3247
3248         push @$cmd, '-drive', "if=pflash,unit=0,format=raw,readonly,file=$OVMF_CODE";
3249         push @$cmd, '-drive', "if=pflash,unit=1,format=$format,id=drive-efidisk0,file=$path";
3250     }
3251
3252
3253     # add usb controllers
3254     my @usbcontrollers = PVE::QemuServer::USB::get_usb_controllers($conf, $bridges, $q35, $usbdesc->{format}, $MAX_USB_DEVICES);
3255     push @$devices, @usbcontrollers if @usbcontrollers;
3256     my $vga = $conf->{vga};
3257
3258     my $qxlnum = vga_conf_has_spice($vga);
3259     $vga = 'qxl' if $qxlnum;
3260
3261     if (!$vga) {
3262         if (qemu_machine_feature_enabled($machine_type, $kvmver, 2, 9)) {
3263             $vga = (!$winversion || $winversion >= 6) ? 'std' : 'cirrus';
3264         } else {
3265             $vga = ($winversion >= 6) ? 'std' : 'cirrus';
3266         }
3267     }
3268
3269     # enable absolute mouse coordinates (needed by vnc)
3270     my $tablet;
3271     if (defined($conf->{tablet})) {
3272         $tablet = $conf->{tablet};
3273     } else {
3274         $tablet = $defaults->{tablet};
3275         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
3276         $tablet = 0 if $vga =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
3277     }
3278
3279     push @$devices, '-device', print_tabletdevice_full($conf) if $tablet;
3280
3281     my $kvm_off = 0;
3282     my $gpu_passthrough;
3283
3284     # host pci devices
3285     for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
3286         my $d = parse_hostpci($conf->{"hostpci$i"});
3287         next if !$d;
3288
3289         my $pcie = $d->{pcie};
3290         if($pcie){
3291             die "q35 machine model is not enabled" if !$q35;
3292             $pciaddr = print_pcie_addr("hostpci$i");
3293         }else{
3294             $pciaddr = print_pci_addr("hostpci$i", $bridges);
3295         }
3296
3297         my $rombar = defined($d->{rombar}) && !$d->{rombar} ? ',rombar=0' : '';
3298         my $romfile = $d->{romfile};
3299
3300         my $xvga = '';
3301         if ($d->{'x-vga'}) {
3302             $xvga = ',x-vga=on';
3303             $kvm_off = 1;
3304             $vga = 'none';
3305             $gpu_passthrough = 1;
3306
3307             if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3308                 $xvga = "";
3309             }
3310         }
3311         my $pcidevices = $d->{pciid};
3312         my $multifunction = 1 if @$pcidevices > 1;
3313
3314         my $j=0;
3315         foreach my $pcidevice (@$pcidevices) {
3316
3317             my $id = "hostpci$i";
3318             $id .= ".$j" if $multifunction;
3319             my $addr = $pciaddr;
3320             $addr .= ".$j" if $multifunction;
3321             my $devicestr = "vfio-pci,host=$pcidevice->{id}.$pcidevice->{function},id=$id$addr";
3322
3323             if($j == 0){
3324                 $devicestr .= "$rombar$xvga";
3325                 $devicestr .= ",multifunction=on" if $multifunction;
3326                 $devicestr .= ",romfile=/usr/share/kvm/$romfile" if $romfile;
3327             }
3328
3329             push @$devices, '-device', $devicestr;
3330             $j++;
3331         }
3332     }
3333
3334     # usb devices
3335     my @usbdevices = PVE::QemuServer::USB::get_usb_devices($conf, $usbdesc->{format}, $MAX_USB_DEVICES);
3336     push @$devices, @usbdevices if @usbdevices;
3337     # serial devices
3338     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3339         if (my $path = $conf->{"serial$i"}) {
3340             if ($path eq 'socket') {
3341                 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
3342                 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server,nowait";
3343                 push @$devices, '-device', "isa-serial,chardev=serial$i";
3344             } else {
3345                 die "no such serial device\n" if ! -c $path;
3346                 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
3347                 push @$devices, '-device', "isa-serial,chardev=serial$i";
3348             }
3349         }
3350     }
3351
3352     # parallel devices
3353     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
3354         if (my $path = $conf->{"parallel$i"}) {
3355             die "no such parallel device\n" if ! -c $path;
3356             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
3357             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
3358             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
3359         }
3360     }
3361
3362
3363     my $sockets = 1;
3364     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3365     $sockets = $conf->{sockets} if  $conf->{sockets};
3366
3367     my $cores = $conf->{cores} || 1;
3368
3369     my $maxcpus = $sockets * $cores;
3370
3371     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
3372
3373     my $allowed_vcpus = $cpuinfo->{cpus};
3374
3375     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n"
3376         if ($allowed_vcpus < $maxcpus);
3377
3378     if($hotplug_features->{cpu} && qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 7)) {
3379
3380         push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3381         for (my $i = 2; $i <= $vcpus; $i++)  {
3382             my $cpustr = print_cpu_device($conf,$i);
3383             push @$cmd, '-device', $cpustr;
3384         }
3385
3386     } else {
3387
3388         push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3389     }
3390     push @$cmd, '-nodefaults';
3391
3392     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
3393
3394     my $bootindex_hash = {};
3395     my $i = 1;
3396     foreach my $o (split(//, $bootorder)) {
3397         $bootindex_hash->{$o} = $i*100;
3398         $i++;
3399     }
3400
3401     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
3402
3403     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
3404
3405     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
3406
3407     push @$cmd, '-vga', $vga if $vga && $vga !~ m/^serial\d+$/; # for kvm 77 and later
3408
3409     if ($vga && $vga !~ m/^serial\d+$/ && $vga ne 'none'){
3410         my $socket = vnc_socket($vmid);
3411         push @$cmd,  '-vnc', "unix:$socket,x509,password";
3412     } else {
3413         push @$cmd, '-nographic';
3414     }
3415
3416     # time drift fix
3417     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
3418
3419     my $useLocaltime = $conf->{localtime};
3420
3421     if ($winversion >= 5) { # windows
3422         $useLocaltime = 1 if !defined($conf->{localtime});
3423
3424         # use time drift fix when acpi is enabled
3425         if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
3426             $tdf = 1 if !defined($conf->{tdf});
3427         }
3428     }
3429
3430     if ($winversion >= 6) {
3431         push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3432         push @$cmd, '-no-hpet';
3433     }
3434
3435     push @$rtcFlags, 'driftfix=slew' if $tdf;
3436
3437     if (!$kvm) {
3438         push @$machineFlags, 'accel=tcg';
3439     }
3440
3441     if ($machine_type) {
3442         push @$machineFlags, "type=${machine_type}";
3443     }
3444
3445     if ($conf->{startdate}) {
3446         push @$rtcFlags, "base=$conf->{startdate}";
3447     } elsif ($useLocaltime) {
3448         push @$rtcFlags, 'base=localtime';
3449     }
3450
3451     my $cpu = $kvm ? "kvm64" : "qemu64";
3452     if (my $cputype = $conf->{cpu}) {
3453         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
3454             or die "Cannot parse cpu description: $cputype\n";
3455         $cpu = $cpuconf->{cputype};
3456         $kvm_off = 1 if $cpuconf->{hidden};
3457
3458         if (defined(my $flags = $cpuconf->{flags})) {
3459             push @$cpuFlags, split(";", $flags);
3460         }
3461     }
3462
3463     push @$cpuFlags , '+lahf_lm' if $cpu eq 'kvm64';
3464
3465     push @$cpuFlags , '-x2apic'
3466         if $conf->{ostype} && $conf->{ostype} eq 'solaris';
3467
3468     push @$cpuFlags, '+sep' if $cpu eq 'kvm64' || $cpu eq 'kvm32';
3469
3470     push @$cpuFlags, '-rdtscp' if $cpu =~ m/^Opteron/;
3471
3472     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3473
3474         push @$cpuFlags , '+kvm_pv_unhalt' if $kvm;
3475         push @$cpuFlags , '+kvm_pv_eoi' if $kvm;
3476     }
3477
3478     add_hyperv_enlightenments($cpuFlags, $winversion, $machine_type, $kvmver, $conf->{bios}, $gpu_passthrough) if $kvm;
3479
3480     push @$cpuFlags, 'enforce' if $cpu ne 'host' && $kvm;
3481
3482     push @$cpuFlags, 'kvm=off' if $kvm_off;
3483
3484     my $cpu_vendor = $cpu_vendor_list->{$cpu} ||
3485         die "internal error"; # should not happen
3486
3487     push @$cpuFlags, "vendor=${cpu_vendor}"
3488         if $cpu_vendor ne 'default';
3489
3490     $cpu .= "," . join(',', @$cpuFlags) if scalar(@$cpuFlags);
3491
3492     push @$cmd, '-cpu', $cpu;
3493
3494     PVE::QemuServer::Memory::config($conf, $vmid, $sockets, $cores, $defaults, $hotplug_features, $cmd);
3495
3496     push @$cmd, '-S' if $conf->{freeze};
3497
3498     push @$cmd, '-k', $conf->{keyboard} if defined($conf->{keyboard});
3499
3500     # enable sound
3501     #my $soundhw = $conf->{soundhw} || $defaults->{soundhw};
3502     #push @$cmd, '-soundhw', 'es1370';
3503     #push @$cmd, '-soundhw', $soundhw if $soundhw;
3504
3505     if (parse_guest_agent($conf)->{enabled}) {
3506         my $qgasocket = qmp_socket($vmid, 1);
3507         my $pciaddr = print_pci_addr("qga0", $bridges);
3508         push @$devices, '-chardev', "socket,path=$qgasocket,server,nowait,id=qga0";
3509         push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3510         push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3511     }
3512
3513     my $spice_port;
3514
3515     if ($qxlnum) {
3516         if ($qxlnum > 1) {
3517             if ($winversion){
3518                 for(my $i = 1; $i < $qxlnum; $i++){
3519                     my $pciaddr = print_pci_addr("vga$i", $bridges);
3520                     push @$devices, '-device', "qxl,id=vga$i,ram_size=67108864,vram_size=33554432$pciaddr";
3521                 }
3522             } else {
3523                 # assume other OS works like Linux
3524                 push @$cmd, '-global', 'qxl-vga.ram_size=134217728';
3525                 push @$cmd, '-global', 'qxl-vga.vram_size=67108864';
3526             }
3527         }
3528
3529         my $pciaddr = print_pci_addr("spice", $bridges);
3530
3531         my $nodename = PVE::INotify::nodename();
3532         my $pfamily = PVE::Tools::get_host_address_family($nodename);
3533         my @nodeaddrs = PVE::Tools::getaddrinfo_all('localhost', family => $pfamily);
3534         die "failed to get an ip address of type $pfamily for 'localhost'\n" if !@nodeaddrs;
3535         my $localhost = PVE::Network::addr_to_ip($nodeaddrs[0]->{addr});
3536         $spice_port = PVE::Tools::next_spice_port($pfamily, $localhost);
3537
3538         push @$devices, '-spice', "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";
3539
3540         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3541         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3542         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3543     }
3544
3545     # enable balloon by default, unless explicitly disabled
3546     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3547         $pciaddr = print_pci_addr("balloon0", $bridges);
3548         push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
3549     }
3550
3551     if ($conf->{watchdog}) {
3552         my $wdopts = parse_watchdog($conf->{watchdog});
3553         $pciaddr = print_pci_addr("watchdog", $bridges);
3554         my $watchdog = $wdopts->{model} || 'i6300esb';
3555         push @$devices, '-device', "$watchdog$pciaddr";
3556         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3557     }
3558
3559     my $vollist = [];
3560     my $scsicontroller = {};
3561     my $ahcicontroller = {};
3562     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3563
3564     # Add iscsi initiator name if available
3565     if (my $initiator = get_initiator_name()) {
3566         push @$devices, '-iscsi', "initiator-name=$initiator";
3567     }
3568
3569     foreach_drive($conf, sub {
3570         my ($ds, $drive) = @_;
3571
3572         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3573             push @$vollist, $drive->{file};
3574         }
3575
3576         # ignore efidisk here, already added in bios/fw handling code above
3577         return if $drive->{interface} eq 'efidisk';
3578
3579         $use_virtio = 1 if $ds =~ m/^virtio/;
3580
3581         if (drive_is_cdrom ($drive)) {
3582             if ($bootindex_hash->{d}) {
3583                 $drive->{bootindex} = $bootindex_hash->{d};
3584                 $bootindex_hash->{d} += 1;
3585             }
3586         } else {
3587             if ($bootindex_hash->{c}) {
3588                 $drive->{bootindex} = $bootindex_hash->{c} if $conf->{bootdisk} && ($conf->{bootdisk} eq $ds);
3589                 $bootindex_hash->{c} += 1;
3590             }
3591         }
3592
3593         if($drive->{interface} eq 'virtio'){
3594            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3595         }
3596
3597         if ($drive->{interface} eq 'scsi') {
3598
3599             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
3600
3601             $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges);
3602             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
3603
3604             my $iothread = '';
3605             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
3606                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3607                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3608             } elsif ($drive->{iothread}) {
3609                 warn "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n";
3610             }
3611
3612             my $queues = '';
3613             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
3614                 $queues = ",num_queues=$drive->{queues}";
3615             }
3616
3617             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues" if !$scsicontroller->{$controller};
3618             $scsicontroller->{$controller}=1;
3619         }
3620
3621         if ($drive->{interface} eq 'sata') {
3622            my $controller = int($drive->{index} / $MAX_SATA_DISKS);
3623            $pciaddr = print_pci_addr("ahci$controller", $bridges);
3624            push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr" if !$ahcicontroller->{$controller};
3625            $ahcicontroller->{$controller}=1;
3626         }
3627
3628         my $drive_cmd = print_drive_full($storecfg, $vmid, $drive);
3629         push @$devices, '-drive',$drive_cmd;
3630         push @$devices, '-device', print_drivedevice_full($storecfg, $conf, $vmid, $drive, $bridges);
3631     });
3632
3633     for (my $i = 0; $i < $MAX_NETS; $i++) {
3634          next if !$conf->{"net$i"};
3635          my $d = parse_net($conf->{"net$i"});
3636          next if !$d;
3637
3638          $use_virtio = 1 if $d->{model} eq 'virtio';
3639
3640          if ($bootindex_hash->{n}) {
3641             $d->{bootindex} = $bootindex_hash->{n};
3642             $bootindex_hash->{n} += 1;
3643          }
3644
3645          my $netdevfull = print_netdev_full($vmid,$conf,$d,"net$i");
3646          push @$devices, '-netdev', $netdevfull;
3647
3648          my $netdevicefull = print_netdevice_full($vmid, $conf, $d, "net$i", $bridges, $use_old_bios_files);
3649          push @$devices, '-device', $netdevicefull;
3650     }
3651
3652     if (!$q35) {
3653         # add pci bridges
3654         if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3655            $bridges->{1} = 1;
3656            $bridges->{2} = 1;
3657         }
3658
3659         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
3660
3661         while (my ($k, $v) = each %$bridges) {
3662             $pciaddr = print_pci_addr("pci.$k");
3663             unshift @$devices, '-device', "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr" if $k > 0;
3664         }
3665     }
3666
3667     # add custom args
3668     if ($conf->{args}) {
3669         my $aa = PVE::Tools::split_args($conf->{args});
3670         push @$cmd, @$aa;
3671     }
3672
3673     push @$cmd, @$devices;
3674     push @$cmd, '-rtc', join(',', @$rtcFlags)
3675         if scalar(@$rtcFlags);
3676     push @$cmd, '-machine', join(',', @$machineFlags)
3677         if scalar(@$machineFlags);
3678     push @$cmd, '-global', join(',', @$globalFlags)
3679         if scalar(@$globalFlags);
3680
3681     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
3682 }
3683
3684 sub vnc_socket {
3685     my ($vmid) = @_;
3686     return "${var_run_tmpdir}/$vmid.vnc";
3687 }
3688
3689 sub spice_port {
3690     my ($vmid) = @_;
3691
3692     my $res = vm_mon_cmd($vmid, 'query-spice');
3693
3694     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
3695 }
3696
3697 sub qmp_socket {
3698     my ($vmid, $qga, $name) = @_;
3699     my $sockettype = $qga ? 'qga' : 'qmp';
3700     my $ext = $name ? '-'.$name : '';
3701     return "${var_run_tmpdir}/$vmid$ext.$sockettype";
3702 }
3703
3704 sub pidfile_name {
3705     my ($vmid) = @_;
3706     return "${var_run_tmpdir}/$vmid.pid";
3707 }
3708
3709 sub vm_devices_list {
3710     my ($vmid) = @_;
3711
3712     my $res = vm_mon_cmd($vmid, 'query-pci');
3713     my $devices_to_check = [];
3714     my $devices = {};
3715     foreach my $pcibus (@$res) {
3716         push @$devices_to_check, @{$pcibus->{devices}},
3717     }
3718
3719     while (@$devices_to_check) {
3720         my $to_check = [];
3721         for my $d (@$devices_to_check) {
3722             $devices->{$d->{'qdev_id'}} = 1 if $d->{'qdev_id'};
3723             next if !$d->{'pci_bridge'};
3724
3725             $devices->{$d->{'qdev_id'}} += scalar(@{$d->{'pci_bridge'}->{devices}});
3726             push @$to_check, @{$d->{'pci_bridge'}->{devices}};
3727         }
3728         $devices_to_check = $to_check;
3729     }
3730
3731     my $resblock = vm_mon_cmd($vmid, 'query-block');
3732     foreach my $block (@$resblock) {
3733         if($block->{device} =~ m/^drive-(\S+)/){
3734                 $devices->{$1} = 1;
3735         }
3736     }
3737
3738     my $resmice = vm_mon_cmd($vmid, 'query-mice');
3739     foreach my $mice (@$resmice) {
3740         if ($mice->{name} eq 'QEMU HID Tablet') {
3741             $devices->{tablet} = 1;
3742             last;
3743         }
3744     }
3745
3746     # for usb devices there is no query-usb
3747     # but we can iterate over the entries in
3748     # qom-list path=/machine/peripheral
3749     my $resperipheral = vm_mon_cmd($vmid, 'qom-list', path => '/machine/peripheral');
3750     foreach my $per (@$resperipheral) {
3751         if ($per->{name} =~ m/^usb\d+$/) {
3752             $devices->{$per->{name}} = 1;
3753         }
3754     }
3755
3756     return $devices;
3757 }
3758
3759 sub vm_deviceplug {
3760     my ($storecfg, $conf, $vmid, $deviceid, $device) = @_;
3761
3762     my $q35 = machine_type_is_q35($conf);
3763
3764     my $devices_list = vm_devices_list($vmid);
3765     return 1 if defined($devices_list->{$deviceid});
3766
3767     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid); # add PCI bridge if we need it for the device
3768
3769     if ($deviceid eq 'tablet') {
3770
3771         qemu_deviceadd($vmid, print_tabletdevice_full($conf));
3772
3773     } elsif ($deviceid =~ m/^usb(\d+)$/) {
3774
3775         die "usb hotplug currently not reliable\n";
3776         # since we can't reliably hot unplug all added usb devices
3777         # and usb passthrough disables live migration
3778         # we disable usb hotplugging for now
3779         qemu_deviceadd($vmid, PVE::QemuServer::USB::print_usbdevice_full($conf, $deviceid, $device));
3780
3781     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3782
3783         qemu_iothread_add($vmid, $deviceid, $device);
3784
3785         qemu_driveadd($storecfg, $vmid, $device);
3786         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
3787
3788         qemu_deviceadd($vmid, $devicefull);
3789         eval { qemu_deviceaddverify($vmid, $deviceid); };
3790         if (my $err = $@) {
3791             eval { qemu_drivedel($vmid, $deviceid); };
3792             warn $@ if $@;
3793             die $err;
3794         }
3795
3796     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3797
3798
3799         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
3800         my $pciaddr = print_pci_addr($deviceid);
3801         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
3802
3803         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
3804
3805         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
3806             qemu_iothread_add($vmid, $deviceid, $device);
3807             $devicefull .= ",iothread=iothread-$deviceid";
3808         }
3809
3810         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
3811             $devicefull .= ",num_queues=$device->{queues}";
3812         }
3813
3814         qemu_deviceadd($vmid, $devicefull);
3815         qemu_deviceaddverify($vmid, $deviceid);
3816
3817     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3818
3819         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device);
3820         qemu_driveadd($storecfg, $vmid, $device);
3821
3822         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
3823         eval { qemu_deviceadd($vmid, $devicefull); };
3824         if (my $err = $@) {
3825             eval { qemu_drivedel($vmid, $deviceid); };
3826             warn $@ if $@;
3827             die $err;
3828         }
3829
3830     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3831
3832         return undef if !qemu_netdevadd($vmid, $conf, $device, $deviceid);
3833
3834         my $machine_type = PVE::QemuServer::qemu_machine_pxe($vmid, $conf);
3835         my $use_old_bios_files = undef;
3836         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3837
3838         my $netdevicefull = print_netdevice_full($vmid, $conf, $device, $deviceid, undef, $use_old_bios_files);
3839         qemu_deviceadd($vmid, $netdevicefull);
3840         eval { qemu_deviceaddverify($vmid, $deviceid); };
3841         if (my $err = $@) {
3842             eval { qemu_netdevdel($vmid, $deviceid); };
3843             warn $@ if $@;
3844             die $err;
3845         }
3846
3847     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
3848
3849         my $bridgeid = $2;
3850         my $pciaddr = print_pci_addr($deviceid);
3851         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
3852
3853         qemu_deviceadd($vmid, $devicefull);
3854         qemu_deviceaddverify($vmid, $deviceid);
3855
3856     } else {
3857         die "can't hotplug device '$deviceid'\n";
3858     }
3859
3860     return 1;
3861 }
3862
3863 # fixme: this should raise exceptions on error!
3864 sub vm_deviceunplug {
3865     my ($vmid, $conf, $deviceid) = @_;
3866
3867     my $devices_list = vm_devices_list($vmid);
3868     return 1 if !defined($devices_list->{$deviceid});
3869
3870     die "can't unplug bootdisk" if $conf->{bootdisk} && $conf->{bootdisk} eq $deviceid;
3871
3872     if ($deviceid eq 'tablet') {
3873
3874         qemu_devicedel($vmid, $deviceid);
3875
3876     } elsif ($deviceid =~ m/^usb\d+$/) {
3877
3878         die "usb hotplug currently not reliable\n";
3879         # when unplugging usb devices this way,
3880         # there may be remaining usb controllers/hubs
3881         # so we disable it for now
3882         qemu_devicedel($vmid, $deviceid);
3883         qemu_devicedelverify($vmid, $deviceid);
3884
3885     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3886
3887         qemu_devicedel($vmid, $deviceid);
3888         qemu_devicedelverify($vmid, $deviceid);
3889         qemu_drivedel($vmid, $deviceid);
3890         qemu_iothread_del($conf, $vmid, $deviceid);
3891
3892     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3893
3894         qemu_devicedel($vmid, $deviceid);
3895         qemu_devicedelverify($vmid, $deviceid);
3896         qemu_iothread_del($conf, $vmid, $deviceid);
3897
3898     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3899
3900         qemu_devicedel($vmid, $deviceid);
3901         qemu_drivedel($vmid, $deviceid);
3902         qemu_deletescsihw($conf, $vmid, $deviceid);
3903
3904     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3905
3906         qemu_devicedel($vmid, $deviceid);
3907         qemu_devicedelverify($vmid, $deviceid);
3908         qemu_netdevdel($vmid, $deviceid);
3909
3910     } else {
3911         die "can't unplug device '$deviceid'\n";
3912     }
3913
3914     return 1;
3915 }
3916
3917 sub qemu_deviceadd {
3918     my ($vmid, $devicefull) = @_;
3919
3920     $devicefull = "driver=".$devicefull;
3921     my %options =  split(/[=,]/, $devicefull);
3922
3923     vm_mon_cmd($vmid, "device_add" , %options);
3924 }
3925
3926 sub qemu_devicedel {
3927     my ($vmid, $deviceid) = @_;
3928
3929     my $ret = vm_mon_cmd($vmid, "device_del", id => $deviceid);
3930 }
3931
3932 sub qemu_iothread_add {
3933     my($vmid, $deviceid, $device) = @_;
3934
3935     if ($device->{iothread}) {
3936         my $iothreads = vm_iothreads_list($vmid);
3937         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
3938     }
3939 }
3940
3941 sub qemu_iothread_del {
3942     my($conf, $vmid, $deviceid) = @_;
3943
3944     my $device = parse_drive($deviceid, $conf->{$deviceid});
3945     if ($device->{iothread}) {
3946         my $iothreads = vm_iothreads_list($vmid);
3947         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
3948     }
3949 }
3950
3951 sub qemu_objectadd {
3952     my($vmid, $objectid, $qomtype) = @_;
3953
3954     vm_mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
3955
3956     return 1;
3957 }
3958
3959 sub qemu_objectdel {
3960     my($vmid, $objectid) = @_;
3961
3962     vm_mon_cmd($vmid, "object-del", id => $objectid);
3963
3964     return 1;
3965 }
3966
3967 sub qemu_driveadd {
3968     my ($storecfg, $vmid, $device) = @_;
3969
3970     my $drive = print_drive_full($storecfg, $vmid, $device);
3971     $drive =~ s/\\/\\\\/g;
3972     my $ret = vm_human_monitor_command($vmid, "drive_add auto \"$drive\"");
3973
3974     # If the command succeeds qemu prints: "OK"
3975     return 1 if $ret =~ m/OK/s;
3976
3977     die "adding drive failed: $ret\n";
3978 }
3979
3980 sub qemu_drivedel {
3981     my($vmid, $deviceid) = @_;
3982
3983     my $ret = vm_human_monitor_command($vmid, "drive_del drive-$deviceid");
3984     $ret =~ s/^\s+//;
3985
3986     return 1 if $ret eq "";
3987
3988     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
3989     return 1 if $ret =~ m/Device \'.*?\' not found/s;
3990
3991     die "deleting drive $deviceid failed : $ret\n";
3992 }
3993
3994 sub qemu_deviceaddverify {
3995     my ($vmid, $deviceid) = @_;
3996
3997     for (my $i = 0; $i <= 5; $i++) {
3998          my $devices_list = vm_devices_list($vmid);
3999          return 1 if defined($devices_list->{$deviceid});
4000          sleep 1;
4001     }
4002
4003     die "error on hotplug device '$deviceid'\n";
4004 }
4005
4006
4007 sub qemu_devicedelverify {
4008     my ($vmid, $deviceid) = @_;
4009
4010     # need to verify that the device is correctly removed as device_del
4011     # is async and empty return is not reliable
4012
4013     for (my $i = 0; $i <= 5; $i++) {
4014          my $devices_list = vm_devices_list($vmid);
4015          return 1 if !defined($devices_list->{$deviceid});
4016          sleep 1;
4017     }
4018
4019     die "error on hot-unplugging device '$deviceid'\n";
4020 }
4021
4022 sub qemu_findorcreatescsihw {
4023     my ($storecfg, $conf, $vmid, $device) = @_;
4024
4025     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4026
4027     my $scsihwid="$controller_prefix$controller";
4028     my $devices_list = vm_devices_list($vmid);
4029
4030     if(!defined($devices_list->{$scsihwid})) {
4031         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device);
4032     }
4033
4034     return 1;
4035 }
4036
4037 sub qemu_deletescsihw {
4038     my ($conf, $vmid, $opt) = @_;
4039
4040     my $device = parse_drive($opt, $conf->{$opt});
4041
4042     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
4043         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
4044         return 1;
4045     }
4046
4047     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4048
4049     my $devices_list = vm_devices_list($vmid);
4050     foreach my $opt (keys %{$devices_list}) {
4051         if (PVE::QemuServer::is_valid_drivename($opt)) {
4052             my $drive = PVE::QemuServer::parse_drive($opt, $conf->{$opt});
4053             if($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
4054                 return 1;
4055             }
4056         }
4057     }
4058
4059     my $scsihwid="scsihw$controller";
4060
4061     vm_deviceunplug($vmid, $conf, $scsihwid);
4062
4063     return 1;
4064 }
4065
4066 sub qemu_add_pci_bridge {
4067     my ($storecfg, $conf, $vmid, $device) = @_;
4068
4069     my $bridges = {};
4070
4071     my $bridgeid;
4072
4073     print_pci_addr($device, $bridges);
4074
4075     while (my ($k, $v) = each %$bridges) {
4076         $bridgeid = $k;
4077     }
4078     return 1 if !defined($bridgeid) || $bridgeid < 1;
4079
4080     my $bridge = "pci.$bridgeid";
4081     my $devices_list = vm_devices_list($vmid);
4082
4083     if (!defined($devices_list->{$bridge})) {
4084         vm_deviceplug($storecfg, $conf, $vmid, $bridge);
4085     }
4086
4087     return 1;
4088 }
4089
4090 sub qemu_set_link_status {
4091     my ($vmid, $device, $up) = @_;
4092
4093     vm_mon_cmd($vmid, "set_link", name => $device,
4094                up => $up ? JSON::true : JSON::false);
4095 }
4096
4097 sub qemu_netdevadd {
4098     my ($vmid, $conf, $device, $deviceid) = @_;
4099
4100     my $netdev = print_netdev_full($vmid, $conf, $device, $deviceid, 1);
4101     my %options =  split(/[=,]/, $netdev);
4102
4103     vm_mon_cmd($vmid, "netdev_add",  %options);
4104     return 1;
4105 }
4106
4107 sub qemu_netdevdel {
4108     my ($vmid, $deviceid) = @_;
4109
4110     vm_mon_cmd($vmid, "netdev_del", id => $deviceid);
4111 }
4112
4113 sub qemu_usb_hotplug {
4114     my ($storecfg, $conf, $vmid, $deviceid, $device) = @_;
4115
4116     return if !$device;
4117
4118     # remove the old one first
4119     vm_deviceunplug($vmid, $conf, $deviceid);
4120
4121     # check if xhci controller is necessary and available
4122     if ($device->{usb3}) {
4123
4124         my $devicelist = vm_devices_list($vmid);
4125
4126         if (!$devicelist->{xhci}) {
4127             my $pciaddr = print_pci_addr("xhci");
4128             qemu_deviceadd($vmid, "nec-usb-xhci,id=xhci$pciaddr");
4129         }
4130     }
4131     my $d = parse_usb_device($device->{host});
4132     $d->{usb3} = $device->{usb3};
4133
4134     # add the new one
4135     vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $d);
4136 }
4137
4138 sub qemu_cpu_hotplug {
4139     my ($vmid, $conf, $vcpus) = @_;
4140
4141     my $machine_type = PVE::QemuServer::get_current_qemu_machine($vmid);
4142
4143     my $sockets = 1;
4144     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
4145     $sockets = $conf->{sockets} if  $conf->{sockets};
4146     my $cores = $conf->{cores} || 1;
4147     my $maxcpus = $sockets * $cores;
4148
4149     $vcpus = $maxcpus if !$vcpus;
4150
4151     die "you can't add more vcpus than maxcpus\n"
4152         if $vcpus > $maxcpus;
4153
4154     my $currentvcpus = $conf->{vcpus} || $maxcpus;
4155
4156     if ($vcpus < $currentvcpus) {
4157
4158         if (qemu_machine_feature_enabled ($machine_type, undef, 2, 7)) {
4159
4160             for (my $i = $currentvcpus; $i > $vcpus; $i--) {
4161                 qemu_devicedel($vmid, "cpu$i");
4162                 my $retry = 0;
4163                 my $currentrunningvcpus = undef;
4164                 while (1) {
4165                     $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
4166                     last if scalar(@{$currentrunningvcpus}) == $i-1;
4167                     raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
4168                     $retry++;
4169                     sleep 1;
4170                 }
4171                 #update conf after each succesfull cpu unplug
4172                 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4173                 PVE::QemuConfig->write_config($vmid, $conf);
4174             }
4175         } else {
4176             die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
4177         }
4178
4179         return;
4180     }
4181
4182     my $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
4183     die "vcpus in running vm does not match its configuration\n"
4184         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
4185
4186     if (qemu_machine_feature_enabled ($machine_type, undef, 2, 7)) {
4187
4188         for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
4189             my $cpustr = print_cpu_device($conf, $i);
4190             qemu_deviceadd($vmid, $cpustr);
4191
4192             my $retry = 0;
4193             my $currentrunningvcpus = undef;
4194             while (1) {
4195                 $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
4196                 last if scalar(@{$currentrunningvcpus}) == $i;
4197                 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
4198                 sleep 1;
4199                 $retry++;
4200             }
4201             #update conf after each succesfull cpu hotplug
4202             $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4203             PVE::QemuConfig->write_config($vmid, $conf);
4204         }
4205     } else {
4206
4207         for (my $i = $currentvcpus; $i < $vcpus; $i++) {
4208             vm_mon_cmd($vmid, "cpu-add", id => int($i));
4209         }
4210     }
4211 }
4212
4213 sub qemu_block_set_io_throttle {
4214     my ($vmid, $deviceid,
4215         $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
4216         $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max,
4217         $bps_max_length, $bps_rd_max_length, $bps_wr_max_length,
4218         $iops_max_length, $iops_rd_max_length, $iops_wr_max_length) = @_;
4219
4220     return if !check_running($vmid) ;
4221
4222     vm_mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
4223         bps => int($bps),
4224         bps_rd => int($bps_rd),
4225         bps_wr => int($bps_wr),
4226         iops => int($iops),
4227         iops_rd => int($iops_rd),
4228         iops_wr => int($iops_wr),
4229         bps_max => int($bps_max),
4230         bps_rd_max => int($bps_rd_max),
4231         bps_wr_max => int($bps_wr_max),
4232         iops_max => int($iops_max),
4233         iops_rd_max => int($iops_rd_max),
4234         iops_wr_max => int($iops_wr_max),
4235         bps_max_length => int($bps_max_length),
4236         bps_rd_max_length => int($bps_rd_max_length),
4237         bps_wr_max_length => int($bps_wr_max_length),
4238         iops_max_length => int($iops_max_length),
4239         iops_rd_max_length => int($iops_rd_max_length),
4240         iops_wr_max_length => int($iops_wr_max_length),
4241     );
4242
4243 }
4244
4245 # old code, only used to shutdown old VM after update
4246 sub __read_avail {
4247     my ($fh, $timeout) = @_;
4248
4249     my $sel = new IO::Select;
4250     $sel->add($fh);
4251
4252     my $res = '';
4253     my $buf;
4254
4255     my @ready;
4256     while (scalar (@ready = $sel->can_read($timeout))) {
4257         my $count;
4258         if ($count = $fh->sysread($buf, 8192)) {
4259             if ($buf =~ /^(.*)\(qemu\) $/s) {
4260                 $res .= $1;
4261                 last;
4262             } else {
4263                 $res .= $buf;
4264             }
4265         } else {
4266             if (!defined($count)) {
4267                 die "$!\n";
4268             }
4269             last;
4270         }
4271     }
4272
4273     die "monitor read timeout\n" if !scalar(@ready);
4274
4275     return $res;
4276 }
4277
4278 sub qemu_block_resize {
4279     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4280
4281     my $running = check_running($vmid);
4282
4283     $size = 0 if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4284
4285     return if !$running;
4286
4287     vm_mon_cmd($vmid, "block_resize", device => $deviceid, size => int($size));
4288
4289 }
4290
4291 sub qemu_volume_snapshot {
4292     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4293
4294     my $running = check_running($vmid);
4295
4296     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4297         vm_mon_cmd($vmid, 'blockdev-snapshot-internal-sync', device => $deviceid, name => $snap);
4298     } else {
4299         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4300     }
4301 }
4302
4303 sub qemu_volume_snapshot_delete {
4304     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4305
4306     my $running = check_running($vmid);
4307
4308     if($running) {
4309
4310         $running = undef;
4311         my $conf = PVE::QemuConfig->load_config($vmid);
4312         foreach_drive($conf, sub {
4313             my ($ds, $drive) = @_;
4314             $running = 1 if $drive->{file} eq $volid;
4315         });
4316     }
4317
4318     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4319         vm_mon_cmd($vmid, 'blockdev-snapshot-delete-internal-sync', device => $deviceid, name => $snap);
4320     } else {
4321         PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4322     }
4323 }
4324
4325 sub set_migration_caps {
4326     my ($vmid) = @_;
4327
4328     my $cap_ref = [];
4329
4330     my $enabled_cap = {
4331         "auto-converge" => 1,
4332         "xbzrle" => 1,
4333         "x-rdma-pin-all" => 0,
4334         "zero-blocks" => 0,
4335         "compress" => 0
4336     };
4337
4338     my $supported_capabilities = vm_mon_cmd_nocheck($vmid, "query-migrate-capabilities");
4339
4340     for my $supported_capability (@$supported_capabilities) {
4341         push @$cap_ref, {
4342             capability => $supported_capability->{capability},
4343             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4344         };
4345     }
4346
4347     vm_mon_cmd_nocheck($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4348 }
4349
4350 my $fast_plug_option = {
4351     'lock' => 1,
4352     'name' => 1,
4353     'onboot' => 1,
4354     'shares' => 1,
4355     'startup' => 1,
4356     'description' => 1,
4357     'protection' => 1,
4358     'vmstatestorage' => 1,
4359 };
4360
4361 # hotplug changes in [PENDING]
4362 # $selection hash can be used to only apply specified options, for
4363 # example: { cores => 1 } (only apply changed 'cores')
4364 # $errors ref is used to return error messages
4365 sub vmconfig_hotplug_pending {
4366     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4367
4368     my $defaults = load_defaults();
4369
4370     # commit values which do not have any impact on running VM first
4371     # Note: those option cannot raise errors, we we do not care about
4372     # $selection and always apply them.
4373
4374     my $add_error = sub {
4375         my ($opt, $msg) = @_;
4376         $errors->{$opt} = "hotplug problem - $msg";
4377     };
4378
4379     my $changes = 0;
4380     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4381         if ($fast_plug_option->{$opt}) {
4382             $conf->{$opt} = $conf->{pending}->{$opt};
4383             delete $conf->{pending}->{$opt};
4384             $changes = 1;
4385         }
4386     }
4387
4388     if ($changes) {
4389         PVE::QemuConfig->write_config($vmid, $conf);
4390         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4391     }
4392
4393     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4394
4395     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4396     while (my ($opt, $force) = each %$pending_delete_hash) {
4397         next if $selection && !$selection->{$opt};
4398         eval {
4399             if ($opt eq 'hotplug') {
4400                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
4401             } elsif ($opt eq 'tablet') {
4402                 die "skip\n" if !$hotplug_features->{usb};
4403                 if ($defaults->{tablet}) {
4404                     vm_deviceplug($storecfg, $conf, $vmid, $opt);
4405                 } else {
4406                     vm_deviceunplug($vmid, $conf, $opt);
4407                 }
4408             } elsif ($opt =~ m/^usb\d+/) {
4409                 die "skip\n";
4410                 # since we cannot reliably hot unplug usb devices
4411                 # we are disabling it
4412                 die "skip\n" if !$hotplug_features->{usb} || $conf->{$opt} =~ m/spice/i;
4413                 vm_deviceunplug($vmid, $conf, $opt);
4414             } elsif ($opt eq 'vcpus') {
4415                 die "skip\n" if !$hotplug_features->{cpu};
4416                 qemu_cpu_hotplug($vmid, $conf, undef);
4417             } elsif ($opt eq 'balloon') {
4418                 # enable balloon device is not hotpluggable
4419                 die "skip\n" if defined($conf->{balloon}) && $conf->{balloon} == 0;
4420                 # here we reset the ballooning value to memory
4421                 my $balloon = $conf->{memory} || $defaults->{memory};
4422                 vm_mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4423             } elsif ($fast_plug_option->{$opt}) {
4424                 # do nothing
4425             } elsif ($opt =~ m/^net(\d+)$/) {
4426                 die "skip\n" if !$hotplug_features->{network};
4427                 vm_deviceunplug($vmid, $conf, $opt);
4428             } elsif (is_valid_drivename($opt)) {
4429                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4430                 vm_deviceunplug($vmid, $conf, $opt);
4431                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4432             } elsif ($opt =~ m/^memory$/) {
4433                 die "skip\n" if !$hotplug_features->{memory};
4434                 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
4435             } elsif ($opt eq 'cpuunits') {
4436                 cgroups_write("cpu", $vmid, "cpu.shares", $defaults->{cpuunits});
4437             } elsif ($opt eq 'cpulimit') {
4438                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", -1);
4439             } else {
4440                 die "skip\n";
4441             }
4442         };
4443         if (my $err = $@) {
4444             &$add_error($opt, $err) if $err ne "skip\n";
4445         } else {
4446             # save new config if hotplug was successful
4447             delete $conf->{$opt};
4448             vmconfig_undelete_pending_option($conf, $opt);
4449             PVE::QemuConfig->write_config($vmid, $conf);
4450             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4451         }
4452     }
4453
4454     my $apply_pending_cloudinit;
4455     $apply_pending_cloudinit = sub {
4456         my ($key, $value) = @_;
4457         $apply_pending_cloudinit = sub {}; # once is enough
4458
4459         my @cloudinit_opts = keys %$confdesc_cloudinit;
4460         foreach my $opt (keys %{$conf->{pending}}) {
4461             next if !grep { $_ eq $opt } @cloudinit_opts;
4462             $conf->{$opt} = delete $conf->{pending}->{$opt};
4463         }
4464
4465         my $new_conf = { %$conf };
4466         $new_conf->{$key} = $value;
4467         PVE::QemuServer::Cloudinit::generate_cloudinitconfig($new_conf, $vmid);
4468     };
4469
4470     foreach my $opt (keys %{$conf->{pending}}) {
4471         next if $selection && !$selection->{$opt};
4472         my $value = $conf->{pending}->{$opt};
4473         eval {
4474             if ($opt eq 'hotplug') {
4475                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
4476             } elsif ($opt eq 'tablet') {
4477                 die "skip\n" if !$hotplug_features->{usb};
4478                 if ($value == 1) {
4479                     vm_deviceplug($storecfg, $conf, $vmid, $opt);
4480                 } elsif ($value == 0) {
4481                     vm_deviceunplug($vmid, $conf, $opt);
4482                 }
4483             } elsif ($opt =~ m/^usb\d+$/) {
4484                 die "skip\n";
4485                 # since we cannot reliably hot unplug usb devices
4486                 # we are disabling it
4487                 die "skip\n" if !$hotplug_features->{usb} || $value =~ m/spice/i;
4488                 my $d = eval { PVE::JSONSchema::parse_property_string($usbdesc->{format}, $value) };
4489                 die "skip\n" if !$d;
4490                 qemu_usb_hotplug($storecfg, $conf, $vmid, $opt, $d);
4491             } elsif ($opt eq 'vcpus') {
4492                 die "skip\n" if !$hotplug_features->{cpu};
4493                 qemu_cpu_hotplug($vmid, $conf, $value);
4494             } elsif ($opt eq 'balloon') {
4495                 # enable/disable balloning device is not hotpluggable
4496                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
4497                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
4498                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
4499
4500                 # allow manual ballooning if shares is set to zero
4501                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
4502                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
4503                     vm_mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4504                 }
4505             } elsif ($opt =~ m/^net(\d+)$/) {
4506                 # some changes can be done without hotplug
4507                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
4508                                     $vmid, $opt, $value);
4509             } elsif (is_valid_drivename($opt)) {
4510                 # some changes can be done without hotplug
4511                 my $drive = parse_drive($opt, $value);
4512                 if (drive_is_cloudinit($drive)) {
4513                     &$apply_pending_cloudinit($opt, $value);
4514                 }
4515                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
4516                                      $vmid, $opt, $value, 1);
4517             } elsif ($opt =~ m/^memory$/) { #dimms
4518                 die "skip\n" if !$hotplug_features->{memory};
4519                 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
4520             } elsif ($opt eq 'cpuunits') {
4521                 cgroups_write("cpu", $vmid, "cpu.shares", $conf->{pending}->{$opt});
4522             } elsif ($opt eq 'cpulimit') {
4523                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
4524                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", $cpulimit);
4525             } else {
4526                 die "skip\n";  # skip non-hot-pluggable options
4527             }
4528         };
4529         if (my $err = $@) {
4530             &$add_error($opt, $err) if $err ne "skip\n";
4531         } else {
4532             # save new config if hotplug was successful
4533             $conf->{$opt} = $value;
4534             delete $conf->{pending}->{$opt};
4535             PVE::QemuConfig->write_config($vmid, $conf);
4536             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4537         }
4538     }
4539 }
4540
4541 sub try_deallocate_drive {
4542     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
4543
4544     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
4545         my $volid = $drive->{file};
4546         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
4547             my $sid = PVE::Storage::parse_volume_id($volid);
4548             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
4549
4550             # check if the disk is really unused
4551             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
4552                 if is_volume_in_use($storecfg, $conf, $key, $volid);
4553             PVE::Storage::vdisk_free($storecfg, $volid);
4554             return 1;
4555         } else {
4556             # If vm is not owner of this disk remove from config
4557             return 1;
4558         }
4559     }
4560
4561     return undef;
4562 }
4563
4564 sub vmconfig_delete_or_detach_drive {
4565     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
4566
4567     my $drive = parse_drive($opt, $conf->{$opt});
4568
4569     my $rpcenv = PVE::RPCEnvironment::get();
4570     my $authuser = $rpcenv->get_user();
4571
4572     if ($force) {
4573         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
4574         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
4575     } else {
4576         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
4577     }
4578 }
4579
4580 sub vmconfig_apply_pending {
4581     my ($vmid, $conf, $storecfg) = @_;
4582
4583     # cold plug
4584
4585     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4586     while (my ($opt, $force) = each %$pending_delete_hash) {
4587         die "internal error" if $opt =~ m/^unused/;
4588         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4589         if (!defined($conf->{$opt})) {
4590             vmconfig_undelete_pending_option($conf, $opt);
4591             PVE::QemuConfig->write_config($vmid, $conf);
4592         } elsif (is_valid_drivename($opt)) {
4593             vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4594             vmconfig_undelete_pending_option($conf, $opt);
4595             delete $conf->{$opt};
4596             PVE::QemuConfig->write_config($vmid, $conf);
4597         } else {
4598             vmconfig_undelete_pending_option($conf, $opt);
4599             delete $conf->{$opt};
4600             PVE::QemuConfig->write_config($vmid, $conf);
4601         }
4602     }
4603
4604     $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4605
4606     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4607         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4608
4609         if (defined($conf->{$opt}) && ($conf->{$opt} eq $conf->{pending}->{$opt})) {
4610             # skip if nothing changed
4611         } elsif (is_valid_drivename($opt)) {
4612             vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
4613                 if defined($conf->{$opt});
4614             $conf->{$opt} = $conf->{pending}->{$opt};
4615         } else {
4616             $conf->{$opt} = $conf->{pending}->{$opt};
4617         }
4618
4619         delete $conf->{pending}->{$opt};
4620         PVE::QemuConfig->write_config($vmid, $conf);
4621     }
4622 }
4623
4624 my $safe_num_ne = sub {
4625     my ($a, $b) = @_;
4626
4627     return 0 if !defined($a) && !defined($b);
4628     return 1 if !defined($a);
4629     return 1 if !defined($b);
4630
4631     return $a != $b;
4632 };
4633
4634 my $safe_string_ne = sub {
4635     my ($a, $b) = @_;
4636
4637     return 0 if !defined($a) && !defined($b);
4638     return 1 if !defined($a);
4639     return 1 if !defined($b);
4640
4641     return $a ne $b;
4642 };
4643
4644 sub vmconfig_update_net {
4645     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value) = @_;
4646
4647     my $newnet = parse_net($value);
4648
4649     if ($conf->{$opt}) {
4650         my $oldnet = parse_net($conf->{$opt});
4651
4652         if (&$safe_string_ne($oldnet->{model}, $newnet->{model}) ||
4653             &$safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
4654             &$safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
4655             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
4656
4657             # for non online change, we try to hot-unplug
4658             die "skip\n" if !$hotplug;
4659             vm_deviceunplug($vmid, $conf, $opt);
4660         } else {
4661
4662             die "internal error" if $opt !~ m/net(\d+)/;
4663             my $iface = "tap${vmid}i$1";
4664
4665             if (&$safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
4666                 &$safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
4667                 &$safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
4668                 &$safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
4669                 PVE::Network::tap_unplug($iface);
4670                 PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
4671             } elsif (&$safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
4672                 # Rate can be applied on its own but any change above needs to
4673                 # include the rate in tap_plug since OVS resets everything.
4674                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
4675             }
4676
4677             if (&$safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
4678                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
4679             }
4680
4681             return 1;
4682         }
4683     }
4684
4685     if ($hotplug) {
4686         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet);
4687     } else {
4688         die "skip\n";
4689     }
4690 }
4691
4692 sub vmconfig_update_disk {
4693     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $force) = @_;
4694
4695     # fixme: do we need force?
4696
4697     my $drive = parse_drive($opt, $value);
4698
4699     if ($conf->{$opt}) {
4700
4701         if (my $old_drive = parse_drive($opt, $conf->{$opt}))  {
4702
4703             my $media = $drive->{media} || 'disk';
4704             my $oldmedia = $old_drive->{media} || 'disk';
4705             die "unable to change media type\n" if $media ne $oldmedia;
4706
4707             if (!drive_is_cdrom($old_drive)) {
4708
4709                 if ($drive->{file} ne $old_drive->{file}) {
4710
4711                     die "skip\n" if !$hotplug;
4712
4713                     # unplug and register as unused
4714                     vm_deviceunplug($vmid, $conf, $opt);
4715                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
4716
4717                 } else {
4718                     # update existing disk
4719
4720                     # skip non hotpluggable value
4721                     if (&$safe_string_ne($drive->{discard}, $old_drive->{discard}) ||
4722                         &$safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
4723                         &$safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
4724                         &$safe_string_ne($drive->{cache}, $old_drive->{cache})) {
4725                         die "skip\n";
4726                     }
4727
4728                     # apply throttle
4729                     if (&$safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
4730                         &$safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
4731                         &$safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
4732                         &$safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
4733                         &$safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
4734                         &$safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
4735                         &$safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
4736                         &$safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
4737                         &$safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
4738                         &$safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
4739                         &$safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
4740                         &$safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max}) ||
4741                         &$safe_num_ne($drive->{bps_max_length}, $old_drive->{bps_max_length}) ||
4742                         &$safe_num_ne($drive->{bps_rd_max_length}, $old_drive->{bps_rd_max_length}) ||
4743                         &$safe_num_ne($drive->{bps_wr_max_length}, $old_drive->{bps_wr_max_length}) ||
4744                         &$safe_num_ne($drive->{iops_max_length}, $old_drive->{iops_max_length}) ||
4745                         &$safe_num_ne($drive->{iops_rd_max_length}, $old_drive->{iops_rd_max_length}) ||
4746                         &$safe_num_ne($drive->{iops_wr_max_length}, $old_drive->{iops_wr_max_length})) {
4747
4748                         qemu_block_set_io_throttle($vmid,"drive-$opt",
4749                                                    ($drive->{mbps} || 0)*1024*1024,
4750                                                    ($drive->{mbps_rd} || 0)*1024*1024,
4751                                                    ($drive->{mbps_wr} || 0)*1024*1024,
4752                                                    $drive->{iops} || 0,
4753                                                    $drive->{iops_rd} || 0,
4754                                                    $drive->{iops_wr} || 0,
4755                                                    ($drive->{mbps_max} || 0)*1024*1024,
4756                                                    ($drive->{mbps_rd_max} || 0)*1024*1024,
4757                                                    ($drive->{mbps_wr_max} || 0)*1024*1024,
4758                                                    $drive->{iops_max} || 0,
4759                                                    $drive->{iops_rd_max} || 0,
4760                                                    $drive->{iops_wr_max} || 0,
4761                                                    $drive->{bps_max_length} || 1,
4762                                                    $drive->{bps_rd_max_length} || 1,
4763                                                    $drive->{bps_wr_max_length} || 1,
4764                                                    $drive->{iops_max_length} || 1,
4765                                                    $drive->{iops_rd_max_length} || 1,
4766                                                    $drive->{iops_wr_max_length} || 1);
4767
4768                     }
4769
4770                     return 1;
4771                 }
4772
4773             } else { # cdrom
4774
4775                 if ($drive->{file} eq 'none') {
4776                     vm_mon_cmd($vmid, "eject",force => JSON::true,device => "drive-$opt");
4777                     if (drive_is_cloudinit($old_drive)) {
4778                         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive);
4779                     }
4780                 } else {
4781                     my $path = get_iso_path($storecfg, $vmid, $drive->{file});
4782                     vm_mon_cmd($vmid, "eject", force => JSON::true,device => "drive-$opt"); # force eject if locked
4783                     vm_mon_cmd($vmid, "change", device => "drive-$opt",target => "$path") if $path;
4784                 }
4785
4786                 return 1;
4787             }
4788         }
4789     }
4790
4791     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
4792     # hotplug new disks
4793     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
4794     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive);
4795 }
4796
4797 sub vm_start {
4798     my ($storecfg, $vmid, $statefile, $skiplock, $migratedfrom, $paused,
4799         $forcemachine, $spice_ticket, $migration_network, $migration_type, $targetstorage) = @_;
4800
4801     PVE::QemuConfig->lock_config($vmid, sub {
4802         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
4803
4804         die "you can't start a vm if it's a template\n" if PVE::QemuConfig->is_template($conf);
4805
4806         PVE::QemuConfig->check_lock($conf) if !$skiplock;
4807
4808         die "VM $vmid already running\n" if check_running($vmid, undef, $migratedfrom);
4809
4810         if (!$statefile && scalar(keys %{$conf->{pending}})) {
4811             vmconfig_apply_pending($vmid, $conf, $storecfg);
4812             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4813         }
4814
4815         PVE::QemuServer::Cloudinit::generate_cloudinitconfig($conf, $vmid);
4816
4817         my $defaults = load_defaults();
4818
4819         # set environment variable useful inside network script
4820         $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
4821
4822         my $local_volumes = {};
4823
4824         if ($targetstorage) {
4825             foreach_drive($conf, sub {
4826                 my ($ds, $drive) = @_;
4827
4828                 return if drive_is_cdrom($drive);
4829
4830                 my $volid = $drive->{file};
4831
4832                 return if !$volid;
4833
4834                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
4835
4836                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
4837                 return if $scfg->{shared};
4838                 $local_volumes->{$ds} = [$volid, $storeid, $volname];
4839             });
4840
4841             my $format = undef;
4842
4843             foreach my $opt (sort keys %$local_volumes) {
4844
4845                 my ($volid, $storeid, $volname) = @{$local_volumes->{$opt}};
4846                 my $drive = parse_drive($opt, $conf->{$opt});
4847
4848                 #if remote storage is specified, use default format
4849                 if ($targetstorage && $targetstorage ne "1") {
4850                     $storeid = $targetstorage;
4851                     my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
4852                     $format = $defFormat;
4853                 } else {
4854                     #else we use same format than original
4855                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
4856                     $format = qemu_img_format($scfg, $volid);
4857                 }
4858
4859                 my $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $format, undef, ($drive->{size}/1024));
4860                 my $newdrive = $drive;
4861                 $newdrive->{format} = $format;
4862                 $newdrive->{file} = $newvolid;
4863                 my $drivestr = PVE::QemuServer::print_drive($vmid, $newdrive);
4864                 $local_volumes->{$opt} = $drivestr;
4865                 #pass drive to conf for command line
4866                 $conf->{$opt} = $drivestr;
4867             }
4868         }
4869
4870         my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine);
4871
4872         my $migrate_port = 0;
4873         my $migrate_uri;
4874         if ($statefile) {
4875             if ($statefile eq 'tcp') {
4876                 my $localip = "localhost";
4877                 my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
4878                 my $nodename = PVE::INotify::nodename();
4879
4880                 if (!defined($migration_type)) {
4881                     if (defined($datacenterconf->{migration}->{type})) {
4882                         $migration_type = $datacenterconf->{migration}->{type};
4883                     } else {
4884                         $migration_type = 'secure';
4885                     }
4886                 }
4887
4888                 if ($migration_type eq 'insecure') {
4889                     my $migrate_network_addr = PVE::Cluster::get_local_migration_ip($migration_network);
4890                     if ($migrate_network_addr) {
4891                         $localip = $migrate_network_addr;
4892                     } else {
4893                         $localip = PVE::Cluster::remote_node_ip($nodename, 1);
4894                     }
4895
4896                     $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
4897                 }
4898
4899                 my $pfamily = PVE::Tools::get_host_address_family($nodename);
4900                 $migrate_port = PVE::Tools::next_migrate_port($pfamily);
4901                 $migrate_uri = "tcp:${localip}:${migrate_port}";
4902                 push @$cmd, '-incoming', $migrate_uri;
4903                 push @$cmd, '-S';
4904
4905             } elsif ($statefile eq 'unix') {
4906                 # should be default for secure migrations as a ssh TCP forward
4907                 # tunnel is not deterministic reliable ready and fails regurarly
4908                 # to set up in time, so use UNIX socket forwards
4909                 my $socket_addr = "/run/qemu-server/$vmid.migrate";
4910                 unlink $socket_addr;
4911
4912                 $migrate_uri = "unix:$socket_addr";
4913
4914                 push @$cmd, '-incoming', $migrate_uri;
4915                 push @$cmd, '-S';
4916
4917             } else {
4918                 push @$cmd, '-loadstate', $statefile;
4919             }
4920         } elsif ($paused) {
4921             push @$cmd, '-S';
4922         }
4923
4924         # host pci devices
4925         for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
4926           my $d = parse_hostpci($conf->{"hostpci$i"});
4927           next if !$d;
4928           my $pcidevices = $d->{pciid};
4929           foreach my $pcidevice (@$pcidevices) {
4930                 my $pciid = $pcidevice->{id}.".".$pcidevice->{function};
4931
4932                 my $info = pci_device_info("0000:$pciid");
4933                 die "IOMMU not present\n" if !check_iommu_support();
4934                 die "no pci device info for device '$pciid'\n" if !$info;
4935                 die "can't unbind/bind pci group to vfio '$pciid'\n" if !pci_dev_group_bind_to_vfio($pciid);
4936                 die "can't reset pci device '$pciid'\n" if $info->{has_fl_reset} and !pci_dev_reset($info);
4937           }
4938         }
4939
4940         PVE::Storage::activate_volumes($storecfg, $vollist);
4941
4942         if (!check_running($vmid, 1)) {
4943             eval {
4944                 run_command(['/bin/systemctl', 'stop', "$vmid.scope"],
4945                     outfunc => sub {}, errfunc => sub {});
4946             };
4947         }
4948
4949         my $cpuunits = defined($conf->{cpuunits}) ? $conf->{cpuunits}
4950                                                   : $defaults->{cpuunits};
4951
4952         my $start_timeout = $conf->{hugepages} ? 300 : 30;
4953         my %run_params = (timeout => $statefile ? undef : $start_timeout, umask => 0077);
4954
4955         my %properties = (
4956             Slice => 'qemu.slice',
4957             KillMode => 'none',
4958             CPUShares => $cpuunits
4959         );
4960
4961         if (my $cpulimit = $conf->{cpulimit}) {
4962             $properties{CPUQuota} = int($cpulimit * 100);
4963         }
4964         $properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
4965
4966         my $run_qemu = sub {
4967             PVE::Tools::run_fork sub {
4968                 PVE::Systemd::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
4969                 run_command($cmd, %run_params);
4970             };
4971         };
4972
4973         if ($conf->{hugepages}) {
4974
4975             my $code = sub {
4976                 my $hugepages_topology = PVE::QemuServer::Memory::hugepages_topology($conf);
4977                 my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
4978
4979                 PVE::QemuServer::Memory::hugepages_mount();
4980                 PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
4981
4982                 eval { $run_qemu->() };
4983                 if (my $err = $@) {
4984                     PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology);
4985                     die $err;
4986                 }
4987
4988                 PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology);
4989             };
4990             eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
4991
4992         } else {
4993             eval { $run_qemu->() };
4994         }
4995
4996         if (my $err = $@) {
4997             # deactivate volumes if start fails
4998             eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
4999             die "start failed: $err";
5000         }
5001
5002         print "migration listens on $migrate_uri\n" if $migrate_uri;
5003
5004         if ($statefile && $statefile ne 'tcp' && $statefile ne 'unix')  {
5005             eval { vm_mon_cmd_nocheck($vmid, "cont"); };
5006             warn $@ if $@;
5007         }
5008
5009         #start nbd server for storage migration
5010         if ($targetstorage) {
5011             my $nodename = PVE::INotify::nodename();
5012             my $migrate_network_addr = PVE::Cluster::get_local_migration_ip($migration_network);
5013             my $localip = $migrate_network_addr ? $migrate_network_addr : PVE::Cluster::remote_node_ip($nodename, 1);
5014             my $pfamily = PVE::Tools::get_host_address_family($nodename);
5015             $migrate_port = PVE::Tools::next_migrate_port($pfamily);
5016
5017             vm_mon_cmd_nocheck($vmid, "nbd-server-start", addr => { type => 'inet', data => { host => "${localip}", port => "${migrate_port}" } } );
5018
5019             $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5020
5021             foreach my $opt (sort keys %$local_volumes) {
5022                 my $volid = $local_volumes->{$opt};
5023                 vm_mon_cmd_nocheck($vmid, "nbd-server-add", device => "drive-$opt", writable => JSON::true );
5024                 my $migrate_storage_uri = "nbd:${localip}:${migrate_port}:exportname=drive-$opt";
5025                 print "storage migration listens on $migrate_storage_uri volume:$volid\n";
5026             }
5027         }
5028
5029         if ($migratedfrom) {
5030             eval {
5031                 set_migration_caps($vmid);
5032             };
5033             warn $@ if $@;
5034
5035             if ($spice_port) {
5036                 print "spice listens on port $spice_port\n";
5037                 if ($spice_ticket) {
5038                     vm_mon_cmd_nocheck($vmid, "set_password", protocol => 'spice', password => $spice_ticket);
5039                     vm_mon_cmd_nocheck($vmid, "expire_password", protocol => 'spice', time => "+30");
5040                 }
5041             }
5042
5043         } else {
5044             vm_mon_cmd_nocheck($vmid, "balloon", value => $conf->{balloon}*1024*1024)
5045                 if !$statefile && $conf->{balloon};
5046
5047             foreach my $opt (keys %$conf) {
5048                 next if $opt !~  m/^net\d+$/;
5049                 my $nicconf = parse_net($conf->{$opt});
5050                 qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
5051             }
5052         }
5053
5054         vm_mon_cmd_nocheck($vmid, 'qom-set',
5055                     path => "machine/peripheral/balloon0",
5056                     property => "guest-stats-polling-interval",
5057                     value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
5058
5059     });
5060 }
5061
5062 sub vm_mon_cmd {
5063     my ($vmid, $execute, %params) = @_;
5064
5065     my $cmd = { execute => $execute, arguments => \%params };
5066     vm_qmp_command($vmid, $cmd);
5067 }
5068
5069 sub vm_mon_cmd_nocheck {
5070     my ($vmid, $execute, %params) = @_;
5071
5072     my $cmd = { execute => $execute, arguments => \%params };
5073     vm_qmp_command($vmid, $cmd, 1);
5074 }
5075
5076 sub vm_qmp_command {
5077     my ($vmid, $cmd, $nocheck) = @_;
5078
5079     my $res;
5080
5081     my $timeout;
5082     if ($cmd->{arguments} && $cmd->{arguments}->{timeout}) {
5083         $timeout = $cmd->{arguments}->{timeout};
5084         delete $cmd->{arguments}->{timeout};
5085     }
5086
5087     eval {
5088         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
5089         my $sname = qmp_socket($vmid);
5090         if (-e $sname) { # test if VM is reasonambe new and supports qmp/qga
5091             my $qmpclient = PVE::QMPClient->new();
5092
5093             $res = $qmpclient->cmd($vmid, $cmd, $timeout);
5094         } else {
5095             die "unable to open monitor socket\n";
5096         }
5097     };
5098     if (my $err = $@) {
5099         syslog("err", "VM $vmid qmp command failed - $err");
5100         die $err;
5101     }
5102
5103     return $res;
5104 }
5105
5106 sub vm_human_monitor_command {
5107     my ($vmid, $cmdline) = @_;
5108
5109     my $res;
5110
5111     my $cmd = {
5112         execute => 'human-monitor-command',
5113         arguments => { 'command-line' => $cmdline},
5114     };
5115
5116     return vm_qmp_command($vmid, $cmd);
5117 }
5118
5119 sub vm_commandline {
5120     my ($storecfg, $vmid) = @_;
5121
5122     my $conf = PVE::QemuConfig->load_config($vmid);
5123
5124     my $defaults = load_defaults();
5125
5126     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults);
5127
5128     return PVE::Tools::cmd2string($cmd);
5129 }
5130
5131 sub vm_reset {
5132     my ($vmid, $skiplock) = @_;
5133
5134     PVE::QemuConfig->lock_config($vmid, sub {
5135
5136         my $conf = PVE::QemuConfig->load_config($vmid);
5137
5138         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5139
5140         vm_mon_cmd($vmid, "system_reset");
5141     });
5142 }
5143
5144 sub get_vm_volumes {
5145     my ($conf) = @_;
5146
5147     my $vollist = [];
5148     foreach_volid($conf, sub {
5149         my ($volid, $attr) = @_;
5150
5151         return if $volid =~ m|^/|;
5152
5153         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5154         return if !$sid;
5155
5156         push @$vollist, $volid;
5157     });
5158
5159     return $vollist;
5160 }
5161
5162 sub vm_stop_cleanup {
5163     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
5164
5165     eval {
5166
5167         if (!$keepActive) {
5168             my $vollist = get_vm_volumes($conf);
5169             PVE::Storage::deactivate_volumes($storecfg, $vollist);
5170         }
5171
5172         foreach my $ext (qw(mon qmp pid vnc qga)) {
5173             unlink "/var/run/qemu-server/${vmid}.$ext";
5174         }
5175
5176         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
5177     };
5178     warn $@ if $@; # avoid errors - just warn
5179 }
5180
5181 # Note: use $nockeck to skip tests if VM configuration file exists.
5182 # We need that when migration VMs to other nodes (files already moved)
5183 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
5184 sub vm_stop {
5185     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
5186
5187     $force = 1 if !defined($force) && !$shutdown;
5188
5189     if ($migratedfrom){
5190         my $pid = check_running($vmid, $nocheck, $migratedfrom);
5191         kill 15, $pid if $pid;
5192         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
5193         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
5194         return;
5195     }
5196
5197     PVE::QemuConfig->lock_config($vmid, sub {
5198
5199         my $pid = check_running($vmid, $nocheck);
5200         return if !$pid;
5201
5202         my $conf;
5203         if (!$nocheck) {
5204             $conf = PVE::QemuConfig->load_config($vmid);
5205             PVE::QemuConfig->check_lock($conf) if !$skiplock;
5206             if (!defined($timeout) && $shutdown && $conf->{startup}) {
5207                 my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
5208                 $timeout = $opts->{down} if $opts->{down};
5209             }
5210         }
5211
5212         $timeout = 60 if !defined($timeout);
5213
5214         eval {
5215             if ($shutdown) {
5216                 if (defined($conf) && parse_guest_agent($conf)->{enabled}) {
5217                     vm_qmp_command($vmid, { execute => "guest-shutdown" }, $nocheck);
5218                 } else {
5219                     vm_qmp_command($vmid, { execute => "system_powerdown" }, $nocheck);
5220                 }
5221             } else {
5222                 vm_qmp_command($vmid, { execute => "quit" }, $nocheck);
5223             }
5224         };
5225         my $err = $@;
5226
5227         if (!$err) {
5228             my $count = 0;
5229             while (($count < $timeout) && check_running($vmid, $nocheck)) {
5230                 $count++;
5231                 sleep 1;
5232             }
5233
5234             if ($count >= $timeout) {
5235                 if ($force) {
5236                     warn "VM still running - terminating now with SIGTERM\n";
5237                     kill 15, $pid;
5238                 } else {
5239                     die "VM quit/powerdown failed - got timeout\n";
5240                 }
5241             } else {
5242                 vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5243                 return;
5244             }
5245         } else {
5246             if ($force) {
5247                 warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
5248                 kill 15, $pid;
5249             } else {
5250                 die "VM quit/powerdown failed\n";
5251             }
5252         }
5253
5254         # wait again
5255         $timeout = 10;
5256
5257         my $count = 0;
5258         while (($count < $timeout) && check_running($vmid, $nocheck)) {
5259             $count++;
5260             sleep 1;
5261         }
5262
5263         if ($count >= $timeout) {
5264             warn "VM still running - terminating now with SIGKILL\n";
5265             kill 9, $pid;
5266             sleep 1;
5267         }
5268
5269         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5270    });
5271 }
5272
5273 sub vm_suspend {
5274     my ($vmid, $skiplock) = @_;
5275
5276     PVE::QemuConfig->lock_config($vmid, sub {
5277
5278         my $conf = PVE::QemuConfig->load_config($vmid);
5279
5280         PVE::QemuConfig->check_lock($conf)
5281             if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
5282
5283         vm_mon_cmd($vmid, "stop");
5284     });
5285 }
5286
5287 sub vm_resume {
5288     my ($vmid, $skiplock, $nocheck) = @_;
5289
5290     PVE::QemuConfig->lock_config($vmid, sub {
5291
5292         my $res = vm_mon_cmd($vmid, 'query-status');
5293         my $resume_cmd = 'cont';
5294
5295         if ($res->{status} && $res->{status} eq 'suspended') {
5296             $resume_cmd = 'system_wakeup';
5297         }
5298
5299         if (!$nocheck) {
5300
5301             my $conf = PVE::QemuConfig->load_config($vmid);
5302
5303             PVE::QemuConfig->check_lock($conf)
5304                 if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
5305
5306             vm_mon_cmd($vmid, $resume_cmd);
5307
5308         } else {
5309             vm_mon_cmd_nocheck($vmid, $resume_cmd);
5310         }
5311     });
5312 }
5313
5314 sub vm_sendkey {
5315     my ($vmid, $skiplock, $key) = @_;
5316
5317     PVE::QemuConfig->lock_config($vmid, sub {
5318
5319         my $conf = PVE::QemuConfig->load_config($vmid);
5320
5321         # there is no qmp command, so we use the human monitor command
5322         vm_human_monitor_command($vmid, "sendkey $key");
5323     });
5324 }
5325
5326 sub vm_destroy {
5327     my ($storecfg, $vmid, $skiplock) = @_;
5328
5329     PVE::QemuConfig->lock_config($vmid, sub {
5330
5331         my $conf = PVE::QemuConfig->load_config($vmid);
5332
5333         if (!check_running($vmid)) {
5334             destroy_vm($storecfg, $vmid, undef, $skiplock);
5335         } else {
5336             die "VM $vmid is running - destroy failed\n";
5337         }
5338     });
5339 }
5340
5341 # pci helpers
5342
5343 sub file_write {
5344     my ($filename, $buf) = @_;
5345
5346     my $fh = IO::File->new($filename, "w");
5347     return undef if !$fh;
5348
5349     my $res = print $fh $buf;
5350
5351     $fh->close();
5352
5353     return $res;
5354 }
5355
5356 sub pci_device_info {
5357     my ($name) = @_;
5358
5359     my $res;
5360
5361     return undef if $name !~ m/^([a-f0-9]{4}):([a-f0-9]{2}):([a-f0-9]{2})\.([a-f0-9])$/;
5362     my ($domain, $bus, $slot, $func) = ($1, $2, $3, $4);
5363
5364     my $irq = file_read_firstline("$pcisysfs/devices/$name/irq");
5365     return undef if !defined($irq) || $irq !~ m/^\d+$/;
5366
5367     my $vendor = file_read_firstline("$pcisysfs/devices/$name/vendor");
5368     return undef if !defined($vendor) || $vendor !~ s/^0x//;
5369
5370     my $product = file_read_firstline("$pcisysfs/devices/$name/device");
5371     return undef if !defined($product) || $product !~ s/^0x//;
5372
5373     $res = {
5374         name => $name,
5375         vendor => $vendor,
5376         product => $product,
5377         domain => $domain,
5378         bus => $bus,
5379         slot => $slot,
5380         func => $func,
5381         irq => $irq,
5382         has_fl_reset => -f "$pcisysfs/devices/$name/reset" || 0,
5383     };
5384
5385     return $res;
5386 }
5387
5388 sub pci_dev_reset {
5389     my ($dev) = @_;
5390
5391     my $name = $dev->{name};
5392
5393     my $fn = "$pcisysfs/devices/$name/reset";
5394
5395     return file_write($fn, "1");
5396 }
5397
5398 sub pci_dev_bind_to_vfio {
5399     my ($dev) = @_;
5400
5401     my $name = $dev->{name};
5402
5403     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
5404
5405     if (!-d $vfio_basedir) {
5406         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
5407     }
5408     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
5409
5410     my $testdir = "$vfio_basedir/$name";
5411     return 1 if -d $testdir;
5412
5413     my $data = "$dev->{vendor} $dev->{product}";
5414     return undef if !file_write("$vfio_basedir/new_id", $data);
5415
5416     my $fn = "$pcisysfs/devices/$name/driver/unbind";
5417     if (!file_write($fn, $name)) {
5418         return undef if -f $fn;
5419     }
5420
5421     $fn = "$vfio_basedir/bind";
5422     if (! -d $testdir) {
5423         return undef if !file_write($fn, $name);
5424     }
5425
5426     return -d $testdir;
5427 }
5428
5429 sub pci_dev_group_bind_to_vfio {
5430     my ($pciid) = @_;
5431
5432     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
5433
5434     if (!-d $vfio_basedir) {
5435         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
5436     }
5437     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
5438
5439     # get IOMMU group devices
5440     opendir(my $D, "$pcisysfs/devices/0000:$pciid/iommu_group/devices/") || die "Cannot open iommu_group: $!\n";
5441       my @devs = grep /^0000:/, readdir($D);
5442     closedir($D);
5443
5444     foreach my $pciid (@devs) {
5445         $pciid =~ m/^([:\.\da-f]+)$/ or die "PCI ID $pciid not valid!\n";
5446
5447         # pci bridges, switches or root ports are not supported
5448         # they have a pci_bus subdirectory so skip them
5449         next if (-e "$pcisysfs/devices/$pciid/pci_bus");
5450
5451         my $info = pci_device_info($1);
5452         pci_dev_bind_to_vfio($info) || die "Cannot bind $pciid to vfio\n";
5453     }
5454
5455     return 1;
5456 }
5457
5458 # vzdump restore implementaion
5459
5460 sub tar_archive_read_firstfile {
5461     my $archive = shift;
5462
5463     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
5464
5465     # try to detect archive type first
5466     my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
5467         die "unable to open file '$archive'\n";
5468     my $firstfile = <$fh>;
5469     kill 15, $pid;
5470     close $fh;
5471
5472     die "ERROR: archive contaions no data\n" if !$firstfile;
5473     chomp $firstfile;
5474
5475     return $firstfile;
5476 }
5477
5478 sub tar_restore_cleanup {
5479     my ($storecfg, $statfile) = @_;
5480
5481     print STDERR "starting cleanup\n";
5482
5483     if (my $fd = IO::File->new($statfile, "r")) {
5484         while (defined(my $line = <$fd>)) {
5485             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5486                 my $volid = $2;
5487                 eval {
5488                     if ($volid =~ m|^/|) {
5489                         unlink $volid || die 'unlink failed\n';
5490                     } else {
5491                         PVE::Storage::vdisk_free($storecfg, $volid);
5492                     }
5493                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
5494                 };
5495                 print STDERR "unable to cleanup '$volid' - $@" if $@;
5496             } else {
5497                 print STDERR "unable to parse line in statfile - $line";
5498             }
5499         }
5500         $fd->close();
5501     }
5502 }
5503
5504 sub restore_archive {
5505     my ($archive, $vmid, $user, $opts) = @_;
5506
5507     my $format = $opts->{format};
5508     my $comp;
5509
5510     if ($archive =~ m/\.tgz$/ || $archive =~ m/\.tar\.gz$/) {
5511         $format = 'tar' if !$format;
5512         $comp = 'gzip';
5513     } elsif ($archive =~ m/\.tar$/) {
5514         $format = 'tar' if !$format;
5515     } elsif ($archive =~ m/.tar.lzo$/) {
5516         $format = 'tar' if !$format;
5517         $comp = 'lzop';
5518     } elsif ($archive =~ m/\.vma$/) {
5519         $format = 'vma' if !$format;
5520     } elsif ($archive =~ m/\.vma\.gz$/) {
5521         $format = 'vma' if !$format;
5522         $comp = 'gzip';
5523     } elsif ($archive =~ m/\.vma\.lzo$/) {
5524         $format = 'vma' if !$format;
5525         $comp = 'lzop';
5526     } else {
5527         $format = 'vma' if !$format; # default
5528     }
5529
5530     # try to detect archive format
5531     if ($format eq 'tar') {
5532         return restore_tar_archive($archive, $vmid, $user, $opts);
5533     } else {
5534         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
5535     }
5536 }
5537
5538 sub restore_update_config_line {
5539     my ($outfd, $cookie, $vmid, $map, $line, $unique) = @_;
5540
5541     return if $line =~ m/^\#qmdump\#/;
5542     return if $line =~ m/^\#vzdump\#/;
5543     return if $line =~ m/^lock:/;
5544     return if $line =~ m/^unused\d+:/;
5545     return if $line =~ m/^parent:/;
5546     return if $line =~ m/^template:/; # restored VM is never a template
5547
5548     my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
5549     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
5550         # try to convert old 1.X settings
5551         my ($id, $ind, $ethcfg) = ($1, $2, $3);
5552         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
5553             my ($model, $macaddr) = split(/\=/, $devconfig);
5554             $macaddr = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if !$macaddr || $unique;
5555             my $net = {
5556                 model => $model,
5557                 bridge => "vmbr$ind",
5558                 macaddr => $macaddr,
5559             };
5560             my $netstr = print_net($net);
5561
5562             print $outfd "net$cookie->{netcount}: $netstr\n";
5563             $cookie->{netcount}++;
5564         }
5565     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
5566         my ($id, $netstr) = ($1, $2);
5567         my $net = parse_net($netstr);
5568         $net->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if $net->{macaddr};
5569         $netstr = print_net($net);
5570         print $outfd "$id: $netstr\n";
5571     } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk)\d+):\s*(\S+)\s*$/) {
5572         my $virtdev = $1;
5573         my $value = $3;
5574         my $di = parse_drive($virtdev, $value);
5575         if (defined($di->{backup}) && !$di->{backup}) {
5576             print $outfd "#$line";
5577         } elsif ($map->{$virtdev}) {
5578             delete $di->{format}; # format can change on restore
5579             $di->{file} = $map->{$virtdev};
5580             $value = print_drive($vmid, $di);
5581             print $outfd "$virtdev: $value\n";
5582         } else {
5583             print $outfd $line;
5584         }
5585     } elsif (($line =~ m/^vmgenid: (.*)/)) {
5586         my $vmgenid = $1;
5587         if ($vmgenid ne '0') {
5588             # always generate a new vmgenid if there was a valid one setup
5589             $vmgenid = generate_uuid();
5590         }
5591         print $outfd "vmgenid: $vmgenid\n";
5592     } elsif (($line =~ m/^(smbios1: )(.*)/) && $unique) {
5593         my ($uuid, $uuid_str);
5594         UUID::generate($uuid);
5595         UUID::unparse($uuid, $uuid_str);
5596         my $smbios1 = parse_smbios1($2);
5597         $smbios1->{uuid} = $uuid_str;
5598         print $outfd $1.print_smbios1($smbios1)."\n";
5599     } else {
5600         print $outfd $line;
5601     }
5602 }
5603
5604 sub scan_volids {
5605     my ($cfg, $vmid) = @_;
5606
5607     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid);
5608
5609     my $volid_hash = {};
5610     foreach my $storeid (keys %$info) {
5611         foreach my $item (@{$info->{$storeid}}) {
5612             next if !($item->{volid} && $item->{size});
5613             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
5614             $volid_hash->{$item->{volid}} = $item;
5615         }
5616     }
5617
5618     return $volid_hash;
5619 }
5620
5621 sub is_volume_in_use {
5622     my ($storecfg, $conf, $skip_drive, $volid) = @_;
5623
5624     my $path = PVE::Storage::path($storecfg, $volid);
5625
5626     my $scan_config = sub {
5627         my ($cref, $snapname) = @_;
5628
5629         foreach my $key (keys %$cref) {
5630             my $value = $cref->{$key};
5631             if (is_valid_drivename($key)) {
5632                 next if $skip_drive && $key eq $skip_drive;
5633                 my $drive = parse_drive($key, $value);
5634                 next if !$drive || !$drive->{file} || drive_is_cdrom($drive);
5635                 return 1 if $volid eq $drive->{file};
5636                 if ($drive->{file} =~ m!^/!) {
5637                     return 1 if $drive->{file} eq $path;
5638                 } else {
5639                     my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file}, 1);
5640                     next if !$storeid;
5641                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid, 1);
5642                     next if !$scfg;
5643                     return 1 if $path eq PVE::Storage::path($storecfg, $drive->{file}, $snapname);
5644                 }
5645             }
5646         }
5647
5648         return 0;
5649     };
5650
5651     return 1 if &$scan_config($conf);
5652
5653     undef $skip_drive;
5654
5655     foreach my $snapname (keys %{$conf->{snapshots}}) {
5656         return 1 if &$scan_config($conf->{snapshots}->{$snapname}, $snapname);
5657     }
5658
5659     return 0;
5660 }
5661
5662 sub update_disksize {
5663     my ($vmid, $conf, $volid_hash) = @_;
5664
5665     my $changes;
5666     my $prefix = "VM $vmid:";
5667
5668     # used and unused disks
5669     my $referenced = {};
5670
5671     # Note: it is allowed to define multiple storages with same path (alias), so
5672     # we need to check both 'volid' and real 'path' (two different volid can point
5673     # to the same path).
5674
5675     my $referencedpath = {};
5676
5677     # update size info
5678     foreach my $opt (keys %$conf) {
5679         if (is_valid_drivename($opt)) {
5680             my $drive = parse_drive($opt, $conf->{$opt});
5681             my $volid = $drive->{file};
5682             next if !$volid;
5683
5684             $referenced->{$volid} = 1;
5685             if ($volid_hash->{$volid} &&
5686                 (my $path = $volid_hash->{$volid}->{path})) {
5687                 $referencedpath->{$path} = 1;
5688             }
5689
5690             next if drive_is_cdrom($drive);
5691             next if !$volid_hash->{$volid};
5692
5693             $drive->{size} = $volid_hash->{$volid}->{size};
5694             my $new = print_drive($vmid, $drive);
5695             if ($new ne $conf->{$opt}) {
5696                 $changes = 1;
5697                 $conf->{$opt} = $new;
5698                 print "$prefix update disk '$opt' information.\n";
5699             }
5700         }
5701     }
5702
5703     # remove 'unusedX' entry if volume is used
5704     foreach my $opt (keys %$conf) {
5705         next if $opt !~ m/^unused\d+$/;
5706         my $volid = $conf->{$opt};
5707         my $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
5708         if ($referenced->{$volid} || ($path && $referencedpath->{$path})) {
5709             print "$prefix remove entry '$opt', its volume '$volid' is in use.\n";
5710             $changes = 1;
5711             delete $conf->{$opt};
5712         }
5713
5714         $referenced->{$volid} = 1;
5715         $referencedpath->{$path} = 1 if $path;
5716     }
5717
5718     foreach my $volid (sort keys %$volid_hash) {
5719         next if $volid =~ m/vm-$vmid-state-/;
5720         next if $referenced->{$volid};
5721         my $path = $volid_hash->{$volid}->{path};
5722         next if !$path; # just to be sure
5723         next if $referencedpath->{$path};
5724         $changes = 1;
5725         my $key = PVE::QemuConfig->add_unused_volume($conf, $volid);
5726         print "$prefix add unreferenced volume '$volid' as '$key' to config.\n";
5727         $referencedpath->{$path} = 1; # avoid to add more than once (aliases)
5728     }
5729
5730     return $changes;
5731 }
5732
5733 sub rescan {
5734     my ($vmid, $nolock, $dryrun) = @_;
5735
5736     my $cfg = PVE::Storage::config();
5737
5738     # FIXME: Remove once our RBD plugin can handle CT and VM on a single storage
5739     # see: https://pve.proxmox.com/pipermail/pve-devel/2018-July/032900.html
5740     foreach my $stor (keys %{$cfg->{ids}}) {
5741         delete($cfg->{ids}->{$stor}) if ! $cfg->{ids}->{$stor}->{content}->{images};
5742     }
5743
5744     print "rescan volumes...\n";
5745     my $volid_hash = scan_volids($cfg, $vmid);
5746
5747     my $updatefn =  sub {
5748         my ($vmid) = @_;
5749
5750         my $conf = PVE::QemuConfig->load_config($vmid);
5751
5752         PVE::QemuConfig->check_lock($conf);
5753
5754         my $vm_volids = {};
5755         foreach my $volid (keys %$volid_hash) {
5756             my $info = $volid_hash->{$volid};
5757             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
5758         }
5759
5760         my $changes = update_disksize($vmid, $conf, $vm_volids);
5761
5762         PVE::QemuConfig->write_config($vmid, $conf) if $changes && !$dryrun;
5763     };
5764
5765     if (defined($vmid)) {
5766         if ($nolock) {
5767             &$updatefn($vmid);
5768         } else {
5769             PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
5770         }
5771     } else {
5772         my $vmlist = config_list();
5773         foreach my $vmid (keys %$vmlist) {
5774             if ($nolock) {
5775                 &$updatefn($vmid);
5776             } else {
5777                 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
5778             }
5779         }
5780     }
5781 }
5782
5783 sub restore_vma_archive {
5784     my ($archive, $vmid, $user, $opts, $comp) = @_;
5785
5786     my $readfrom = $archive;
5787
5788     my $cfg = PVE::Storage::config();
5789     my $commands = [];
5790     my $bwlimit = $opts->{bwlimit};
5791
5792     my $dbg_cmdstring = '';
5793     my $add_pipe = sub {
5794         my ($cmd) = @_;
5795         push @$commands, $cmd;
5796         $dbg_cmdstring .= ' | ' if length($dbg_cmdstring);
5797         $dbg_cmdstring .= PVE::Tools::cmd2string($cmd);
5798         $readfrom = '-';
5799     };
5800
5801     my $input = undef;
5802     if ($archive eq '-') {
5803         $input = '<&STDIN';
5804     } else {
5805         # If we use a backup from a PVE defined storage we also consider that
5806         # storage's rate limit:
5807         my (undef, $volid) = PVE::Storage::path_to_volume_id($cfg, $archive);
5808         if (defined($volid)) {
5809             my ($sid, undef) = PVE::Storage::parse_volume_id($volid);
5810             my $readlimit = PVE::Storage::get_bandwidth_limit('restore', [$sid], $bwlimit);
5811             if ($readlimit) {
5812                 print STDERR "applying read rate limit: $readlimit\n";
5813                 my $cstream = ['cstream', '-t', $readlimit*1024, '--', $readfrom];
5814                 $add_pipe->($cstream);
5815             }
5816         }
5817     }
5818
5819     if ($comp) {
5820         my $cmd;
5821         if ($comp eq 'gzip') {
5822             $cmd = ['zcat', $readfrom];
5823         } elsif ($comp eq 'lzop') {
5824             $cmd = ['lzop', '-d', '-c', $readfrom];
5825         } else {
5826             die "unknown compression method '$comp'\n";
5827         }
5828         $add_pipe->($cmd);
5829     }
5830
5831     my $tmpdir = "/var/tmp/vzdumptmp$$";
5832     rmtree $tmpdir;
5833
5834     # disable interrupts (always do cleanups)
5835     local $SIG{INT} =
5836         local $SIG{TERM} =
5837         local $SIG{QUIT} =
5838         local $SIG{HUP} = sub { warn "got interrupt - ignored\n"; };
5839
5840     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
5841     POSIX::mkfifo($mapfifo, 0600);
5842     my $fifofh;
5843
5844     my $openfifo = sub {
5845         open($fifofh, '>', $mapfifo) || die $!;
5846     };
5847
5848     $add_pipe->(['vma', 'extract', '-v', '-r', $mapfifo, $readfrom, $tmpdir]);
5849
5850     my $oldtimeout;
5851     my $timeout = 5;
5852
5853     my $devinfo = {};
5854
5855     my $rpcenv = PVE::RPCEnvironment::get();
5856
5857     my $conffile = PVE::QemuConfig->config_file($vmid);
5858     my $tmpfn = "$conffile.$$.tmp";
5859
5860     # Note: $oldconf is undef if VM does not exists
5861     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
5862     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
5863
5864     my %storage_limits;
5865
5866     my $print_devmap = sub {
5867         my $virtdev_hash = {};
5868
5869         my $cfgfn = "$tmpdir/qemu-server.conf";
5870
5871         # we can read the config - that is already extracted
5872         my $fh = IO::File->new($cfgfn, "r") ||
5873             "unable to read qemu-server.conf - $!\n";
5874
5875         my $fwcfgfn = "$tmpdir/qemu-server.fw";
5876         if (-f $fwcfgfn) {
5877             my $pve_firewall_dir = '/etc/pve/firewall';
5878             mkdir $pve_firewall_dir; # make sure the dir exists
5879             PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
5880         }
5881
5882         while (defined(my $line = <$fh>)) {
5883             if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
5884                 my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
5885                 die "archive does not contain data for drive '$virtdev'\n"
5886                     if !$devinfo->{$devname};
5887                 if (defined($opts->{storage})) {
5888                     $storeid = $opts->{storage} || 'local';
5889                 } elsif (!$storeid) {
5890                     $storeid = 'local';
5891                 }
5892                 $format = 'raw' if !$format;
5893                 $devinfo->{$devname}->{devname} = $devname;
5894                 $devinfo->{$devname}->{virtdev} = $virtdev;
5895                 $devinfo->{$devname}->{format} = $format;
5896                 $devinfo->{$devname}->{storeid} = $storeid;
5897
5898                 # check permission on storage
5899                 my $pool = $opts->{pool}; # todo: do we need that?
5900                 if ($user ne 'root@pam') {
5901                     $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
5902                 }
5903
5904                 $storage_limits{$storeid} = $bwlimit;
5905
5906                 $virtdev_hash->{$virtdev} = $devinfo->{$devname};
5907             }
5908         }
5909
5910         foreach my $key (keys %storage_limits) {
5911             my $limit = PVE::Storage::get_bandwidth_limit('restore', [$key], $bwlimit);
5912             next if !$limit;
5913             print STDERR "rate limit for storage $key: $limit KiB/s\n";
5914             $storage_limits{$key} = $limit * 1024;
5915         }
5916
5917         foreach my $devname (keys %$devinfo) {
5918             die "found no device mapping information for device '$devname'\n"
5919                 if !$devinfo->{$devname}->{virtdev};
5920         }
5921
5922         # create empty/temp config
5923         if ($oldconf) {
5924             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
5925             foreach_drive($oldconf, sub {
5926                 my ($ds, $drive) = @_;
5927
5928                 return if drive_is_cdrom($drive);
5929
5930                 my $volid = $drive->{file};
5931
5932                 return if !$volid || $volid =~ m|^/|;
5933
5934                 my ($path, $owner) = PVE::Storage::path($cfg, $volid);
5935                 return if !$path || !$owner || ($owner != $vmid);
5936
5937                 # Note: only delete disk we want to restore
5938                 # other volumes will become unused
5939                 if ($virtdev_hash->{$ds}) {
5940                     eval { PVE::Storage::vdisk_free($cfg, $volid); };
5941                     if (my $err = $@) {
5942                         warn $err;
5943                     }
5944                 }
5945             });
5946
5947             # delete vmstate files
5948             # since after the restore we have no snapshots anymore
5949             foreach my $snapname (keys %{$oldconf->{snapshots}}) {
5950                 my $snap = $oldconf->{snapshots}->{$snapname};
5951                 if ($snap->{vmstate}) {
5952                     eval { PVE::Storage::vdisk_free($cfg, $snap->{vmstate}); };
5953                     if (my $err = $@) {
5954                         warn $err;
5955                     }
5956                 }
5957             }
5958         }
5959
5960         my $map = {};
5961         foreach my $virtdev (sort keys %$virtdev_hash) {
5962             my $d = $virtdev_hash->{$virtdev};
5963             my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
5964             my $storeid = $d->{storeid};
5965             my $scfg = PVE::Storage::storage_config($cfg, $storeid);
5966
5967             my $map_opts = '';
5968             if (my $limit = $storage_limits{$storeid}) {
5969                 $map_opts .= "throttling.bps=$limit:throttling.group=$storeid:";
5970             }
5971
5972             # test if requested format is supported
5973             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($cfg, $storeid);
5974             my $supported = grep { $_ eq $d->{format} } @$validFormats;
5975             $d->{format} = $defFormat if !$supported;
5976
5977             my $volid = PVE::Storage::vdisk_alloc($cfg, $storeid, $vmid,
5978                                                   $d->{format}, undef, $alloc_size);
5979             print STDERR "new volume ID is '$volid'\n";
5980             $d->{volid} = $volid;
5981             my $path = PVE::Storage::path($cfg, $volid);
5982
5983             PVE::Storage::activate_volumes($cfg,[$volid]);
5984
5985             my $write_zeros = 1;
5986             if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
5987                 $write_zeros = 0;
5988             }
5989
5990             print $fifofh "${map_opts}format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
5991
5992             print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
5993             $map->{$virtdev} = $volid;
5994         }
5995
5996         $fh->seek(0, 0) || die "seek failed - $!\n";
5997
5998         my $outfd = new IO::File ($tmpfn, "w") ||
5999             die "unable to write config for VM $vmid\n";
6000
6001         my $cookie = { netcount => 0 };
6002         while (defined(my $line = <$fh>)) {
6003             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
6004         }
6005
6006         $fh->close();
6007         $outfd->close();
6008     };
6009
6010     eval {
6011         # enable interrupts
6012         local $SIG{INT} =
6013             local $SIG{TERM} =
6014             local $SIG{QUIT} =
6015             local $SIG{HUP} =
6016             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6017         local $SIG{ALRM} = sub { die "got timeout\n"; };
6018
6019         $oldtimeout = alarm($timeout);
6020
6021         my $parser = sub {
6022             my $line = shift;
6023
6024             print "$line\n";
6025
6026             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
6027                 my ($dev_id, $size, $devname) = ($1, $2, $3);
6028                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
6029             } elsif ($line =~ m/^CTIME: /) {
6030                 # we correctly received the vma config, so we can disable
6031                 # the timeout now for disk allocation (set to 10 minutes, so
6032                 # that we always timeout if something goes wrong)
6033                 alarm(600);
6034                 &$print_devmap();
6035                 print $fifofh "done\n";
6036                 my $tmp = $oldtimeout || 0;
6037                 $oldtimeout = undef;
6038                 alarm($tmp);
6039                 close($fifofh);
6040             }
6041         };
6042
6043         print "restore vma archive: $dbg_cmdstring\n";
6044         run_command($commands, input => $input, outfunc => $parser, afterfork => $openfifo);
6045     };
6046     my $err = $@;
6047
6048     alarm($oldtimeout) if $oldtimeout;
6049
6050     my $vollist = [];
6051     foreach my $devname (keys %$devinfo) {
6052         my $volid = $devinfo->{$devname}->{volid};
6053         push @$vollist, $volid if $volid;
6054     }
6055
6056     PVE::Storage::deactivate_volumes($cfg, $vollist);
6057
6058     unlink $mapfifo;
6059
6060     if ($err) {
6061         rmtree $tmpdir;
6062         unlink $tmpfn;
6063
6064         foreach my $devname (keys %$devinfo) {
6065             my $volid = $devinfo->{$devname}->{volid};
6066             next if !$volid;
6067             eval {
6068                 if ($volid =~ m|^/|) {
6069                     unlink $volid || die 'unlink failed\n';
6070                 } else {
6071                     PVE::Storage::vdisk_free($cfg, $volid);
6072                 }
6073                 print STDERR "temporary volume '$volid' sucessfuly removed\n";
6074             };
6075             print STDERR "unable to cleanup '$volid' - $@" if $@;
6076         }
6077         die $err;
6078     }
6079
6080     rmtree $tmpdir;
6081
6082     rename($tmpfn, $conffile) ||
6083         die "unable to commit configuration file '$conffile'\n";
6084
6085     PVE::Cluster::cfs_update(); # make sure we read new file
6086
6087     eval { rescan($vmid, 1); };
6088     warn $@ if $@;
6089 }
6090
6091 sub restore_tar_archive {
6092     my ($archive, $vmid, $user, $opts) = @_;
6093
6094     if ($archive ne '-') {
6095         my $firstfile = tar_archive_read_firstfile($archive);
6096         die "ERROR: file '$archive' dos not lock like a QemuServer vzdump backup\n"
6097             if $firstfile ne 'qemu-server.conf';
6098     }
6099
6100     my $storecfg = PVE::Storage::config();
6101
6102     # destroy existing data - keep empty config
6103     my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
6104     destroy_vm($storecfg, $vmid, 1) if -f $vmcfgfn;
6105
6106     my $tocmd = "/usr/lib/qemu-server/qmextract";
6107
6108     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
6109     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
6110     $tocmd .= ' --prealloc' if $opts->{prealloc};
6111     $tocmd .= ' --info' if $opts->{info};
6112
6113     # tar option "xf" does not autodetect compression when read from STDIN,
6114     # so we pipe to zcat
6115     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
6116         PVE::Tools::shellquote("--to-command=$tocmd");
6117
6118     my $tmpdir = "/var/tmp/vzdumptmp$$";
6119     mkpath $tmpdir;
6120
6121     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
6122     local $ENV{VZDUMP_VMID} = $vmid;
6123     local $ENV{VZDUMP_USER} = $user;
6124
6125     my $conffile = PVE::QemuConfig->config_file($vmid);
6126     my $tmpfn = "$conffile.$$.tmp";
6127
6128     # disable interrupts (always do cleanups)
6129     local $SIG{INT} =
6130         local $SIG{TERM} =
6131         local $SIG{QUIT} =
6132         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
6133
6134     eval {
6135         # enable interrupts
6136         local $SIG{INT} =
6137             local $SIG{TERM} =
6138             local $SIG{QUIT} =
6139             local $SIG{HUP} =
6140             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6141
6142         if ($archive eq '-') {
6143             print "extracting archive from STDIN\n";
6144             run_command($cmd, input => "<&STDIN");
6145         } else {
6146             print "extracting archive '$archive'\n";
6147             run_command($cmd);
6148         }
6149
6150         return if $opts->{info};
6151
6152         # read new mapping
6153         my $map = {};
6154         my $statfile = "$tmpdir/qmrestore.stat";
6155         if (my $fd = IO::File->new($statfile, "r")) {
6156             while (defined (my $line = <$fd>)) {
6157                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
6158                     $map->{$1} = $2 if $1;
6159                 } else {
6160                     print STDERR "unable to parse line in statfile - $line\n";
6161                 }
6162             }
6163             $fd->close();
6164         }
6165
6166         my $confsrc = "$tmpdir/qemu-server.conf";
6167
6168         my $srcfd = new IO::File($confsrc, "r") ||
6169             die "unable to open file '$confsrc'\n";
6170
6171         my $outfd = new IO::File ($tmpfn, "w") ||
6172             die "unable to write config for VM $vmid\n";
6173
6174         my $cookie = { netcount => 0 };
6175         while (defined (my $line = <$srcfd>)) {
6176             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
6177         }
6178
6179         $srcfd->close();
6180         $outfd->close();
6181     };
6182     my $err = $@;
6183
6184     if ($err) {
6185
6186         unlink $tmpfn;
6187
6188         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
6189
6190         die $err;
6191     }
6192
6193     rmtree $tmpdir;
6194
6195     rename $tmpfn, $conffile ||
6196         die "unable to commit configuration file '$conffile'\n";
6197
6198     PVE::Cluster::cfs_update(); # make sure we read new file
6199
6200     eval { rescan($vmid, 1); };
6201     warn $@ if $@;
6202 };
6203
6204 sub foreach_storage_used_by_vm {
6205     my ($conf, $func) = @_;
6206
6207     my $sidhash = {};
6208
6209     foreach_drive($conf, sub {
6210         my ($ds, $drive) = @_;
6211         return if drive_is_cdrom($drive);
6212
6213         my $volid = $drive->{file};
6214
6215         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
6216         $sidhash->{$sid} = $sid if $sid;
6217     });
6218
6219     foreach my $sid (sort keys %$sidhash) {
6220         &$func($sid);
6221     }
6222 }
6223
6224 sub do_snapshots_with_qemu {
6225     my ($storecfg, $volid) = @_;
6226
6227     my $storage_name = PVE::Storage::parse_volume_id($volid);
6228
6229     if ($qemu_snap_storage->{$storecfg->{ids}->{$storage_name}->{type}}
6230         && !$storecfg->{ids}->{$storage_name}->{krbd}){
6231         return 1;
6232     }
6233
6234     if ($volid =~ m/\.(qcow2|qed)$/){
6235         return 1;
6236     }
6237
6238     return undef;
6239 }
6240
6241 sub qga_check_running {
6242     my ($vmid, $nowarn) = @_;
6243
6244     eval { vm_mon_cmd($vmid, "guest-ping", timeout => 3); };
6245     if ($@) {
6246         warn "Qemu Guest Agent is not running - $@" if !$nowarn;
6247         return 0;
6248     }
6249     return 1;
6250 }
6251
6252 sub template_create {
6253     my ($vmid, $conf, $disk) = @_;
6254
6255     my $storecfg = PVE::Storage::config();
6256
6257     foreach_drive($conf, sub {
6258         my ($ds, $drive) = @_;
6259
6260         return if drive_is_cdrom($drive);
6261         return if $disk && $ds ne $disk;
6262
6263         my $volid = $drive->{file};
6264         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
6265
6266         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
6267         $drive->{file} = $voliddst;
6268         $conf->{$ds} = print_drive($vmid, $drive);
6269         PVE::QemuConfig->write_config($vmid, $conf);
6270     });
6271 }
6272
6273 sub qemu_img_convert {
6274     my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized) = @_;
6275
6276     my $storecfg = PVE::Storage::config();
6277     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
6278     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
6279
6280     if ($src_storeid && $dst_storeid) {
6281
6282         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
6283
6284         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
6285         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6286
6287         my $src_format = qemu_img_format($src_scfg, $src_volname);
6288         my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
6289
6290         my $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
6291         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6292
6293         my $cmd = [];
6294         push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
6295         push @$cmd, '-l', "snapshot.name=$snapname" if($snapname && $src_format eq "qcow2");
6296         push @$cmd, '-t', 'none' if $dst_scfg->{type} eq 'zfspool';
6297         push @$cmd, '-T', 'none' if $src_scfg->{type} eq 'zfspool';
6298         push @$cmd, '-f', $src_format, '-O', $dst_format, $src_path;
6299         if ($is_zero_initialized) {
6300             push @$cmd, "zeroinit:$dst_path";
6301         } else {
6302             push @$cmd, $dst_path;
6303         }
6304
6305         my $parser = sub {
6306             my $line = shift;
6307             if($line =~ m/\((\S+)\/100\%\)/){
6308                 my $percent = $1;
6309                 my $transferred = int($size * $percent / 100);
6310                 my $remaining = $size - $transferred;
6311
6312                 print "transferred: $transferred bytes remaining: $remaining bytes total: $size bytes progression: $percent %\n";
6313             }
6314
6315         };
6316
6317         eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
6318         my $err = $@;
6319         die "copy failed: $err" if $err;
6320     }
6321 }
6322
6323 sub qemu_img_format {
6324     my ($scfg, $volname) = @_;
6325
6326     if ($scfg->{path} && $volname =~ m/\.($QEMU_FORMAT_RE)$/) {
6327         return $1;
6328     } else {
6329         return "raw";
6330     }
6331 }
6332
6333 sub qemu_drive_mirror {
6334     my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $skipcomplete, $qga) = @_;
6335
6336     $jobs = {} if !$jobs;
6337
6338     my $qemu_target;
6339     my $format;
6340     $jobs->{"drive-$drive"} = {};
6341
6342     if ($dst_volid =~ /^nbd:/) {
6343         $qemu_target = $dst_volid;
6344         $format = "nbd";
6345     } else {
6346         my $storecfg = PVE::Storage::config();
6347         my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
6348
6349         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6350
6351         $format = qemu_img_format($dst_scfg, $dst_volname);
6352
6353         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6354
6355         $qemu_target = $is_zero_initialized ? "zeroinit:$dst_path" : $dst_path;
6356     }
6357
6358     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $qemu_target };
6359     $opts->{format} = $format if $format;
6360
6361     print "drive mirror is starting for drive-$drive\n";
6362
6363     eval { vm_mon_cmd($vmid, "drive-mirror", %$opts); }; #if a job already run for this device,it's throw an error
6364
6365     if (my $err = $@) {
6366         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6367         die "mirroring error: $err";
6368     }
6369
6370     qemu_drive_mirror_monitor ($vmid, $vmiddst, $jobs, $skipcomplete, $qga);
6371 }
6372
6373 sub qemu_drive_mirror_monitor {
6374     my ($vmid, $vmiddst, $jobs, $skipcomplete, $qga) = @_;
6375
6376     eval {
6377         my $err_complete = 0;
6378
6379         while (1) {
6380             die "storage migration timed out\n" if $err_complete > 300;
6381
6382             my $stats = vm_mon_cmd($vmid, "query-block-jobs");
6383
6384             my $running_mirror_jobs = {};
6385             foreach my $stat (@$stats) {
6386                 next if $stat->{type} ne 'mirror';
6387                 $running_mirror_jobs->{$stat->{device}} = $stat;
6388             }
6389
6390             my $readycounter = 0;
6391
6392             foreach my $job (keys %$jobs) {
6393
6394                 if(defined($jobs->{$job}->{complete}) && !defined($running_mirror_jobs->{$job})) {
6395                     print "$job : finished\n";
6396                     delete $jobs->{$job};
6397                     next;
6398                 }
6399
6400                 die "$job: mirroring has been cancelled\n" if !defined($running_mirror_jobs->{$job});
6401
6402                 my $busy = $running_mirror_jobs->{$job}->{busy};
6403                 my $ready = $running_mirror_jobs->{$job}->{ready};
6404                 if (my $total = $running_mirror_jobs->{$job}->{len}) {
6405                     my $transferred = $running_mirror_jobs->{$job}->{offset} || 0;
6406                     my $remaining = $total - $transferred;
6407                     my $percent = sprintf "%.2f", ($transferred * 100 / $total);
6408
6409                     print "$job: transferred: $transferred bytes remaining: $remaining bytes total: $total bytes progression: $percent % busy: $busy ready: $ready \n";
6410                 }
6411
6412                 $readycounter++ if $running_mirror_jobs->{$job}->{ready};
6413             }
6414
6415             last if scalar(keys %$jobs) == 0;
6416
6417             if ($readycounter == scalar(keys %$jobs)) {
6418                 print "all mirroring jobs are ready \n";
6419                 last if $skipcomplete; #do the complete later
6420
6421                 if ($vmiddst && $vmiddst != $vmid) {
6422                     my $agent_running = $qga && qga_check_running($vmid);
6423                     if ($agent_running) {
6424                         print "freeze filesystem\n";
6425                         eval { PVE::QemuServer::vm_mon_cmd($vmid, "guest-fsfreeze-freeze"); };
6426                     } else {
6427                         print "suspend vm\n";
6428                         eval { PVE::QemuServer::vm_suspend($vmid, 1); };
6429                     }
6430
6431                     # if we clone a disk for a new target vm, we don't switch the disk
6432                     PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs);
6433
6434                     if ($agent_running) {
6435                         print "unfreeze filesystem\n";
6436                         eval { PVE::QemuServer::vm_mon_cmd($vmid, "guest-fsfreeze-thaw"); };
6437                     } else {
6438                         print "resume vm\n";
6439                         eval {  PVE::QemuServer::vm_resume($vmid, 1, 1); };
6440                     }
6441
6442                     last;
6443                 } else {
6444
6445                     foreach my $job (keys %$jobs) {
6446                         # try to switch the disk if source and destination are on the same guest
6447                         print "$job: Completing block job...\n";
6448
6449                         eval { vm_mon_cmd($vmid, "block-job-complete", device => $job) };
6450                         if ($@ =~ m/cannot be completed/) {
6451                             print "$job: Block job cannot be completed, try again.\n";
6452                             $err_complete++;
6453                         }else {
6454                             print "$job: Completed successfully.\n";
6455                             $jobs->{$job}->{complete} = 1;
6456                         }
6457                     }
6458                 }
6459             }
6460             sleep 1;
6461         }
6462     };
6463     my $err = $@;
6464
6465     if ($err) {
6466         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6467         die "mirroring error: $err";
6468     }
6469
6470 }
6471
6472 sub qemu_blockjobs_cancel {
6473     my ($vmid, $jobs) = @_;
6474
6475     foreach my $job (keys %$jobs) {
6476         print "$job: Cancelling block job\n";
6477         eval { vm_mon_cmd($vmid, "block-job-cancel", device => $job); };
6478         $jobs->{$job}->{cancel} = 1;
6479     }
6480
6481     while (1) {
6482         my $stats = vm_mon_cmd($vmid, "query-block-jobs");
6483
6484         my $running_jobs = {};
6485         foreach my $stat (@$stats) {
6486             $running_jobs->{$stat->{device}} = $stat;
6487         }
6488
6489         foreach my $job (keys %$jobs) {
6490
6491             if (defined($jobs->{$job}->{cancel}) && !defined($running_jobs->{$job})) {
6492                 print "$job: Done.\n";
6493                 delete $jobs->{$job};
6494             }
6495         }
6496
6497         last if scalar(keys %$jobs) == 0;
6498
6499         sleep 1;
6500     }
6501 }
6502
6503 sub clone_disk {
6504     my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
6505         $newvmid, $storage, $format, $full, $newvollist, $jobs, $skipcomplete, $qga) = @_;
6506
6507     my $newvolid;
6508
6509     if (!$full) {
6510         print "create linked clone of drive $drivename ($drive->{file})\n";
6511         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
6512         push @$newvollist, $newvolid;
6513     } else {
6514
6515         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6516         $storeid = $storage if $storage;
6517
6518         my $dst_format = resolve_dst_disk_format($storecfg, $storeid, $volname, $format);
6519         my ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 3);
6520
6521         print "create full clone of drive $drivename ($drive->{file})\n";
6522         my $name = undef;
6523         if (drive_is_cloudinit($drive)) {
6524             $name = "vm-$newvmid-cloudinit";
6525             # cloudinit only supports raw and qcow2 atm:
6526             if ($dst_format eq 'qcow2') {
6527                 $name .= '.qcow2';
6528             } elsif ($dst_format ne 'raw') {
6529                 die "clone: unhandled format for cloudinit image\n";
6530             }
6531         }
6532         $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $newvmid, $dst_format, $name, ($size/1024));
6533         push @$newvollist, $newvolid;
6534
6535         PVE::Storage::activate_volumes($storecfg, [$newvolid]);
6536
6537         my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
6538         if (!$running || $snapname) {
6539             qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit);
6540         } else {
6541
6542             my $kvmver = get_running_qemu_version ($vmid);
6543             if (!qemu_machine_feature_enabled (undef, $kvmver, 2, 7)) {
6544                 die "drive-mirror with iothread requires qemu version 2.7 or higher\n"
6545                     if $drive->{iothread};
6546             }
6547
6548             qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid, $sparseinit, $jobs, $skipcomplete, $qga);
6549         }
6550     }
6551
6552     my ($size) = PVE::Storage::volume_size_info($storecfg, $newvolid, 3);
6553
6554     my $disk = $drive;
6555     $disk->{format} = undef;
6556     $disk->{file} = $newvolid;
6557     $disk->{size} = $size;
6558
6559     return $disk;
6560 }
6561
6562 # this only works if VM is running
6563 sub get_current_qemu_machine {
6564     my ($vmid) = @_;
6565
6566     my $cmd = { execute => 'query-machines', arguments => {} };
6567     my $res = vm_qmp_command($vmid, $cmd);
6568
6569     my ($current, $default);
6570     foreach my $e (@$res) {
6571         $default = $e->{name} if $e->{'is-default'};
6572         $current = $e->{name} if $e->{'is-current'};
6573     }
6574
6575     # fallback to the default machine if current is not supported by qemu
6576     return $current || $default || 'pc';
6577 }
6578
6579 sub get_running_qemu_version {
6580     my ($vmid) = @_;
6581     my $cmd = { execute => 'query-version', arguments => {} };
6582     my $res = vm_qmp_command($vmid, $cmd);
6583     return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
6584 }
6585
6586 sub qemu_machine_feature_enabled {
6587     my ($machine, $kvmver, $version_major, $version_minor) = @_;
6588
6589     my $current_major;
6590     my $current_minor;
6591
6592     if ($machine && $machine =~ m/^(pc(-i440fx|-q35)?-(\d+)\.(\d+))/) {
6593
6594         $current_major = $3;
6595         $current_minor = $4;
6596
6597     } elsif ($kvmver =~ m/^(\d+)\.(\d+)/) {
6598
6599         $current_major = $1;
6600         $current_minor = $2;
6601     }
6602
6603     return 1 if $current_major > $version_major ||
6604                 ($current_major == $version_major &&
6605                  $current_minor >= $version_minor);
6606 }
6607
6608 sub qemu_machine_pxe {
6609     my ($vmid, $conf, $machine) = @_;
6610
6611     $machine =  PVE::QemuServer::get_current_qemu_machine($vmid) if !$machine;
6612
6613     if ($conf->{machine} && $conf->{machine} =~ m/\.pxe$/) {
6614         $machine .= '.pxe';
6615     }
6616
6617     return $machine;
6618 }
6619
6620 sub qemu_use_old_bios_files {
6621     my ($machine_type) = @_;
6622
6623     return if !$machine_type;
6624
6625     my $use_old_bios_files = undef;
6626
6627     if ($machine_type =~ m/^(\S+)\.pxe$/) {
6628         $machine_type = $1;
6629         $use_old_bios_files = 1;
6630     } else {
6631         my $kvmver = kvm_user_version();
6632         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
6633         # load new efi bios files on migration. So this hack is required to allow
6634         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
6635         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
6636         $use_old_bios_files = !qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 4);
6637     }
6638
6639     return ($use_old_bios_files, $machine_type);
6640 }
6641
6642 sub create_efidisk {
6643     my ($storecfg, $storeid, $vmid, $fmt) = @_;
6644
6645     die "EFI vars default image not found\n" if ! -f $OVMF_VARS;
6646
6647     my $vars_size = PVE::Tools::convert_size(-s $OVMF_VARS, 'b' => 'kb');
6648     my $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $fmt, undef, $vars_size);
6649     PVE::Storage::activate_volumes($storecfg, [$volid]);
6650
6651     my $path = PVE::Storage::path($storecfg, $volid);
6652     eval {
6653         run_command(['/usr/bin/qemu-img', 'convert', '-n', '-f', 'raw', '-O', $fmt, $OVMF_VARS, $path]);
6654     };
6655     die "Copying EFI vars image failed: $@" if $@;
6656
6657     return ($volid, $vars_size);
6658 }
6659
6660 sub lspci {
6661
6662     my $devices = {};
6663
6664     dir_glob_foreach("$pcisysfs/devices", '[a-f0-9]{4}:([a-f0-9]{2}:[a-f0-9]{2})\.([0-9])', sub {
6665             my (undef, $id, $function) = @_;
6666             my $res = { id => $id, function => $function};
6667             push @{$devices->{$id}}, $res;
6668     });
6669
6670     # Entries should be sorted by functions.
6671     foreach my $id (keys %$devices) {
6672         my $dev = $devices->{$id};
6673         $devices->{$id} = [ sort { $a->{function} <=> $b->{function} } @$dev ];
6674     }
6675
6676     return $devices;
6677 }
6678
6679 sub vm_iothreads_list {
6680     my ($vmid) = @_;
6681
6682     my $res = vm_mon_cmd($vmid, 'query-iothreads');
6683
6684     my $iothreads = {};
6685     foreach my $iothread (@$res) {
6686         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
6687     }
6688
6689     return $iothreads;
6690 }
6691
6692 sub scsihw_infos {
6693     my ($conf, $drive) = @_;
6694
6695     my $maxdev = 0;
6696
6697     if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)) {
6698         $maxdev = 7;
6699     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
6700         $maxdev = 1;
6701     } else {
6702         $maxdev = 256;
6703     }
6704
6705     my $controller = int($drive->{index} / $maxdev);
6706     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single') ? "virtioscsi" : "scsihw";
6707
6708     return ($maxdev, $controller, $controller_prefix);
6709 }
6710
6711 sub add_hyperv_enlightenments {
6712     my ($cpuFlags, $winversion, $machine_type, $kvmver, $bios, $gpu_passthrough) = @_;
6713
6714     return if $winversion < 6;
6715     return if $bios && $bios eq 'ovmf' && $winversion < 8;
6716
6717     push @$cpuFlags , 'hv_vendor_id=proxmox' if $gpu_passthrough;
6718
6719     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
6720         push @$cpuFlags , 'hv_spinlocks=0x1fff';
6721         push @$cpuFlags , 'hv_vapic';
6722         push @$cpuFlags , 'hv_time';
6723     } else {
6724         push @$cpuFlags , 'hv_spinlocks=0xffff';
6725     }
6726
6727     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 6)) {
6728         push @$cpuFlags , 'hv_reset';
6729         push @$cpuFlags , 'hv_vpindex';
6730         push @$cpuFlags , 'hv_runtime';
6731     }
6732
6733     if ($winversion >= 7) {
6734         push @$cpuFlags , 'hv_relaxed';
6735
6736         if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 12)) {
6737             push @$cpuFlags , 'hv_synic';
6738             push @$cpuFlags , 'hv_stimer';
6739         }
6740     }
6741 }
6742
6743 sub windows_version {
6744     my ($ostype) = @_;
6745
6746     return 0 if !$ostype;
6747
6748     my $winversion = 0;
6749
6750     if($ostype eq 'wxp' || $ostype eq 'w2k3' || $ostype eq 'w2k') {
6751         $winversion = 5;
6752     } elsif($ostype eq 'w2k8' || $ostype eq 'wvista') {
6753         $winversion = 6;
6754     } elsif ($ostype =~ m/^win(\d+)$/) {
6755         $winversion = $1;
6756     }
6757
6758     return $winversion;
6759 }
6760
6761 sub resolve_dst_disk_format {
6762         my ($storecfg, $storeid, $src_volname, $format) = @_;
6763         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
6764
6765         if (!$format) {
6766             # if no target format is specified, use the source disk format as hint
6767             if ($src_volname) {
6768                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6769                 $format = qemu_img_format($scfg, $src_volname);
6770             } else {
6771                 return $defFormat;
6772             }
6773         }
6774
6775         # test if requested format is supported - else use default
6776         my $supported = grep { $_ eq $format } @$validFormats;
6777         $format = $defFormat if !$supported;
6778         return $format;
6779 }
6780
6781 sub resolve_first_disk {
6782     my $conf = shift;
6783     my @disks = PVE::QemuServer::valid_drive_names();
6784     my $firstdisk;
6785     foreach my $ds (reverse @disks) {
6786         next if !$conf->{$ds};
6787         my $disk = PVE::QemuServer::parse_drive($ds, $conf->{$ds});
6788         next if PVE::QemuServer::drive_is_cdrom($disk);
6789         $firstdisk = $ds;
6790     }
6791     return $firstdisk;
6792 }
6793
6794 sub generate_uuid {
6795     my ($uuid, $uuid_str);
6796     UUID::generate($uuid);
6797     UUID::unparse($uuid, $uuid_str);
6798     return $uuid_str;
6799 }
6800
6801 sub generate_smbios1_uuid {
6802     return "uuid=".generate_uuid();
6803 }
6804
6805 # bash completion helper
6806
6807 sub complete_backup_archives {
6808     my ($cmdname, $pname, $cvalue) = @_;
6809
6810     my $cfg = PVE::Storage::config();
6811
6812     my $storeid;
6813
6814     if ($cvalue =~ m/^([^:]+):/) {
6815         $storeid = $1;
6816     }
6817
6818     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
6819
6820     my $res = [];
6821     foreach my $id (keys %$data) {
6822         foreach my $item (@{$data->{$id}}) {
6823             next if $item->{format} !~ m/^vma\.(gz|lzo)$/;
6824             push @$res, $item->{volid} if defined($item->{volid});
6825         }
6826     }
6827
6828     return $res;
6829 }
6830
6831 my $complete_vmid_full = sub {
6832     my ($running) = @_;
6833
6834     my $idlist = vmstatus();
6835
6836     my $res = [];
6837
6838     foreach my $id (keys %$idlist) {
6839         my $d = $idlist->{$id};
6840         if (defined($running)) {
6841             next if $d->{template};
6842             next if $running && $d->{status} ne 'running';
6843             next if !$running && $d->{status} eq 'running';
6844         }
6845         push @$res, $id;
6846
6847     }
6848     return $res;
6849 };
6850
6851 sub complete_vmid {
6852     return &$complete_vmid_full();
6853 }
6854
6855 sub complete_vmid_stopped {
6856     return &$complete_vmid_full(0);
6857 }
6858
6859 sub complete_vmid_running {
6860     return &$complete_vmid_full(1);
6861 }
6862
6863 sub complete_storage {
6864
6865     my $cfg = PVE::Storage::config();
6866     my $ids = $cfg->{ids};
6867
6868     my $res = [];
6869     foreach my $sid (keys %$ids) {
6870         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
6871         next if !$ids->{$sid}->{content}->{images};
6872         push @$res, $sid;
6873     }
6874
6875     return $res;
6876 }
6877
6878 sub nbd_stop {
6879     my ($vmid) = @_;
6880
6881     vm_mon_cmd($vmid, 'nbd-server-stop');
6882 }
6883
6884 1;