PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5 use POSIX;
   6 use IO::Handle;
   7 use IO::Select;
   8 use IO::File;
   9 use IO::Dir;
  10 use IO::Socket::UNIX;
  11 use File::Basename;
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use Digest::SHA;
  16 use Fcntl ':flock';
  17 use Cwd 'abs_path';
  18 use IPC::Open3;
  19 use JSON;
  20 use Fcntl;
  21 use PVE::SafeSyslog;
  22 use Storable qw(dclone);
  23 use PVE::Exception qw(raise raise_param_exc);
  24 use PVE::Storage;
  25 use PVE::Tools qw(run_command lock_file lock_file_full file_read_firstline dir_glob_foreach $IPV6RE);
  26 use PVE::JSONSchema qw(get_standard_option);
  27 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file cfs_lock_file);
  28 use PVE::INotify;
  29 use PVE::ProcFSTools;
  30 use PVE::QemuConfig;
  31 use PVE::QMPClient;
  32 use PVE::RPCEnvironment;
  33 use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr);
  34 use PVE::QemuServer::Memory;
  35 use PVE::QemuServer::USB qw(parse_usb_device);
  36 use PVE::QemuServer::Cloudinit;
  37 use PVE::Systemd;
  38 use Time::HiRes qw(gettimeofday);
  39 use File::Copy qw(copy);
  40 use URI::Escape;
  41
  42 my $EDK2_FW_BASE = '/usr/share/pve-edk2-firmware/';
  43 my $OVMF_CODE = "$EDK2_FW_BASE/OVMF_CODE.fd";
  44 my $OVMF_VARS = "$EDK2_FW_BASE/OVMF_VARS.fd";
  45
  46 my $qemu_snap_storage = {rbd => 1, sheepdog => 1};
  47
  48 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  49
  50 my $QEMU_FORMAT_RE = qr/raw|cow|qcow|qcow2|qed|vmdk|cloop/;
  51
  52 # Note about locking: we use flock on the config file protect
  53 # against concurent actions.
  54 # Aditionaly, we have a 'lock' setting in the config file. This
  55 # can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
  56 # allowed when such lock is set. But you can ignore this kind of
  57 # lock with the --skiplock flag.
  58
  59 cfs_register_file('/qemu-server/',
  60                   \&parse_vm_config,
  61                   \&write_vm_config);
  62
  63 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
  64     description => "Some command save/restore state from this location.",
  65     type => 'string',
  66     maxLength => 128,
  67     optional => 1,
  68 });
  69
  70 PVE::JSONSchema::register_standard_option('pve-snapshot-name', {
  71     description => "The name of the snapshot.",
  72     type => 'string', format => 'pve-configid',
  73     maxLength => 40,
  74 });
  75
  76 PVE::JSONSchema::register_standard_option('pve-qm-image-format', {
  77     type => 'string',
  78     enum => [qw(raw cow qcow qed qcow2 vmdk cloop)],
  79     description => "The drive's backing file's data format.",
  80     optional => 1,
  81 });
  82
  83 PVE::JSONSchema::register_standard_option('pve-qemu-machine', {
  84         description => "Specifies the Qemu machine type.",
  85         type => 'string',
  86         pattern => '(pc|pc(-i440fx)?-\d+\.\d+(\.pxe)?|q35|pc-q35-\d+\.\d+(\.pxe)?)',
  87         maxLength => 40,
  88         optional => 1,
  89 });
  90
  91 #no warnings 'redefine';
  92
  93 sub cgroups_write {
  94    my ($controller, $vmid, $option, $value) = @_;
  95
  96    my $path = "/sys/fs/cgroup/$controller/qemu.slice/$vmid.scope/$option";
  97    PVE::ProcFSTools::write_proc_entry($path, $value);
  98
  99 }
 100
 101 my $nodename = PVE::INotify::nodename();
 102
 103 mkdir "/etc/pve/nodes/$nodename";
 104 my $confdir = "/etc/pve/nodes/$nodename/qemu-server";
 105 mkdir $confdir;
 106
 107 my $var_run_tmpdir = "/var/run/qemu-server";
 108 mkdir $var_run_tmpdir;
 109
 110 my $lock_dir = "/var/lock/qemu-server";
 111 mkdir $lock_dir;
 112
 113 my $pcisysfs = "/sys/bus/pci";
 114
 115 my $cpu_vendor_list = {
 116     # Intel CPUs
 117     486 => 'GenuineIntel',
 118     pentium => 'GenuineIntel',
 119     pentium2  => 'GenuineIntel',
 120     pentium3  => 'GenuineIntel',
 121     coreduo => 'GenuineIntel',
 122     core2duo => 'GenuineIntel',
 123     Conroe  => 'GenuineIntel',
 124     Penryn  => 'GenuineIntel',
 125     Nehalem  => 'GenuineIntel',
 126     'Nehalem-IBRS'  => 'GenuineIntel',
 127     Westmere => 'GenuineIntel',
 128     'Westmere-IBRS' => 'GenuineIntel',
 129     SandyBridge => 'GenuineIntel',
 130     'SandyBridge-IBRS' => 'GenuineIntel',
 131     IvyBridge => 'GenuineIntel',
 132     'IvyBridge-IBRS' => 'GenuineIntel',
 133     Haswell => 'GenuineIntel',
 134     'Haswell-IBRS' => 'GenuineIntel',
 135     'Haswell-noTSX' => 'GenuineIntel',
 136     'Haswell-noTSX-IBRS' => 'GenuineIntel',
 137     Broadwell => 'GenuineIntel',
 138     'Broadwell-IBRS' => 'GenuineIntel',
 139     'Broadwell-noTSX' => 'GenuineIntel',
 140     'Broadwell-noTSX-IBRS' => 'GenuineIntel',
 141     'Skylake-Client' => 'GenuineIntel',
 142     'Skylake-Client-IBRS' => 'GenuineIntel',
 143     'Skylake-Server' => 'GenuineIntel',
 144     'Skylake-Server-IBRS' => 'GenuineIntel',
 145
 146     # AMD CPUs
 147     athlon => 'AuthenticAMD',
 148     phenom  => 'AuthenticAMD',
 149     Opteron_G1  => 'AuthenticAMD',
 150     Opteron_G2  => 'AuthenticAMD',
 151     Opteron_G3  => 'AuthenticAMD',
 152     Opteron_G4  => 'AuthenticAMD',
 153     Opteron_G5  => 'AuthenticAMD',
 154     EPYC => 'AuthenticAMD',
 155     'EPYC-IBPB' => 'AuthenticAMD',
 156
 157     # generic types, use vendor from host node
 158     host => 'default',
 159     kvm32 => 'default',
 160     kvm64 => 'default',
 161     qemu32 => 'default',
 162     qemu64 => 'default',
 163     max => 'default',
 164 };
 165
 166 my $cpu_flag = qr/[+-](pcid|spec-ctrl|ibpb|ssbd|virt-ssbd|amd-ssbd|amd-no-ssb|pdpe1gb)/;
 167
 168 my $cpu_fmt = {
 169     cputype => {
 170         description => "Emulated CPU type.",
 171         type => 'string',
 172         enum => [ sort { "\L$a" cmp "\L$b" } keys %$cpu_vendor_list ],
 173         default => 'kvm64',
 174         default_key => 1,
 175     },
 176     hidden => {
 177         description => "Do not identify as a KVM virtual machine.",
 178         type => 'boolean',
 179         optional => 1,
 180         default => 0
 181     },
 182     flags => {
 183         description => "List of additional CPU flags separated by ';'."
 184                      . " Use '+FLAG' to enable, '-FLAG' to disable a flag."
 185                      . " Currently supported flags: 'pcid', 'spec-ctrl', 'ibpb', 'ssbd', 'virt-ssbd', 'amd-ssbd', 'amd-no-ssb', 'pdpe1gb'.",
 186         format_description => '+FLAG[;-FLAG...]',
 187         type => 'string',
 188         pattern => qr/$cpu_flag(;$cpu_flag)*/,
 189         optional => 1,
 190     },
 191 };
 192
 193 my $watchdog_fmt = {
 194     model => {
 195         default_key => 1,
 196         type => 'string',
 197         enum => [qw(i6300esb ib700)],
 198         description => "Watchdog type to emulate.",
 199         default => 'i6300esb',
 200         optional => 1,
 201     },
 202     action => {
 203         type => 'string',
 204         enum => [qw(reset shutdown poweroff pause debug none)],
 205         description => "The action to perform if after activation the guest fails to poll the watchdog in time.",
 206         optional => 1,
 207     },
 208 };
 209 PVE::JSONSchema::register_format('pve-qm-watchdog', $watchdog_fmt);
 210
 211 my $agent_fmt = {
 212     enabled => {
 213         description => "Enable/disable Qemu GuestAgent.",
 214         type => 'boolean',
 215         default => 0,
 216         default_key => 1,
 217     },
 218     fstrim_cloned_disks => {
 219         description => "Run fstrim after cloning/moving a disk.",
 220         type => 'boolean',
 221         optional => 1,
 222         default => 0
 223     },
 224 };
 225
 226 my $confdesc = {
 227     onboot => {
 228         optional => 1,
 229         type => 'boolean',
 230         description => "Specifies whether a VM will be started during system bootup.",
 231         default => 0,
 232     },
 233     autostart => {
 234         optional => 1,
 235         type => 'boolean',
 236         description => "Automatic restart after crash (currently ignored).",
 237         default => 0,
 238     },
 239     hotplug => {
 240         optional => 1,
 241         type => 'string', format => 'pve-hotplug-features',
 242         description => "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
 243         default => 'network,disk,usb',
 244     },
 245     reboot => {
 246         optional => 1,
 247         type => 'boolean',
 248         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 249         default => 1,
 250     },
 251     lock => {
 252         optional => 1,
 253         type => 'string',
 254         description => "Lock/unlock the VM.",
 255         enum => [qw(migrate backup snapshot rollback)],
 256     },
 257     cpulimit => {
 258         optional => 1,
 259         type => 'number',
 260         description => "Limit of CPU usage.",
 261         verbose_description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
 262         minimum => 0,
 263         maximum => 128,
 264         default => 0,
 265     },
 266     cpuunits => {
 267         optional => 1,
 268         type => 'integer',
 269         description => "CPU weight for a VM.",
 270         verbose_description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.",
 271         minimum => 2,
 272         maximum => 262144,
 273         default => 1024,
 274     },
 275     memory => {
 276         optional => 1,
 277         type => 'integer',
 278         description => "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
 279         minimum => 16,
 280         default => 512,
 281     },
 282     balloon => {
 283         optional => 1,
 284         type => 'integer',
 285         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 286         minimum => 0,
 287     },
 288     shares => {
 289         optional => 1,
 290         type => 'integer',
 291         description => "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning. Auto-ballooning is done by pvestatd.",
 292         minimum => 0,
 293         maximum => 50000,
 294         default => 1000,
 295     },
 296     keyboard => {
 297         optional => 1,
 298         type => 'string',
 299         description => "Keybord layout for vnc server. Default is read from the '/etc/pve/datacenter.cfg' configuration file.".
 300                        "It should not be necessary to set it.",
 301         enum => PVE::Tools::kvmkeymaplist(),
 302         default => undef,
 303     },
 304     name => {
 305         optional => 1,
 306         type => 'string', format => 'dns-name',
 307         description => "Set a name for the VM. Only used on the configuration web interface.",
 308     },
 309     scsihw => {
 310         optional => 1,
 311         type => 'string',
 312         description => "SCSI controller model",
 313         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 314         default => 'lsi',
 315     },
 316     description => {
 317         optional => 1,
 318         type => 'string',
 319         description => "Description for the VM. Only used on the configuration web interface. This is saved as comment inside the configuration file.",
 320     },
 321     ostype => {
 322         optional => 1,
 323         type => 'string',
 324         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 l24 l26 solaris)],
 325         description => "Specify guest operating system.",
 326         verbose_description => <<EODESC,
 327 Specify guest operating system. This is used to enable special
 328 optimization/features for specific operating systems:
 329
 330 [horizontal]
 331 other;; unspecified OS
 332 wxp;; Microsoft Windows XP
 333 w2k;; Microsoft Windows 2000
 334 w2k3;; Microsoft Windows 2003
 335 w2k8;; Microsoft Windows 2008
 336 wvista;; Microsoft Windows Vista
 337 win7;; Microsoft Windows 7
 338 win8;; Microsoft Windows 8/2012/2012r2
 339 win10;; Microsoft Windows 10/2016
 340 l24;; Linux 2.4 Kernel
 341 l26;; Linux 2.6/3.X Kernel
 342 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
 343 EODESC
 344     },
 345     boot => {
 346         optional => 1,
 347         type => 'string',
 348         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n).",
 349         pattern => '[acdn]{1,4}',
 350         default => 'cdn',
 351     },
 352     bootdisk => {
 353         optional => 1,
 354         type => 'string', format => 'pve-qm-bootdisk',
 355         description => "Enable booting from specified disk.",
 356         pattern => '(ide|sata|scsi|virtio)\d+',
 357     },
 358     smp => {
 359         optional => 1,
 360         type => 'integer',
 361         description => "The number of CPUs. Please use option -sockets instead.",
 362         minimum => 1,
 363         default => 1,
 364     },
 365     sockets => {
 366         optional => 1,
 367         type => 'integer',
 368         description => "The number of CPU sockets.",
 369         minimum => 1,
 370         default => 1,
 371     },
 372     cores => {
 373         optional => 1,
 374         type => 'integer',
 375         description => "The number of cores per socket.",
 376         minimum => 1,
 377         default => 1,
 378     },
 379     numa => {
 380         optional => 1,
 381         type => 'boolean',
 382         description => "Enable/disable NUMA.",
 383         default => 0,
 384     },
 385     hugepages => {
 386         optional => 1,
 387         type => 'string',
 388         description => "Enable/disable hugepages memory.",
 389         enum => [qw(any 2 1024)],
 390     },
 391     vcpus => {
 392         optional => 1,
 393         type => 'integer',
 394         description => "Number of hotplugged vcpus.",
 395         minimum => 1,
 396         default => 0,
 397     },
 398     acpi => {
 399         optional => 1,
 400         type => 'boolean',
 401         description => "Enable/disable ACPI.",
 402         default => 1,
 403     },
 404     agent => {
 405         optional => 1,
 406         description => "Enable/disable Qemu GuestAgent and its properties.",
 407         type => 'string',
 408         format => $agent_fmt,
 409     },
 410     kvm => {
 411         optional => 1,
 412         type => 'boolean',
 413         description => "Enable/disable KVM hardware virtualization.",
 414         default => 1,
 415     },
 416     tdf => {
 417         optional => 1,
 418         type => 'boolean',
 419         description => "Enable/disable time drift fix.",
 420         default => 0,
 421     },
 422     localtime => {
 423         optional => 1,
 424         type => 'boolean',
 425         description => "Set the real time clock to local time. This is enabled by default if ostype indicates a Microsoft OS.",
 426     },
 427     freeze => {
 428         optional => 1,
 429         type => 'boolean',
 430         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 431     },
 432     vga => {
 433         optional => 1,
 434         type => 'string',
 435         description => "Select the VGA type.",
 436         verbose_description => "Select the VGA type. If you want to use high resolution" .
 437             " modes (>= 1280x1024x16) then you should use the options " .
 438             "'std' or 'vmware'. Default is 'std' for win8/win7/w2k8, and " .
 439             "'cirrus' for other OS types. The 'qxl' option enables the SPICE " .
 440             "display sever. For win* OS you can select how many independent " .
 441             "displays you want, Linux guests can add displays them self. " .
 442             "You can also run without any graphic card, using a serial device" .
 443             " as terminal.",
 444         enum => [qw(std cirrus vmware qxl serial0 serial1 serial2 serial3 qxl2 qxl3 qxl4)],
 445     },
 446     watchdog => {
 447         optional => 1,
 448         type => 'string', format => 'pve-qm-watchdog',
 449         description => "Create a virtual hardware watchdog device.",
 450         verbose_description => "Create a virtual hardware watchdog device. Once enabled" .
 451             " (by a guest action), the watchdog must be periodically polled " .
 452             "by an agent inside the guest or else the watchdog will reset " .
 453             "the guest (or execute the respective action specified)",
 454     },
 455     startdate => {
 456         optional => 1,
 457         type => 'string',
 458         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 459         description => "Set the initial date of the real time clock. Valid format for date are: 'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 460         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 461         default => 'now',
 462     },
 463     startup =>  get_standard_option('pve-startup-order'),
 464     template => {
 465         optional => 1,
 466         type => 'boolean',
 467         description => "Enable/disable Template.",
 468         default => 0,
 469     },
 470     args => {
 471         optional => 1,
 472         type => 'string',
 473         description => "Arbitrary arguments passed to kvm.",
 474         verbose_description => <<EODESCR,
 475 Arbitrary arguments passed to kvm, for example:
 476
 477 args: -no-reboot -no-hpet
 478
 479 NOTE: this option is for experts only.
 480 EODESCR
 481     },
 482     tablet => {
 483         optional => 1,
 484         type => 'boolean',
 485         default => 1,
 486         description => "Enable/disable the USB tablet device.",
 487         verbose_description => "Enable/disable the USB tablet device. This device is " .
 488             "usually needed to allow absolute mouse positioning with VNC. " .
 489             "Else the mouse runs out of sync with normal VNC clients. " .
 490             "If you're running lots of console-only guests on one host, " .
 491             "you may consider disabling this to save some context switches. " .
 492             "This is turned off by default if you use spice (-vga=qxl).",
 493     },
 494     migrate_speed => {
 495         optional => 1,
 496         type => 'integer',
 497         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 498         minimum => 0,
 499         default => 0,
 500     },
 501     migrate_downtime => {
 502         optional => 1,
 503         type => 'number',
 504         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 505         minimum => 0,
 506         default => 0.1,
 507     },
 508     cdrom => {
 509         optional => 1,
 510         type => 'string', format => 'pve-qm-ide',
 511         typetext => '<volume>',
 512         description => "This is an alias for option -ide2",
 513     },
 514     cpu => {
 515         optional => 1,
 516         description => "Emulated CPU type.",
 517         type => 'string',
 518         format => $cpu_fmt,
 519     },
 520     parent => get_standard_option('pve-snapshot-name', {
 521         optional => 1,
 522         description => "Parent snapshot name. This is used internally, and should not be modified.",
 523     }),
 524     snaptime => {
 525         optional => 1,
 526         description => "Timestamp for snapshots.",
 527         type => 'integer',
 528         minimum => 0,
 529     },
 530     vmstate => {
 531         optional => 1,
 532         type => 'string', format => 'pve-volume-id',
 533         description => "Reference to a volume which stores the VM state. This is used internally for snapshots.",
 534     },
 535     vmstatestorage => get_standard_option('pve-storage-id', {
 536         description => "Default storage for VM state volumes/files.",
 537         optional => 1,
 538     }),
 539     runningmachine => get_standard_option('pve-qemu-machine', {
 540         description => "Specifies the Qemu machine type of the running vm. This is used internally for snapshots.",
 541     }),
 542     machine => get_standard_option('pve-qemu-machine'),
 543     smbios1 => {
 544         description => "Specify SMBIOS type 1 fields.",
 545         type => 'string', format => 'pve-qm-smbios1',
 546         maxLength => 256,
 547         optional => 1,
 548     },
 549     protection => {
 550         optional => 1,
 551         type => 'boolean',
 552         description => "Sets the protection flag of the VM. This will disable the remove VM and remove disk operations.",
 553         default => 0,
 554     },
 555     bios => {
 556         optional => 1,
 557         type => 'string',
 558         enum => [ qw(seabios ovmf) ],
 559         description => "Select BIOS implementation.",
 560         default => 'seabios',
 561     },
 562     vmgenid => {
 563         type => 'string',
 564         pattern => '(?:[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}|[01])',
 565         format_description => 'UUID',
 566         description => "Set VM Generation ID UUID. Use special value 1 to autogenerate one (API only). Use special value 0 to disable explicitly.",
 567         optional => 1,
 568     },
 569 };
 570
 571 my $confdesc_cloudinit = {
 572     citype => {
 573         optional => 1,
 574         type => 'string',
 575         description => 'Specifies the cloud-init configuration format. The default depends on the configured operating system type (`ostype`. We use the `nocloud` format for Linux, and `configdrive2` for windows.',
 576         enum => ['configdrive2', 'nocloud'],
 577     },
 578     ciuser => {
 579         optional => 1,
 580         type => 'string',
 581         description => "cloud-init: User name to change ssh keys and password for instead of the image's configured default user.",
 582     },
 583     cipassword => {
 584         optional => 1,
 585         type => 'string',
 586         description => 'cloud-init: Password to assign the user. Using this is generally not recommended. Use ssh keys instead. Also note that older cloud-init versions do not support hashed passwords.',
 587     },
 588     searchdomain => {
 589         optional => 1,
 590         type => 'string',
 591         description => "cloud-init: Sets DNS search domains for a container. Create will automatically use the setting from the host if neither searchdomain nor nameserver are set.",
 592     },
 593     nameserver => {
 594         optional => 1,
 595         type => 'string', format => 'address-list',
 596         description => "cloud-init: Sets DNS server IP address for a container. Create will automatically use the setting from the host if neither searchdomain nor nameserver are set.",
 597     },
 598     sshkeys => {
 599         optional => 1,
 600         type => 'string',
 601         format => 'urlencoded',
 602         description => "cloud-init: Setup public SSH keys (one key per line, OpenSSH format).",
 603     },
 604 };
 605
 606 # what about other qemu settings ?
 607 #cpu => 'string',
 608 #machine => 'string',
 609 #fda => 'file',
 610 #fdb => 'file',
 611 #mtdblock => 'file',
 612 #sd => 'file',
 613 #pflash => 'file',
 614 #snapshot => 'bool',
 615 #bootp => 'file',
 616 ##tftp => 'dir',
 617 ##smb => 'dir',
 618 #kernel => 'file',
 619 #append => 'string',
 620 #initrd => 'file',
 621 ##soundhw => 'string',
 622
 623 while (my ($k, $v) = each %$confdesc) {
 624     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 625 }
 626
 627 my $MAX_IDE_DISKS = 4;
 628 my $MAX_SCSI_DISKS = 14;
 629 my $MAX_VIRTIO_DISKS = 16;
 630 my $MAX_SATA_DISKS = 6;
 631 my $MAX_USB_DEVICES = 5;
 632 my $MAX_NETS = 32;
 633 my $MAX_UNUSED_DISKS = 8;
 634 my $MAX_HOSTPCI_DEVICES = 4;
 635 my $MAX_SERIAL_PORTS = 4;
 636 my $MAX_PARALLEL_PORTS = 3;
 637 my $MAX_NUMA = 8;
 638
 639 my $numa_fmt = {
 640     cpus => {
 641         type => "string",
 642         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 643         description => "CPUs accessing this NUMA node.",
 644         format_description => "id[-id];...",
 645     },
 646     memory => {
 647         type => "number",
 648         description => "Amount of memory this NUMA node provides.",
 649         optional => 1,
 650     },
 651     hostnodes => {
 652         type => "string",
 653         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 654         description => "Host NUMA nodes to use.",
 655         format_description => "id[-id];...",
 656         optional => 1,
 657     },
 658     policy => {
 659         type => 'string',
 660         enum => [qw(preferred bind interleave)],
 661         description => "NUMA allocation policy.",
 662         optional => 1,
 663     },
 664 };
 665 PVE::JSONSchema::register_format('pve-qm-numanode', $numa_fmt);
 666 my $numadesc = {
 667     optional => 1,
 668     type => 'string', format => $numa_fmt,
 669     description => "NUMA topology.",
 670 };
 671 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 672
 673 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 674     $confdesc->{"numa$i"} = $numadesc;
 675 }
 676
 677 my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000',  'pcnet',  'virtio',
 678                       'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3',
 679                       'e1000-82540em', 'e1000-82544gc', 'e1000-82545em'];
 680 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 681
 682 my $net_fmt_bridge_descr = <<__EOD__;
 683 Bridge to attach the network device to. The Proxmox VE standard bridge
 684 is called 'vmbr0'.
 685
 686 If you do not specify a bridge, we create a kvm user (NATed) network
 687 device, which provides DHCP and DNS services. The following addresses
 688 are used:
 689
 690  10.0.2.2   Gateway
 691  10.0.2.3   DNS Server
 692  10.0.2.4   SMB Server
 693
 694 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 695 __EOD__
 696
 697 my $net_fmt = {
 698     macaddr => {
 699         type => 'string',
 700         pattern => qr/[0-9a-f]{2}(?::[0-9a-f]{2}){5}/i,
 701         description => "MAC address. That address must be unique withing your network. This is automatically generated if not specified.",
 702         format_description => "XX:XX:XX:XX:XX:XX",
 703         optional => 1,
 704     },
 705     model => {
 706         type => 'string',
 707         description => "Network Card Model. The 'virtio' model provides the best performance with very low CPU overhead. If your guest does not support this driver, it is usually best to use 'e1000'.",
 708         enum => $nic_model_list,
 709         default_key => 1,
 710     },
 711     (map { $_ => { keyAlias => 'model', alias => 'macaddr' }} @$nic_model_list),
 712     bridge => {
 713         type => 'string',
 714         description => $net_fmt_bridge_descr,
 715         format_description => 'bridge',
 716         optional => 1,
 717     },
 718     queues => {
 719         type => 'integer',
 720         minimum => 0, maximum => 16,
 721         description => 'Number of packet queues to be used on the device.',
 722         optional => 1,
 723     },
 724     rate => {
 725         type => 'number',
 726         minimum => 0,
 727         description => "Rate limit in mbps (megabytes per second) as floating point number.",
 728         optional => 1,
 729     },
 730     tag => {
 731         type => 'integer',
 732         minimum => 1, maximum => 4094,
 733         description => 'VLAN tag to apply to packets on this interface.',
 734         optional => 1,
 735     },
 736     trunks => {
 737         type => 'string',
 738         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 739         description => 'VLAN trunks to pass through this interface.',
 740         format_description => 'vlanid[;vlanid...]',
 741         optional => 1,
 742     },
 743     firewall => {
 744         type => 'boolean',
 745         description => 'Whether this interface should be protected by the firewall.',
 746         optional => 1,
 747     },
 748     link_down => {
 749         type => 'boolean',
 750         description => 'Whether this interface should be disconnected (like pulling the plug).',
 751         optional => 1,
 752     },
 753 };
 754
 755 my $netdesc = {
 756     optional => 1,
 757     type => 'string', format => $net_fmt,
 758     description => "Specify network devices.",
 759 };
 760
 761 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 762
 763 my $ipconfig_fmt = {
 764     ip => {
 765         type => 'string',
 766         format => 'pve-ipv4-config',
 767         format_description => 'IPv4Format/CIDR',
 768         description => 'IPv4 address in CIDR format.',
 769         optional => 1,
 770         default => 'dhcp',
 771     },
 772     gw => {
 773         type => 'string',
 774         format => 'ipv4',
 775         format_description => 'GatewayIPv4',
 776         description => 'Default gateway for IPv4 traffic.',
 777         optional => 1,
 778         requires => 'ip',
 779     },
 780     ip6 => {
 781         type => 'string',
 782         format => 'pve-ipv6-config',
 783         format_description => 'IPv6Format/CIDR',
 784         description => 'IPv6 address in CIDR format.',
 785         optional => 1,
 786         default => 'dhcp',
 787     },
 788     gw6 => {
 789         type => 'string',
 790         format => 'ipv6',
 791         format_description => 'GatewayIPv6',
 792         description => 'Default gateway for IPv6 traffic.',
 793         optional => 1,
 794         requires => 'ip6',
 795     },
 796 };
 797 PVE::JSONSchema::register_format('pve-qm-ipconfig', $ipconfig_fmt);
 798 my $ipconfigdesc = {
 799     optional => 1,
 800     type => 'string', format => 'pve-qm-ipconfig',
 801     description => <<'EODESCR',
 802 cloud-init: Specify IP addresses and gateways for the corresponding interface.
 803
 804 IP addresses use CIDR notation, gateways are optional but need an IP of the same type specified.
 805
 806 The special string 'dhcp' can be used for IP addresses to use DHCP, in which case no explicit gateway should be provided.
 807 For IPv6 the special string 'auto' can be used to use stateless autoconfiguration.
 808
 809 If cloud-init is enabled and neither an IPv4 nor an IPv6 address is specified, it defaults to using dhcp on IPv4.
 810 EODESCR
 811 };
 812 PVE::JSONSchema::register_standard_option("pve-qm-ipconfig", $netdesc);
 813
 814 for (my $i = 0; $i < $MAX_NETS; $i++)  {
 815     $confdesc->{"net$i"} = $netdesc;
 816     $confdesc_cloudinit->{"ipconfig$i"} = $ipconfigdesc;
 817 }
 818
 819 foreach my $key (keys %$confdesc_cloudinit) {
 820     $confdesc->{$key} = $confdesc_cloudinit->{$key};
 821 }
 822
 823 PVE::JSONSchema::register_format('pve-volume-id-or-qm-path', \&verify_volume_id_or_qm_path);
 824 sub verify_volume_id_or_qm_path {
 825     my ($volid, $noerr) = @_;
 826
 827     if ($volid eq 'none' || $volid eq 'cdrom' || $volid =~ m|^/|) {
 828         return $volid;
 829     }
 830
 831     # if its neither 'none' nor 'cdrom' nor a path, check if its a volume-id
 832     $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') };
 833     if ($@) {
 834         return undef if $noerr;
 835         die $@;
 836     }
 837     return $volid;
 838 }
 839
 840 my $drivename_hash;
 841
 842 my %drivedesc_base = (
 843     volume => { alias => 'file' },
 844     file => {
 845         type => 'string',
 846         format => 'pve-volume-id-or-qm-path',
 847         default_key => 1,
 848         format_description => 'volume',
 849         description => "The drive's backing volume.",
 850     },
 851     media => {
 852         type => 'string',
 853         enum => [qw(cdrom disk)],
 854         description => "The drive's media type.",
 855         default => 'disk',
 856         optional => 1
 857     },
 858     cyls => {
 859         type => 'integer',
 860         description => "Force the drive's physical geometry to have a specific cylinder count.",
 861         optional => 1
 862     },
 863     heads => {
 864         type => 'integer',
 865         description => "Force the drive's physical geometry to have a specific head count.",
 866         optional => 1
 867     },
 868     secs => {
 869         type => 'integer',
 870         description => "Force the drive's physical geometry to have a specific sector count.",
 871         optional => 1
 872     },
 873     trans => {
 874         type => 'string',
 875         enum => [qw(none lba auto)],
 876         description => "Force disk geometry bios translation mode.",
 877         optional => 1,
 878     },
 879     snapshot => {
 880         type => 'boolean',
 881         description => "Controls qemu's snapshot mode feature."
 882             . " If activated, changes made to the disk are temporary and will"
 883             . " be discarded when the VM is shutdown.",
 884         optional => 1,
 885     },
 886     cache => {
 887         type => 'string',
 888         enum => [qw(none writethrough writeback unsafe directsync)],
 889         description => "The drive's cache mode",
 890         optional => 1,
 891     },
 892     format => get_standard_option('pve-qm-image-format'),
 893     size => {
 894         type => 'string',
 895         format => 'disk-size',
 896         format_description => 'DiskSize',
 897         description => "Disk size. This is purely informational and has no effect.",
 898         optional => 1,
 899     },
 900     backup => {
 901         type => 'boolean',
 902         description => "Whether the drive should be included when making backups.",
 903         optional => 1,
 904     },
 905     replicate => {
 906         type => 'boolean',
 907         description => 'Whether the drive should considered for replication jobs.',
 908         optional => 1,
 909         default => 1,
 910     },
 911     rerror => {
 912         type => 'string',
 913         enum => [qw(ignore report stop)],
 914         description => 'Read error action.',
 915         optional => 1,
 916     },
 917     werror => {
 918         type => 'string',
 919         enum => [qw(enospc ignore report stop)],
 920         description => 'Write error action.',
 921         optional => 1,
 922     },
 923     aio => {
 924         type => 'string',
 925         enum => [qw(native threads)],
 926         description => 'AIO type to use.',
 927         optional => 1,
 928     },
 929     discard => {
 930         type => 'string',
 931         enum => [qw(ignore on)],
 932         description => 'Controls whether to pass discard/trim requests to the underlying storage.',
 933         optional => 1,
 934     },
 935     detect_zeroes => {
 936         type => 'boolean',
 937         description => 'Controls whether to detect and try to optimize writes of zeroes.',
 938         optional => 1,
 939     },
 940     serial => {
 941         type => 'string',
 942         format => 'urlencoded',
 943         format_description => 'serial',
 944         maxLength => 20*3, # *3 since it's %xx url enoded
 945         description => "The drive's reported serial number, url-encoded, up to 20 bytes long.",
 946         optional => 1,
 947     },
 948     shared => {
 949         type => 'boolean',
 950         description => 'Mark this locally-managed volume as available on all nodes',
 951         verbose_description => "Mark this locally-managed volume as available on all nodes.\n\nWARNING: This option does not share the volume automatically, it assumes it is shared already!",
 952         optional => 1,
 953         default => 0,
 954     }
 955 );
 956
 957 my %iothread_fmt = ( iothread => {
 958         type => 'boolean',
 959         description => "Whether to use iothreads for this drive",
 960         optional => 1,
 961 });
 962
 963 my %model_fmt = (
 964     model => {
 965         type => 'string',
 966         format => 'urlencoded',
 967         format_description => 'model',
 968         maxLength => 40*3, # *3 since it's %xx url enoded
 969         description => "The drive's reported model name, url-encoded, up to 40 bytes long.",
 970         optional => 1,
 971     },
 972 );
 973
 974 my %queues_fmt = (
 975     queues => {
 976         type => 'integer',
 977         description => "Number of queues.",
 978         minimum => 2,
 979         optional => 1
 980     }
 981 );
 982
 983 my %scsiblock_fmt = (
 984     scsiblock => {
 985         type => 'boolean',
 986         description => "whether to use scsi-block for full passthrough of host block device\n\nWARNING: can lead to I/O errors in combination with low memory or high memory fragmentation on host",
 987         optional => 1,
 988         default => 0,
 989     },
 990 );
 991
 992 my $add_throttle_desc = sub {
 993     my ($key, $type, $what, $unit, $longunit, $minimum) = @_;
 994     my $d = {
 995         type => $type,
 996         format_description => $unit,
 997         description => "Maximum $what in $longunit.",
 998         optional => 1,
 999     };
1000     $d->{minimum} = $minimum if defined($minimum);
1001     $drivedesc_base{$key} = $d;
1002 };
1003 # throughput: (leaky bucket)
1004 $add_throttle_desc->('bps',     'integer', 'r/w speed',   'bps',  'bytes per second');
1005 $add_throttle_desc->('bps_rd',  'integer', 'read speed',  'bps',  'bytes per second');
1006 $add_throttle_desc->('bps_wr',  'integer', 'write speed', 'bps',  'bytes per second');
1007 $add_throttle_desc->('mbps',    'number',  'r/w speed',   'mbps', 'megabytes per second');
1008 $add_throttle_desc->('mbps_rd', 'number',  'read speed',  'mbps', 'megabytes per second');
1009 $add_throttle_desc->('mbps_wr', 'number',  'write speed', 'mbps', 'megabytes per second');
1010 $add_throttle_desc->('iops',    'integer', 'r/w I/O',     'iops', 'operations per second');
1011 $add_throttle_desc->('iops_rd', 'integer', 'read I/O',    'iops', 'operations per second');
1012 $add_throttle_desc->('iops_wr', 'integer', 'write I/O',   'iops', 'operations per second');
1013
1014 # pools: (pool of IO before throttling starts taking effect)
1015 $add_throttle_desc->('mbps_max',    'number',  'unthrottled r/w pool',       'mbps', 'megabytes per second');
1016 $add_throttle_desc->('mbps_rd_max', 'number',  'unthrottled read pool',      'mbps', 'megabytes per second');
1017 $add_throttle_desc->('mbps_wr_max', 'number',  'unthrottled write pool',     'mbps', 'megabytes per second');
1018 $add_throttle_desc->('iops_max',    'integer', 'unthrottled r/w I/O pool',   'iops', 'operations per second');
1019 $add_throttle_desc->('iops_rd_max', 'integer', 'unthrottled read I/O pool',  'iops', 'operations per second');
1020 $add_throttle_desc->('iops_wr_max', 'integer', 'unthrottled write I/O pool', 'iops', 'operations per second');
1021
1022 # burst lengths
1023 $add_throttle_desc->('bps_max_length',     'integer', 'length of I/O bursts',       'seconds', 'seconds', 1);
1024 $add_throttle_desc->('bps_rd_max_length',  'integer', 'length of read I/O bursts',  'seconds', 'seconds', 1);
1025 $add_throttle_desc->('bps_wr_max_length',  'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
1026 $add_throttle_desc->('iops_max_length',    'integer', 'length of I/O bursts',       'seconds', 'seconds', 1);
1027 $add_throttle_desc->('iops_rd_max_length', 'integer', 'length of read I/O bursts',  'seconds', 'seconds', 1);
1028 $add_throttle_desc->('iops_wr_max_length', 'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
1029
1030 # legacy support
1031 $drivedesc_base{'bps_rd_length'} = { alias => 'bps_rd_max_length' };
1032 $drivedesc_base{'bps_wr_length'} = { alias => 'bps_wr_max_length' };
1033 $drivedesc_base{'iops_rd_length'} = { alias => 'iops_rd_max_length' };
1034 $drivedesc_base{'iops_wr_length'} = { alias => 'iops_wr_max_length' };
1035
1036 my $ide_fmt = {
1037     %drivedesc_base,
1038     %model_fmt,
1039 };
1040 PVE::JSONSchema::register_format("pve-qm-ide", $ide_fmt);
1041
1042 my $idedesc = {
1043     optional => 1,
1044     type => 'string', format => $ide_fmt,
1045     description => "Use volume as IDE hard disk or CD-ROM (n is 0 to " .($MAX_IDE_DISKS -1) . ").",
1046 };
1047 PVE::JSONSchema::register_standard_option("pve-qm-ide", $idedesc);
1048
1049 my $scsi_fmt = {
1050     %drivedesc_base,
1051     %iothread_fmt,
1052     %queues_fmt,
1053     %scsiblock_fmt,
1054 };
1055 my $scsidesc = {
1056     optional => 1,
1057     type => 'string', format => $scsi_fmt,
1058     description => "Use volume as SCSI hard disk or CD-ROM (n is 0 to " . ($MAX_SCSI_DISKS - 1) . ").",
1059 };
1060 PVE::JSONSchema::register_standard_option("pve-qm-scsi", $scsidesc);
1061
1062 my $sata_fmt = {
1063     %drivedesc_base,
1064 };
1065 my $satadesc = {
1066     optional => 1,
1067     type => 'string', format => $sata_fmt,
1068     description => "Use volume as SATA hard disk or CD-ROM (n is 0 to " . ($MAX_SATA_DISKS - 1). ").",
1069 };
1070 PVE::JSONSchema::register_standard_option("pve-qm-sata", $satadesc);
1071
1072 my $virtio_fmt = {
1073     %drivedesc_base,
1074     %iothread_fmt,
1075 };
1076 my $virtiodesc = {
1077     optional => 1,
1078     type => 'string', format => $virtio_fmt,
1079     description => "Use volume as VIRTIO hard disk (n is 0 to " . ($MAX_VIRTIO_DISKS - 1) . ").",
1080 };
1081 PVE::JSONSchema::register_standard_option("pve-qm-virtio", $virtiodesc);
1082
1083 my $alldrive_fmt = {
1084     %drivedesc_base,
1085     %iothread_fmt,
1086     %model_fmt,
1087     %queues_fmt,
1088     %scsiblock_fmt,
1089 };
1090
1091 my $efidisk_fmt = {
1092     volume => { alias => 'file' },
1093     file => {
1094         type => 'string',
1095         format => 'pve-volume-id-or-qm-path',
1096         default_key => 1,
1097         format_description => 'volume',
1098         description => "The drive's backing volume.",
1099     },
1100     format => get_standard_option('pve-qm-image-format'),
1101     size => {
1102         type => 'string',
1103         format => 'disk-size',
1104         format_description => 'DiskSize',
1105         description => "Disk size. This is purely informational and has no effect.",
1106         optional => 1,
1107     },
1108 };
1109
1110 my $efidisk_desc = {
1111     optional => 1,
1112     type => 'string', format => $efidisk_fmt,
1113     description => "Configure a Disk for storing EFI vars",
1114 };
1115
1116 PVE::JSONSchema::register_standard_option("pve-qm-efidisk", $efidisk_desc);
1117
1118 my $usb_fmt = {
1119     host => {
1120         default_key => 1,
1121         type => 'string', format => 'pve-qm-usb-device',
1122         format_description => 'HOSTUSBDEVICE|spice',
1123         description => <<EODESCR,
1124 The Host USB device or port or the value 'spice'. HOSTUSBDEVICE syntax is:
1125
1126  'bus-port(.port)*' (decimal numbers) or
1127  'vendor_id:product_id' (hexadeciaml numbers) or
1128  'spice'
1129
1130 You can use the 'lsusb -t' command to list existing usb devices.
1131
1132 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1133
1134 The value 'spice' can be used to add a usb redirection devices for spice.
1135 EODESCR
1136     },
1137     usb3 => {
1138         optional => 1,
1139         type => 'boolean',
1140         description => "Specifies whether if given host option is a USB3 device or port (this does currently not work reliably with spice redirection and is then ignored).",
1141         default => 0,
1142     },
1143 };
1144
1145 my $usbdesc = {
1146     optional => 1,
1147     type => 'string', format => $usb_fmt,
1148     description => "Configure an USB device (n is 0 to 4).",
1149 };
1150 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
1151
1152 # NOTE: the match-groups of this regex are used in parse_hostpci
1153 my $PCIRE = qr/([a-f0-9]{2}:[a-f0-9]{2})(?:\.([a-f0-9]))?/;
1154 my $hostpci_fmt = {
1155     host => {
1156         default_key => 1,
1157         type => 'string',
1158         pattern => qr/$PCIRE(;$PCIRE)*/,
1159         format_description => 'HOSTPCIID[;HOSTPCIID2...]',
1160         description => <<EODESCR,
1161 Host PCI device pass through. The PCI ID of a host's PCI device or a list
1162 of PCI virtual functions of the host. HOSTPCIID syntax is:
1163
1164 'bus:dev.func' (hexadecimal numbers)
1165
1166 You can us the 'lspci' command to list existing PCI devices.
1167 EODESCR
1168     },
1169     rombar => {
1170         type => 'boolean',
1171         description =>  "Specify whether or not the device's ROM will be visible in the guest's memory map.",
1172         optional => 1,
1173         default => 1,
1174     },
1175     romfile => {
1176         type => 'string',
1177         pattern => '[^,;]+',
1178         format_description => 'string',
1179         description => "Custom pci device rom filename (must be located in /usr/share/kvm/).",
1180         optional => 1,
1181     },
1182     pcie => {
1183         type => 'boolean',
1184         description =>  "Choose the PCI-express bus (needs the 'q35' machine model).",
1185         optional => 1,
1186         default => 0,
1187     },
1188     'x-vga' => {
1189         type => 'boolean',
1190         description =>  "Enable vfio-vga device support.",
1191         optional => 1,
1192         default => 0,
1193     },
1194 };
1195 PVE::JSONSchema::register_format('pve-qm-hostpci', $hostpci_fmt);
1196
1197 my $hostpcidesc = {
1198         optional => 1,
1199         type => 'string', format => 'pve-qm-hostpci',
1200         description => "Map host PCI devices into guest.",
1201         verbose_description =>  <<EODESCR,
1202 Map host PCI devices into guest.
1203
1204 NOTE: This option allows direct access to host hardware. So it is no longer
1205 possible to migrate such machines - use with special care.
1206
1207 CAUTION: Experimental! User reported problems with this option.
1208 EODESCR
1209 };
1210 PVE::JSONSchema::register_standard_option("pve-qm-hostpci", $hostpcidesc);
1211
1212 my $serialdesc = {
1213         optional => 1,
1214         type => 'string',
1215         pattern => '(/dev/.+|socket)',
1216         description =>  "Create a serial device inside the VM (n is 0 to 3)",
1217         verbose_description =>  <<EODESCR,
1218 Create a serial device inside the VM (n is 0 to 3), and pass through a
1219 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1220 host side (use 'qm terminal' to open a terminal connection).
1221
1222 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines - use with special care.
1223
1224 CAUTION: Experimental! User reported problems with this option.
1225 EODESCR
1226 };
1227
1228 my $paralleldesc= {
1229         optional => 1,
1230         type => 'string',
1231         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
1232         description =>  "Map host parallel devices (n is 0 to 2).",
1233         verbose_description =>  <<EODESCR,
1234 Map host parallel devices (n is 0 to 2).
1235
1236 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1237
1238 CAUTION: Experimental! User reported problems with this option.
1239 EODESCR
1240 };
1241
1242 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
1243     $confdesc->{"parallel$i"} = $paralleldesc;
1244 }
1245
1246 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
1247     $confdesc->{"serial$i"} = $serialdesc;
1248 }
1249
1250 for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
1251     $confdesc->{"hostpci$i"} = $hostpcidesc;
1252 }
1253
1254 for (my $i = 0; $i < $MAX_IDE_DISKS; $i++)  {
1255     $drivename_hash->{"ide$i"} = 1;
1256     $confdesc->{"ide$i"} = $idedesc;
1257 }
1258
1259 for (my $i = 0; $i < $MAX_SATA_DISKS; $i++)  {
1260     $drivename_hash->{"sata$i"} = 1;
1261     $confdesc->{"sata$i"} = $satadesc;
1262 }
1263
1264 for (my $i = 0; $i < $MAX_SCSI_DISKS; $i++)  {
1265     $drivename_hash->{"scsi$i"} = 1;
1266     $confdesc->{"scsi$i"} = $scsidesc ;
1267 }
1268
1269 for (my $i = 0; $i < $MAX_VIRTIO_DISKS; $i++)  {
1270     $drivename_hash->{"virtio$i"} = 1;
1271     $confdesc->{"virtio$i"} = $virtiodesc;
1272 }
1273
1274 $drivename_hash->{efidisk0} = 1;
1275 $confdesc->{efidisk0} = $efidisk_desc;
1276
1277 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
1278     $confdesc->{"usb$i"} = $usbdesc;
1279 }
1280
1281 my $unuseddesc = {
1282     optional => 1,
1283     type => 'string', format => 'pve-volume-id',
1284     description => "Reference to unused volumes. This is used internally, and should not be modified manually.",
1285 };
1286
1287 for (my $i = 0; $i < $MAX_UNUSED_DISKS; $i++)  {
1288     $confdesc->{"unused$i"} = $unuseddesc;
1289 }
1290
1291 my $kvm_api_version = 0;
1292
1293 sub kvm_version {
1294
1295     return $kvm_api_version if $kvm_api_version;
1296
1297     my $fh = IO::File->new("</dev/kvm") ||
1298         return 0;
1299
1300     if (my $v = $fh->ioctl(KVM_GET_API_VERSION(), 0)) {
1301         $kvm_api_version = $v;
1302     }
1303
1304     $fh->close();
1305
1306     return  $kvm_api_version;
1307 }
1308
1309 my $kvm_user_version;
1310
1311 sub kvm_user_version {
1312
1313     return $kvm_user_version if $kvm_user_version;
1314
1315     $kvm_user_version = 'unknown';
1316
1317     my $code = sub {
1318         my $line = shift;
1319         if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1320             $kvm_user_version = $2;
1321         }
1322     };
1323
1324     eval { run_command("kvm -version", outfunc => $code); };
1325     warn $@ if $@;
1326
1327     return $kvm_user_version;
1328
1329 }
1330
1331 my $kernel_has_vhost_net = -c '/dev/vhost-net';
1332
1333 sub valid_drive_names {
1334     # order is important - used to autoselect boot disk
1335     return ((map { "ide$_" } (0 .. ($MAX_IDE_DISKS - 1))),
1336             (map { "scsi$_" } (0 .. ($MAX_SCSI_DISKS - 1))),
1337             (map { "virtio$_" } (0 .. ($MAX_VIRTIO_DISKS - 1))),
1338             (map { "sata$_" } (0 .. ($MAX_SATA_DISKS - 1))),
1339             'efidisk0');
1340 }
1341
1342 sub is_valid_drivename {
1343     my $dev = shift;
1344
1345     return defined($drivename_hash->{$dev});
1346 }
1347
1348 sub option_exists {
1349     my $key = shift;
1350     return defined($confdesc->{$key});
1351 }
1352
1353 sub nic_models {
1354     return $nic_model_list;
1355 }
1356
1357 sub os_list_description {
1358
1359     return {
1360         other => 'Other',
1361         wxp => 'Windows XP',
1362         w2k => 'Windows 2000',
1363         w2k3 =>, 'Windows 2003',
1364         w2k8 => 'Windows 2008',
1365         wvista => 'Windows Vista',
1366         win7 => 'Windows 7',
1367         win8 => 'Windows 8/2012',
1368         win10 => 'Windows 10/2016',
1369         l24 => 'Linux 2.4',
1370         l26 => 'Linux 2.6',
1371     };
1372 }
1373
1374 my $cdrom_path;
1375
1376 sub get_cdrom_path {
1377
1378     return  $cdrom_path if $cdrom_path;
1379
1380     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
1381     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
1382     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
1383 }
1384
1385 sub get_iso_path {
1386     my ($storecfg, $vmid, $cdrom) = @_;
1387
1388     if ($cdrom eq 'cdrom') {
1389         return get_cdrom_path();
1390     } elsif ($cdrom eq 'none') {
1391         return '';
1392     } elsif ($cdrom =~ m|^/|) {
1393         return $cdrom;
1394     } else {
1395         return PVE::Storage::path($storecfg, $cdrom);
1396     }
1397 }
1398
1399 # try to convert old style file names to volume IDs
1400 sub filename_to_volume_id {
1401     my ($vmid, $file, $media) = @_;
1402
1403      if (!($file eq 'none' || $file eq 'cdrom' ||
1404           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1405
1406         return undef if $file =~ m|/|;
1407
1408         if ($media && $media eq 'cdrom') {
1409             $file = "local:iso/$file";
1410         } else {
1411             $file = "local:$vmid/$file";
1412         }
1413     }
1414
1415     return $file;
1416 }
1417
1418 sub verify_media_type {
1419     my ($opt, $vtype, $media) = @_;
1420
1421     return if !$media;
1422
1423     my $etype;
1424     if ($media eq 'disk') {
1425         $etype = 'images';
1426     } elsif ($media eq 'cdrom') {
1427         $etype = 'iso';
1428     } else {
1429         die "internal error";
1430     }
1431
1432     return if ($vtype eq $etype);
1433
1434     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1435 }
1436
1437 sub cleanup_drive_path {
1438     my ($opt, $storecfg, $drive) = @_;
1439
1440     # try to convert filesystem paths to volume IDs
1441
1442     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1443         ($drive->{file} !~ m|^/dev/.+|) &&
1444         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1445         ($drive->{file} !~ m/^\d+$/)) {
1446         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1447         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"}) if !$vtype;
1448         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1449         verify_media_type($opt, $vtype, $drive->{media});
1450         $drive->{file} = $volid;
1451     }
1452
1453     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1454 }
1455
1456 sub parse_hotplug_features {
1457     my ($data) = @_;
1458
1459     my $res = {};
1460
1461     return $res if $data eq '0';
1462
1463     $data = $confdesc->{hotplug}->{default} if $data eq '1';
1464
1465     foreach my $feature (PVE::Tools::split_list($data)) {
1466         if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
1467             $res->{$1} = 1;
1468         } else {
1469             die "invalid hotplug feature '$feature'\n";
1470         }
1471     }
1472     return $res;
1473 }
1474
1475 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1476 sub pve_verify_hotplug_features {
1477     my ($value, $noerr) = @_;
1478
1479     return $value if parse_hotplug_features($value);
1480
1481     return undef if $noerr;
1482
1483     die "unable to parse hotplug option\n";
1484 }
1485
1486 # ideX = [volume=]volume-id[,media=d][,cyls=c,heads=h,secs=s[,trans=t]]
1487 #        [,snapshot=on|off][,cache=on|off][,format=f][,backup=yes|no]
1488 #        [,rerror=ignore|report|stop][,werror=enospc|ignore|report|stop]
1489 #        [,aio=native|threads][,discard=ignore|on][,detect_zeroes=on|off]
1490 #        [,iothread=on][,serial=serial][,model=model]
1491
1492 sub parse_drive {
1493     my ($key, $data) = @_;
1494
1495     my ($interface, $index);
1496
1497     if ($key =~ m/^([^\d]+)(\d+)$/) {
1498         $interface = $1;
1499         $index = $2;
1500     } else {
1501         return undef;
1502     }
1503
1504     my $desc = $key =~ /^unused\d+$/ ? $alldrive_fmt
1505                                      : $confdesc->{$key}->{format};
1506     if (!$desc) {
1507         warn "invalid drive key: $key\n";
1508         return undef;
1509     }
1510     my $res = eval { PVE::JSONSchema::parse_property_string($desc, $data) };
1511     return undef if !$res;
1512     $res->{interface} = $interface;
1513     $res->{index} = $index;
1514
1515     my $error = 0;
1516     foreach my $opt (qw(bps bps_rd bps_wr)) {
1517         if (my $bps = defined(delete $res->{$opt})) {
1518             if (defined($res->{"m$opt"})) {
1519                 warn "both $opt and m$opt specified\n";
1520                 ++$error;
1521                 next;
1522             }
1523             $res->{"m$opt"} = sprintf("%.3f", $bps / (1024*1024.0));
1524         }
1525     }
1526
1527     # can't use the schema's 'requires' because of the mbps* => bps* "transforming aliases"
1528     for my $requirement (
1529         [mbps_max => 'mbps'],
1530         [mbps_rd_max => 'mbps_rd'],
1531         [mbps_wr_max => 'mbps_wr'],
1532         [miops_max => 'miops'],
1533         [miops_rd_max => 'miops_rd'],
1534         [miops_wr_max => 'miops_wr'],
1535         [bps_max_length => 'mbps_max'],
1536         [bps_rd_max_length => 'mbps_rd_max'],
1537         [bps_wr_max_length => 'mbps_wr_max'],
1538         [iops_max_length => 'iops_max'],
1539         [iops_rd_max_length => 'iops_rd_max'],
1540         [iops_wr_max_length => 'iops_wr_max']) {
1541         my ($option, $requires) = @$requirement;
1542         if ($res->{$option} && !$res->{$requires}) {
1543             warn "$option requires $requires\n";
1544             ++$error;
1545         }
1546     }
1547
1548     return undef if $error;
1549
1550     return undef if $res->{mbps_rd} && $res->{mbps};
1551     return undef if $res->{mbps_wr} && $res->{mbps};
1552     return undef if $res->{iops_rd} && $res->{iops};
1553     return undef if $res->{iops_wr} && $res->{iops};
1554
1555     if ($res->{media} && ($res->{media} eq 'cdrom')) {
1556         return undef if $res->{snapshot} || $res->{trans} || $res->{format};
1557         return undef if $res->{heads} || $res->{secs} || $res->{cyls};
1558         return undef if $res->{interface} eq 'virtio';
1559     }
1560
1561     if (my $size = $res->{size}) {
1562         return undef if !defined($res->{size} = PVE::JSONSchema::parse_size($size));
1563     }
1564
1565     return $res;
1566 }
1567
1568 sub print_drive {
1569     my ($vmid, $drive) = @_;
1570     my $data = { %$drive };
1571     delete $data->{$_} for qw(index interface);
1572     return PVE::JSONSchema::print_property_string($data, $alldrive_fmt);
1573 }
1574
1575 sub scsi_inquiry {
1576     my($fh, $noerr) = @_;
1577
1578     my $SG_IO = 0x2285;
1579     my $SG_GET_VERSION_NUM = 0x2282;
1580
1581     my $versionbuf = "\x00" x 8;
1582     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1583     if (!$ret) {
1584         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1585         return undef;
1586     }
1587     my $version = unpack("I", $versionbuf);
1588     if ($version < 30000) {
1589         die "scsi generic interface too old\n"  if !$noerr;
1590         return undef;
1591     }
1592
1593     my $buf = "\x00" x 36;
1594     my $sensebuf = "\x00" x 8;
1595     my $cmd = pack("C x3 C x1", 0x12, 36);
1596
1597     # see /usr/include/scsi/sg.h
1598     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1599
1600     my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1601                       length($sensebuf), 0, length($buf), $buf,
1602                       $cmd, $sensebuf, 6000);
1603
1604     $ret = ioctl($fh, $SG_IO, $packet);
1605     if (!$ret) {
1606         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1607         return undef;
1608     }
1609
1610     my @res = unpack($sg_io_hdr_t, $packet);
1611     if ($res[17] || $res[18]) {
1612         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1613         return undef;
1614     }
1615
1616     my $res = {};
1617     (my $byte0, my $byte1, $res->{vendor},
1618      $res->{product}, $res->{revision}) = unpack("C C x6 A8 A16 A4", $buf);
1619
1620     $res->{removable} = $byte1 & 128 ? 1 : 0;
1621     $res->{type} = $byte0 & 31;
1622
1623     return $res;
1624 }
1625
1626 sub path_is_scsi {
1627     my ($path) = @_;
1628
1629     my $fh = IO::File->new("+<$path") || return undef;
1630     my $res = scsi_inquiry($fh, 1);
1631     close($fh);
1632
1633     return $res;
1634 }
1635
1636 sub machine_type_is_q35 {
1637     my ($conf) = @_;
1638
1639     return $conf->{machine} && ($conf->{machine} =~ m/q35/) ? 1 : 0;
1640 }
1641
1642 sub print_tabletdevice_full {
1643     my ($conf) = @_;
1644
1645     my $q35 = machine_type_is_q35($conf);
1646
1647     # we use uhci for old VMs because tablet driver was buggy in older qemu
1648     my $usbbus = $q35 ? "ehci" : "uhci";
1649
1650     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1651 }
1652
1653 sub print_drivedevice_full {
1654     my ($storecfg, $conf, $vmid, $drive, $bridges) = @_;
1655
1656     my $device = '';
1657     my $maxdev = 0;
1658
1659     if ($drive->{interface} eq 'virtio') {
1660         my $pciaddr = print_pci_addr("$drive->{interface}$drive->{index}", $bridges);
1661         $device = "virtio-blk-pci,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}$pciaddr";
1662         $device .= ",iothread=iothread-$drive->{interface}$drive->{index}" if $drive->{iothread};
1663     } elsif ($drive->{interface} eq 'scsi') {
1664
1665         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1666         my $unit = $drive->{index} % $maxdev;
1667         my $devicetype = 'hd';
1668         my $path = '';
1669         if (drive_is_cdrom($drive)) {
1670             $devicetype = 'cd';
1671         } else {
1672             if ($drive->{file} =~ m|^/|) {
1673                 $path = $drive->{file};
1674                 if (my $info = path_is_scsi($path)) {
1675                     if ($info->{type} == 0 && $drive->{scsiblock}) {
1676                         $devicetype = 'block';
1677                     } elsif ($info->{type} == 1) { # tape
1678                         $devicetype = 'generic';
1679                     }
1680                 }
1681             } else {
1682                  $path = PVE::Storage::path($storecfg, $drive->{file});
1683             }
1684
1685             if($path =~ m/^iscsi\:\/\//){
1686                 $devicetype = 'generic';
1687             }
1688         }
1689
1690         if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)){
1691             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1692         } else {
1693             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0,lun=$drive->{index},drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1694         }
1695
1696     } elsif ($drive->{interface} eq 'ide'){
1697         $maxdev = 2;
1698         my $controller = int($drive->{index} / $maxdev);
1699         my $unit = $drive->{index} % $maxdev;
1700         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1701
1702         $device = "ide-$devicetype,bus=ide.$controller,unit=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1703         if ($devicetype eq 'hd' && (my $model = $drive->{model})) {
1704             $model = URI::Escape::uri_unescape($model);
1705             $device .= ",model=$model";
1706         }
1707     } elsif ($drive->{interface} eq 'sata'){
1708         my $controller = int($drive->{index} / $MAX_SATA_DISKS);
1709         my $unit = $drive->{index} % $MAX_SATA_DISKS;
1710         $device = "ide-drive,bus=ahci$controller.$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1711     } elsif ($drive->{interface} eq 'usb') {
1712         die "implement me";
1713         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1714     } else {
1715         die "unsupported interface type";
1716     }
1717
1718     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1719
1720     if (my $serial = $drive->{serial}) {
1721         $serial = URI::Escape::uri_unescape($serial);
1722         $device .= ",serial=$serial";
1723     }
1724
1725
1726     return $device;
1727 }
1728
1729 sub get_initiator_name {
1730     my $initiator;
1731
1732     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return undef;
1733     while (defined(my $line = <$fh>)) {
1734         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1735         $initiator = $1;
1736         last;
1737     }
1738     $fh->close();
1739
1740     return $initiator;
1741 }
1742
1743 sub print_drive_full {
1744     my ($storecfg, $vmid, $drive) = @_;
1745
1746     my $path;
1747     my $volid = $drive->{file};
1748     my $format;
1749
1750     if (drive_is_cdrom($drive)) {
1751         $path = get_iso_path($storecfg, $vmid, $volid);
1752     } else {
1753         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1754         if ($storeid) {
1755             $path = PVE::Storage::path($storecfg, $volid);
1756             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
1757             $format = qemu_img_format($scfg, $volname);
1758         } else {
1759             $path = $volid;
1760             $format = "raw";
1761         }
1762    }
1763
1764     my $opts = '';
1765     my @qemu_drive_options = qw(heads secs cyls trans media format cache rerror werror aio discard);
1766     foreach my $o (@qemu_drive_options) {
1767         $opts .= ",$o=$drive->{$o}" if defined($drive->{$o});
1768     }
1769
1770     # snapshot only accepts on|off
1771     if (defined($drive->{snapshot})) {
1772         my $v = $drive->{snapshot} ? 'on' : 'off';
1773         $opts .= ",snapshot=$v";
1774     }
1775
1776     foreach my $type (['', '-total'], [_rd => '-read'], [_wr => '-write']) {
1777         my ($dir, $qmpname) = @$type;
1778         if (my $v = $drive->{"mbps$dir"}) {
1779             $opts .= ",throttling.bps$qmpname=".int($v*1024*1024);
1780         }
1781         if (my $v = $drive->{"mbps${dir}_max"}) {
1782             $opts .= ",throttling.bps$qmpname-max=".int($v*1024*1024);
1783         }
1784         if (my $v = $drive->{"bps${dir}_max_length"}) {
1785             $opts .= ",throttling.bps$qmpname-max-length=$v";
1786         }
1787         if (my $v = $drive->{"iops${dir}"}) {
1788             $opts .= ",throttling.iops$qmpname=$v";
1789         }
1790         if (my $v = $drive->{"iops${dir}_max"}) {
1791             $opts .= ",throttling.iops$qmpname-max=$v";
1792         }
1793         if (my $v = $drive->{"iops${dir}_max_length"}) {
1794             $opts .= ",throttling.iops$qmpname-max-length=$v";
1795         }
1796     }
1797
1798     $opts .= ",format=$format" if $format && !$drive->{format};
1799
1800     my $cache_direct = 0;
1801
1802     if (my $cache = $drive->{cache}) {
1803         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1804     } elsif (!drive_is_cdrom($drive)) {
1805         $opts .= ",cache=none";
1806         $cache_direct = 1;
1807     }
1808
1809     # aio native works only with O_DIRECT
1810     if (!$drive->{aio}) {
1811         if($cache_direct) {
1812             $opts .= ",aio=native";
1813         } else {
1814             $opts .= ",aio=threads";
1815         }
1816     }
1817
1818     if (!drive_is_cdrom($drive)) {
1819         my $detectzeroes;
1820         if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
1821             $detectzeroes = 'off';
1822         } elsif ($drive->{discard}) {
1823             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
1824         } else {
1825             # This used to be our default with discard not being specified:
1826             $detectzeroes = 'on';
1827         }
1828         $opts .= ",detect-zeroes=$detectzeroes" if $detectzeroes;
1829     }
1830
1831     my $pathinfo = $path ? "file=$path," : '';
1832
1833     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1834 }
1835
1836 sub print_netdevice_full {
1837     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files) = @_;
1838
1839     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
1840
1841     my $device = $net->{model};
1842     if ($net->{model} eq 'virtio') {
1843          $device = 'virtio-net-pci';
1844      };
1845
1846     my $pciaddr = print_pci_addr("$netid", $bridges);
1847     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1848     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
1849         #Consider we have N queues, the number of vectors needed is 2*N + 2 (plus one config interrupt and control vq)
1850         my $vectors = $net->{queues} * 2 + 2;
1851         $tmpstr .= ",vectors=$vectors,mq=on";
1852     }
1853     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
1854
1855     if ($use_old_bios_files) {
1856         my $romfile;
1857         if ($device eq 'virtio-net-pci') {
1858             $romfile = 'pxe-virtio.rom';
1859         } elsif ($device eq 'e1000') {
1860             $romfile = 'pxe-e1000.rom';
1861         } elsif ($device eq 'ne2k') {
1862             $romfile = 'pxe-ne2k_pci.rom';
1863         } elsif ($device eq 'pcnet') {
1864             $romfile = 'pxe-pcnet.rom';
1865         } elsif ($device eq 'rtl8139') {
1866             $romfile = 'pxe-rtl8139.rom';
1867         }
1868         $tmpstr .= ",romfile=$romfile" if $romfile;
1869     }
1870
1871     return $tmpstr;
1872 }
1873
1874 sub print_netdev_full {
1875     my ($vmid, $conf, $net, $netid, $hotplug) = @_;
1876
1877     my $i = '';
1878     if ($netid =~ m/^net(\d+)$/) {
1879         $i = int($1);
1880     }
1881
1882     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
1883
1884     my $ifname = "tap${vmid}i$i";
1885
1886     # kvm uses TUNSETIFF ioctl, and that limits ifname length
1887     die "interface name '$ifname' is too long (max 15 character)\n"
1888         if length($ifname) >= 16;
1889
1890     my $vhostparam = '';
1891     $vhostparam = ',vhost=on' if $kernel_has_vhost_net && $net->{model} eq 'virtio';
1892
1893     my $vmname = $conf->{name} || "vm$vmid";
1894
1895     my $netdev = "";
1896     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
1897
1898     if ($net->{bridge}) {
1899         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script,downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1900     } else {
1901         $netdev = "type=user,id=$netid,hostname=$vmname";
1902     }
1903
1904     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
1905
1906     return $netdev;
1907 }
1908
1909
1910 sub print_cpu_device {
1911     my ($conf, $id) = @_;
1912
1913     my $kvm = $conf->{kvm} // 1;
1914     my $cpu = $kvm ? "kvm64" : "qemu64";
1915     if (my $cputype = $conf->{cpu}) {
1916         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
1917             or die "Cannot parse cpu description: $cputype\n";
1918         $cpu = $cpuconf->{cputype};
1919     }
1920
1921     my $cores = $conf->{cores} || 1;
1922
1923     my $current_core = ($id - 1) % $cores;
1924     my $current_socket = int(($id - 1 - $current_core)/$cores);
1925
1926     return "$cpu-x86_64-cpu,id=cpu$id,socket-id=$current_socket,core-id=$current_core,thread-id=0";
1927 }
1928
1929 sub drive_is_cloudinit {
1930     my ($drive) = @_;
1931     return $drive->{file} =~ m@[:/]vm-\d+-cloudinit(?:\.$QEMU_FORMAT_RE)?$@;
1932 }
1933
1934 sub drive_is_cdrom {
1935     my ($drive, $exclude_cloudinit) = @_;
1936
1937     return 0 if $exclude_cloudinit && drive_is_cloudinit($drive);
1938
1939     return $drive && $drive->{media} && ($drive->{media} eq 'cdrom');
1940
1941 }
1942
1943 sub parse_number_sets {
1944     my ($set) = @_;
1945     my $res = [];
1946     foreach my $part (split(/;/, $set)) {
1947         if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
1948             die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
1949             push @$res, [ $1, $2 ];
1950         } else {
1951             die "invalid range: $part\n";
1952         }
1953     }
1954     return $res;
1955 }
1956
1957 sub parse_numa {
1958     my ($data) = @_;
1959
1960     my $res = PVE::JSONSchema::parse_property_string($numa_fmt, $data);
1961     $res->{cpus} = parse_number_sets($res->{cpus}) if defined($res->{cpus});
1962     $res->{hostnodes} = parse_number_sets($res->{hostnodes}) if defined($res->{hostnodes});
1963     return $res;
1964 }
1965
1966 sub parse_hostpci {
1967     my ($value) = @_;
1968
1969     return undef if !$value;
1970
1971     my $res = PVE::JSONSchema::parse_property_string($hostpci_fmt, $value);
1972
1973     my @idlist = split(/;/, $res->{host});
1974     delete $res->{host};
1975     foreach my $id (@idlist) {
1976         if ($id =~ /^$PCIRE$/) {
1977             if (defined($2)) {
1978                 push @{$res->{pciid}}, { id => $1, function => $2 };
1979             } else {
1980                 my $pcidevices = lspci($1);
1981                 $res->{pciid} = $pcidevices->{$1};
1982             }
1983         } else {
1984             # should have been caught by parse_property_string already
1985             die "failed to parse PCI id: $id\n";
1986         }
1987     }
1988     return $res;
1989 }
1990
1991 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
1992 sub parse_net {
1993     my ($data) = @_;
1994
1995     my $res = eval { PVE::JSONSchema::parse_property_string($net_fmt, $data) };
1996     if ($@) {
1997         warn $@;
1998         return undef;
1999     }
2000     if (!defined($res->{macaddr})) {
2001         my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
2002         $res->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix});
2003     }
2004     return $res;
2005 }
2006
2007 # ipconfigX ip=cidr,gw=ip,ip6=cidr,gw6=ip
2008 sub parse_ipconfig {
2009     my ($data) = @_;
2010
2011     my $res = eval { PVE::JSONSchema::parse_property_string($ipconfig_fmt, $data) };
2012     if ($@) {
2013         warn $@;
2014         return undef;
2015     }
2016
2017     if ($res->{gw} && !$res->{ip}) {
2018         warn 'gateway specified without specifying an IP address';
2019         return undef;
2020     }
2021     if ($res->{gw6} && !$res->{ip6}) {
2022         warn 'IPv6 gateway specified without specifying an IPv6 address';
2023         return undef;
2024     }
2025     if ($res->{gw} && $res->{ip} eq 'dhcp') {
2026         warn 'gateway specified together with DHCP';
2027         return undef;
2028     }
2029     if ($res->{gw6} && $res->{ip6} !~ /^$IPV6RE/) {
2030         # gw6 + auto/dhcp
2031         warn "IPv6 gateway specified together with $res->{ip6} address";
2032         return undef;
2033     }
2034
2035     if (!$res->{ip} && !$res->{ip6}) {
2036         return { ip => 'dhcp', ip6 => 'dhcp' };
2037     }
2038
2039     return $res;
2040 }
2041
2042 sub print_net {
2043     my $net = shift;
2044
2045     return PVE::JSONSchema::print_property_string($net, $net_fmt);
2046 }
2047
2048 sub add_random_macs {
2049     my ($settings) = @_;
2050
2051     foreach my $opt (keys %$settings) {
2052         next if $opt !~ m/^net(\d+)$/;
2053         my $net = parse_net($settings->{$opt});
2054         next if !$net;
2055         $settings->{$opt} = print_net($net);
2056     }
2057 }
2058
2059 sub vm_is_volid_owner {
2060     my ($storecfg, $vmid, $volid) = @_;
2061
2062     if ($volid !~  m|^/|) {
2063         my ($path, $owner);
2064         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
2065         if ($owner && ($owner == $vmid)) {
2066             return 1;
2067         }
2068     }
2069
2070     return undef;
2071 }
2072
2073 sub split_flagged_list {
2074     my $text = shift || '';
2075     $text =~ s/[,;]/ /g;
2076     $text =~ s/^\s+//;
2077     return { map { /^(!?)(.*)$/ && ($2, $1) } ($text =~ /\S+/g) };
2078 }
2079
2080 sub join_flagged_list {
2081     my ($how, $lst) = @_;
2082     join $how, map { $lst->{$_} . $_ } keys %$lst;
2083 }
2084
2085 sub vmconfig_delete_pending_option {
2086     my ($conf, $key, $force) = @_;
2087
2088     delete $conf->{pending}->{$key};
2089     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
2090     $pending_delete_hash->{$key} = $force ? '!' : '';
2091     $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
2092 }
2093
2094 sub vmconfig_undelete_pending_option {
2095     my ($conf, $key) = @_;
2096
2097     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
2098     delete $pending_delete_hash->{$key};
2099
2100     if (%$pending_delete_hash) {
2101         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
2102     } else {
2103         delete $conf->{pending}->{delete};
2104     }
2105 }
2106
2107 sub vmconfig_register_unused_drive {
2108     my ($storecfg, $vmid, $conf, $drive) = @_;
2109
2110     if (drive_is_cloudinit($drive)) {
2111         eval { PVE::Storage::vdisk_free($storecfg, $drive->{file}) };
2112         warn $@ if $@;
2113     } elsif (!drive_is_cdrom($drive)) {
2114         my $volid = $drive->{file};
2115         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
2116             PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
2117         }
2118     }
2119 }
2120
2121 sub vmconfig_cleanup_pending {
2122     my ($conf) = @_;
2123
2124     # remove pending changes when nothing changed
2125     my $changes;
2126     foreach my $opt (keys %{$conf->{pending}}) {
2127         if (defined($conf->{$opt}) && ($conf->{pending}->{$opt} eq  $conf->{$opt})) {
2128             $changes = 1;
2129             delete $conf->{pending}->{$opt};
2130         }
2131     }
2132
2133     my $current_delete_hash = split_flagged_list($conf->{pending}->{delete});
2134     my $pending_delete_hash = {};
2135     while (my ($opt, $force) = each %$current_delete_hash) {
2136         if (defined($conf->{$opt})) {
2137             $pending_delete_hash->{$opt} = $force;
2138         } else {
2139             $changes = 1;
2140         }
2141     }
2142
2143     if (%$pending_delete_hash) {
2144         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
2145     } else {
2146         delete $conf->{pending}->{delete};
2147     }
2148
2149     return $changes;
2150 }
2151
2152 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str]
2153 my $smbios1_fmt = {
2154     uuid => {
2155         type => 'string',
2156         pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
2157         format_description => 'UUID',
2158         description => "Set SMBIOS1 UUID.",
2159         optional => 1,
2160     },
2161     version => {
2162         type => 'string',
2163         pattern => '\S+',
2164         format_description => 'string',
2165         description => "Set SMBIOS1 version.",
2166         optional => 1,
2167     },
2168     serial => {
2169         type => 'string',
2170         pattern => '\S+',
2171         format_description => 'string',
2172         description => "Set SMBIOS1 serial number.",
2173         optional => 1,
2174     },
2175     manufacturer => {
2176         type => 'string',
2177         pattern => '\S+',
2178         format_description => 'string',
2179         description => "Set SMBIOS1 manufacturer.",
2180         optional => 1,
2181     },
2182     product => {
2183         type => 'string',
2184         pattern => '\S+',
2185         format_description => 'string',
2186         description => "Set SMBIOS1 product ID.",
2187         optional => 1,
2188     },
2189     sku => {
2190         type => 'string',
2191         pattern => '\S+',
2192         format_description => 'string',
2193         description => "Set SMBIOS1 SKU string.",
2194         optional => 1,
2195     },
2196     family => {
2197         type => 'string',
2198         pattern => '\S+',
2199         format_description => 'string',
2200         description => "Set SMBIOS1 family string.",
2201         optional => 1,
2202     },
2203 };
2204
2205 sub parse_smbios1 {
2206     my ($data) = @_;
2207
2208     my $res = eval { PVE::JSONSchema::parse_property_string($smbios1_fmt, $data) };
2209     warn $@ if $@;
2210     return $res;
2211 }
2212
2213 sub print_smbios1 {
2214     my ($smbios1) = @_;
2215     return PVE::JSONSchema::print_property_string($smbios1, $smbios1_fmt);
2216 }
2217
2218 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_fmt);
2219
2220 PVE::JSONSchema::register_format('pve-qm-bootdisk', \&verify_bootdisk);
2221 sub verify_bootdisk {
2222     my ($value, $noerr) = @_;
2223
2224     return $value if is_valid_drivename($value);
2225
2226     return undef if $noerr;
2227
2228     die "invalid boot disk '$value'\n";
2229 }
2230
2231 sub parse_watchdog {
2232     my ($value) = @_;
2233
2234     return undef if !$value;
2235
2236     my $res = eval { PVE::JSONSchema::parse_property_string($watchdog_fmt, $value) };
2237     warn $@ if $@;
2238     return $res;
2239 }
2240
2241 sub parse_guest_agent {
2242     my ($value) = @_;
2243
2244     return {} if !defined($value->{agent});
2245
2246     my $res = eval { PVE::JSONSchema::parse_property_string($agent_fmt, $value->{agent}) };
2247     warn $@ if $@;
2248
2249     # if the agent is disabled ignore the other potentially set properties
2250     return {} if !$res->{enabled};
2251     return $res;
2252 }
2253
2254 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
2255 sub verify_usb_device {
2256     my ($value, $noerr) = @_;
2257
2258     return $value if parse_usb_device($value);
2259
2260     return undef if $noerr;
2261
2262     die "unable to parse usb device\n";
2263 }
2264
2265 # add JSON properties for create and set function
2266 sub json_config_properties {
2267     my $prop = shift;
2268
2269     foreach my $opt (keys %$confdesc) {
2270         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate' || $opt eq 'runningmachine';
2271         $prop->{$opt} = $confdesc->{$opt};
2272     }
2273
2274     return $prop;
2275 }
2276
2277 # return copy of $confdesc_cloudinit to generate documentation
2278 sub cloudinit_config_properties {
2279
2280     return dclone($confdesc_cloudinit);
2281 }
2282
2283 sub check_type {
2284     my ($key, $value) = @_;
2285
2286     die "unknown setting '$key'\n" if !$confdesc->{$key};
2287
2288     my $type = $confdesc->{$key}->{type};
2289
2290     if (!defined($value)) {
2291         die "got undefined value\n";
2292     }
2293
2294     if ($value =~ m/[\n\r]/) {
2295         die "property contains a line feed\n";
2296     }
2297
2298     if ($type eq 'boolean') {
2299         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
2300         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
2301         die "type check ('boolean') failed - got '$value'\n";
2302     } elsif ($type eq 'integer') {
2303         return int($1) if $value =~ m/^(\d+)$/;
2304         die "type check ('integer') failed - got '$value'\n";
2305     } elsif ($type eq 'number') {
2306         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
2307         die "type check ('number') failed - got '$value'\n";
2308     } elsif ($type eq 'string') {
2309         if (my $fmt = $confdesc->{$key}->{format}) {
2310             PVE::JSONSchema::check_format($fmt, $value);
2311             return $value;
2312         }
2313         $value =~ s/^\"(.*)\"$/$1/;
2314         return $value;
2315     } else {
2316         die "internal error"
2317     }
2318 }
2319
2320 sub check_iommu_support{
2321     #fixme : need to check IOMMU support
2322     #http://www.linux-kvm.org/page/How_to_assign_devices_with_VT-d_in_KVM
2323
2324     my $iommu=1;
2325     return $iommu;
2326
2327 }
2328
2329 sub touch_config {
2330     my ($vmid) = @_;
2331
2332     my $conf = PVE::QemuConfig->config_file($vmid);
2333     utime undef, undef, $conf;
2334 }
2335
2336 sub destroy_vm {
2337     my ($storecfg, $vmid, $keep_empty_config, $skiplock) = @_;
2338
2339     my $conffile = PVE::QemuConfig->config_file($vmid);
2340
2341     my $conf = PVE::QemuConfig->load_config($vmid);
2342
2343     PVE::QemuConfig->check_lock($conf) if !$skiplock;
2344
2345     if ($conf->{template}) {
2346         # check if any base image is still used by a linked clone
2347         foreach_drive($conf, sub {
2348                 my ($ds, $drive) = @_;
2349
2350                 return if drive_is_cdrom($drive);
2351
2352                 my $volid = $drive->{file};
2353
2354                 return if !$volid || $volid =~ m|^/|;
2355
2356                 die "base volume '$volid' is still in use by linked cloned\n"
2357                     if PVE::Storage::volume_is_base_and_used($storecfg, $volid);
2358
2359         });
2360     }
2361
2362     # only remove disks owned by this VM
2363     foreach_drive($conf, sub {
2364         my ($ds, $drive) = @_;
2365
2366         return if drive_is_cdrom($drive, 1);
2367
2368         my $volid = $drive->{file};
2369
2370         return if !$volid || $volid =~ m|^/|;
2371
2372         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
2373         return if !$path || !$owner || ($owner != $vmid);
2374
2375         eval {
2376             PVE::Storage::vdisk_free($storecfg, $volid);
2377         };
2378         warn "Could not remove disk '$volid', check manually: $@" if $@;
2379
2380     });
2381
2382     if ($keep_empty_config) {
2383         PVE::Tools::file_set_contents($conffile, "memory: 128\n");
2384     } else {
2385         unlink $conffile;
2386     }
2387
2388     # also remove unused disk
2389     eval {
2390         my $dl = PVE::Storage::vdisk_list($storecfg, undef, $vmid);
2391
2392         eval {
2393             PVE::Storage::foreach_volid($dl, sub {
2394                 my ($volid, $sid, $volname, $d) = @_;
2395                 PVE::Storage::vdisk_free($storecfg, $volid);
2396             });
2397         };
2398         warn $@ if $@;
2399
2400     };
2401     warn $@ if $@;
2402 }
2403
2404 sub parse_vm_config {
2405     my ($filename, $raw) = @_;
2406
2407     return undef if !defined($raw);
2408
2409     my $res = {
2410         digest => Digest::SHA::sha1_hex($raw),
2411         snapshots => {},
2412         pending => {},
2413     };
2414
2415     $filename =~ m|/qemu-server/(\d+)\.conf$|
2416         || die "got strange filename '$filename'";
2417
2418     my $vmid = $1;
2419
2420     my $conf = $res;
2421     my $descr;
2422     my $section = '';
2423
2424     my @lines = split(/\n/, $raw);
2425     foreach my $line (@lines) {
2426         next if $line =~ m/^\s*$/;
2427
2428         if ($line =~ m/^\[PENDING\]\s*$/i) {
2429             $section = 'pending';
2430             if (defined($descr)) {
2431                 $descr =~ s/\s+$//;
2432                 $conf->{description} = $descr;
2433             }
2434             $descr = undef;
2435             $conf = $res->{$section} = {};
2436             next;
2437
2438         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2439             $section = $1;
2440             if (defined($descr)) {
2441                 $descr =~ s/\s+$//;
2442                 $conf->{description} = $descr;
2443             }
2444             $descr = undef;
2445             $conf = $res->{snapshots}->{$section} = {};
2446             next;
2447         }
2448
2449         if ($line =~ m/^\#(.*)\s*$/) {
2450             $descr = '' if !defined($descr);
2451             $descr .= PVE::Tools::decode_text($1) . "\n";
2452             next;
2453         }
2454
2455         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2456             $descr = '' if !defined($descr);
2457             $descr .= PVE::Tools::decode_text($2);
2458         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2459             $conf->{snapstate} = $1;
2460         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2461             my $key = $1;
2462             my $value = $2;
2463             $conf->{$key} = $value;
2464         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2465             my $value = $1;
2466             if ($section eq 'pending') {
2467                 $conf->{delete} = $value; # we parse this later
2468             } else {
2469                 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
2470             }
2471         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(.+?)\s*$/) {
2472             my $key = $1;
2473             my $value = $2;
2474             eval { $value = check_type($key, $value); };
2475             if ($@) {
2476                 warn "vm $vmid - unable to parse value of '$key' - $@";
2477             } else {
2478                 $key = 'ide2' if $key eq 'cdrom';
2479                 my $fmt = $confdesc->{$key}->{format};
2480                 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2481                     my $v = parse_drive($key, $value);
2482                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2483                         $v->{file} = $volid;
2484                         $value = print_drive($vmid, $v);
2485                     } else {
2486                         warn "vm $vmid - unable to parse value of '$key'\n";
2487                         next;
2488                     }
2489                 }
2490
2491                 $conf->{$key} = $value;
2492             }
2493         }
2494     }
2495
2496     if (defined($descr)) {
2497         $descr =~ s/\s+$//;
2498         $conf->{description} = $descr;
2499     }
2500     delete $res->{snapstate}; # just to be sure
2501
2502     return $res;
2503 }
2504
2505 sub write_vm_config {
2506     my ($filename, $conf) = @_;
2507
2508     delete $conf->{snapstate}; # just to be sure
2509
2510     if ($conf->{cdrom}) {
2511         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2512         $conf->{ide2} = $conf->{cdrom};
2513         delete $conf->{cdrom};
2514     }
2515
2516     # we do not use 'smp' any longer
2517     if ($conf->{sockets}) {
2518         delete $conf->{smp};
2519     } elsif ($conf->{smp}) {
2520         $conf->{sockets} = $conf->{smp};
2521         delete $conf->{cores};
2522         delete $conf->{smp};
2523     }
2524
2525     my $used_volids = {};
2526
2527     my $cleanup_config = sub {
2528         my ($cref, $pending, $snapname) = @_;
2529
2530         foreach my $key (keys %$cref) {
2531             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2532                 $key eq 'snapstate' || $key eq 'pending';
2533             my $value = $cref->{$key};
2534             if ($key eq 'delete') {
2535                 die "propertry 'delete' is only allowed in [PENDING]\n"
2536                     if !$pending;
2537                 # fixme: check syntax?
2538                 next;
2539             }
2540             eval { $value = check_type($key, $value); };
2541             die "unable to parse value of '$key' - $@" if $@;
2542
2543             $cref->{$key} = $value;
2544
2545             if (!$snapname && is_valid_drivename($key)) {
2546                 my $drive = parse_drive($key, $value);
2547                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2548             }
2549         }
2550     };
2551
2552     &$cleanup_config($conf);
2553
2554     &$cleanup_config($conf->{pending}, 1);
2555
2556     foreach my $snapname (keys %{$conf->{snapshots}}) {
2557         die "internal error" if $snapname eq 'pending';
2558         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2559     }
2560
2561     # remove 'unusedX' settings if we re-add a volume
2562     foreach my $key (keys %$conf) {
2563         my $value = $conf->{$key};
2564         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2565             delete $conf->{$key};
2566         }
2567     }
2568
2569     my $generate_raw_config = sub {
2570         my ($conf, $pending) = @_;
2571
2572         my $raw = '';
2573
2574         # add description as comment to top of file
2575         if (defined(my $descr = $conf->{description})) {
2576             if ($descr) {
2577                 foreach my $cl (split(/\n/, $descr)) {
2578                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2579                 }
2580             } else {
2581                 $raw .= "#\n" if $pending;
2582             }
2583         }
2584
2585         foreach my $key (sort keys %$conf) {
2586             next if $key eq 'digest' || $key eq 'description' || $key eq 'pending' || $key eq 'snapshots';
2587             $raw .= "$key: $conf->{$key}\n";
2588         }
2589         return $raw;
2590     };
2591
2592     my $raw = &$generate_raw_config($conf);
2593
2594     if (scalar(keys %{$conf->{pending}})){
2595         $raw .= "\n[PENDING]\n";
2596         $raw .= &$generate_raw_config($conf->{pending}, 1);
2597     }
2598
2599     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2600         $raw .= "\n[$snapname]\n";
2601         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2602     }
2603
2604     return $raw;
2605 }
2606
2607 sub load_defaults {
2608
2609     my $res = {};
2610
2611     # we use static defaults from our JSON schema configuration
2612     foreach my $key (keys %$confdesc) {
2613         if (defined(my $default = $confdesc->{$key}->{default})) {
2614             $res->{$key} = $default;
2615         }
2616     }
2617
2618     return $res;
2619 }
2620
2621 sub config_list {
2622     my $vmlist = PVE::Cluster::get_vmlist();
2623     my $res = {};
2624     return $res if !$vmlist || !$vmlist->{ids};
2625     my $ids = $vmlist->{ids};
2626
2627     foreach my $vmid (keys %$ids) {
2628         my $d = $ids->{$vmid};
2629         next if !$d->{node} || $d->{node} ne $nodename;
2630         next if !$d->{type} || $d->{type} ne 'qemu';
2631         $res->{$vmid}->{exists} = 1;
2632     }
2633     return $res;
2634 }
2635
2636 # test if VM uses local resources (to prevent migration)
2637 sub check_local_resources {
2638     my ($conf, $noerr) = @_;
2639
2640     my $loc_res = 0;
2641
2642     $loc_res = 1 if $conf->{hostusb}; # old syntax
2643     $loc_res = 1 if $conf->{hostpci}; # old syntax
2644
2645     foreach my $k (keys %$conf) {
2646         next if $k =~ m/^usb/ && ($conf->{$k} eq 'spice');
2647         # sockets are safe: they will recreated be on the target side post-migrate
2648         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2649         $loc_res = 1 if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2650     }
2651
2652     die "VM uses local resources\n" if $loc_res && !$noerr;
2653
2654     return $loc_res;
2655 }
2656
2657 # check if used storages are available on all nodes (use by migrate)
2658 sub check_storage_availability {
2659     my ($storecfg, $conf, $node) = @_;
2660
2661     foreach_drive($conf, sub {
2662         my ($ds, $drive) = @_;
2663
2664         my $volid = $drive->{file};
2665         return if !$volid;
2666
2667         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2668         return if !$sid;
2669
2670         # check if storage is available on both nodes
2671         my $scfg = PVE::Storage::storage_check_node($storecfg, $sid);
2672         PVE::Storage::storage_check_node($storecfg, $sid, $node);
2673    });
2674 }
2675
2676 # list nodes where all VM images are available (used by has_feature API)
2677 sub shared_nodes {
2678     my ($conf, $storecfg) = @_;
2679
2680     my $nodelist = PVE::Cluster::get_nodelist();
2681     my $nodehash = { map { $_ => 1 } @$nodelist };
2682     my $nodename = PVE::INotify::nodename();
2683
2684     foreach_drive($conf, sub {
2685         my ($ds, $drive) = @_;
2686
2687         my $volid = $drive->{file};
2688         return if !$volid;
2689
2690         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2691         if ($storeid) {
2692             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2693             if ($scfg->{disable}) {
2694                 $nodehash = {};
2695             } elsif (my $avail = $scfg->{nodes}) {
2696                 foreach my $node (keys %$nodehash) {
2697                     delete $nodehash->{$node} if !$avail->{$node};
2698                 }
2699             } elsif (!$scfg->{shared}) {
2700                 foreach my $node (keys %$nodehash) {
2701                     delete $nodehash->{$node} if $node ne $nodename
2702                 }
2703             }
2704         }
2705     });
2706
2707     return $nodehash
2708 }
2709
2710 sub check_cmdline {
2711     my ($pidfile, $pid) = @_;
2712
2713     my $fh = IO::File->new("/proc/$pid/cmdline", "r");
2714     if (defined($fh)) {
2715         my $line = <$fh>;
2716         $fh->close;
2717         return undef if !$line;
2718         my @param = split(/\0/, $line);
2719
2720         my $cmd = $param[0];
2721         return if !$cmd || ($cmd !~ m|kvm$| && $cmd !~ m|qemu-system-x86_64$|);
2722
2723         for (my $i = 0; $i < scalar (@param); $i++) {
2724             my $p = $param[$i];
2725             next if !$p;
2726             if (($p eq '-pidfile') || ($p eq '--pidfile')) {
2727                 my $p = $param[$i+1];
2728                 return 1 if $p && ($p eq $pidfile);
2729                 return undef;
2730             }
2731         }
2732     }
2733     return undef;
2734 }
2735
2736 sub check_running {
2737     my ($vmid, $nocheck, $node) = @_;
2738
2739     my $filename = PVE::QemuConfig->config_file($vmid, $node);
2740
2741     die "unable to find configuration file for VM $vmid - no such machine\n"
2742         if !$nocheck && ! -f $filename;
2743
2744     my $pidfile = pidfile_name($vmid);
2745
2746     if (my $fd = IO::File->new("<$pidfile")) {
2747         my $st = stat($fd);
2748         my $line = <$fd>;
2749         close($fd);
2750
2751         my $mtime = $st->mtime;
2752         if ($mtime > time()) {
2753             warn "file '$filename' modified in future\n";
2754         }
2755
2756         if ($line =~ m/^(\d+)$/) {
2757             my $pid = $1;
2758             if (check_cmdline($pidfile, $pid)) {
2759                 if (my $pinfo = PVE::ProcFSTools::check_process_running($pid)) {
2760                     return $pid;
2761                 }
2762             }
2763         }
2764     }
2765
2766     return undef;
2767 }
2768
2769 sub vzlist {
2770
2771     my $vzlist = config_list();
2772
2773     my $fd = IO::Dir->new($var_run_tmpdir) || return $vzlist;
2774
2775     while (defined(my $de = $fd->read)) {
2776         next if $de !~ m/^(\d+)\.pid$/;
2777         my $vmid = $1;
2778         next if !defined($vzlist->{$vmid});
2779         if (my $pid = check_running($vmid)) {
2780             $vzlist->{$vmid}->{pid} = $pid;
2781         }
2782     }
2783
2784     return $vzlist;
2785 }
2786
2787 sub disksize {
2788     my ($storecfg, $conf) = @_;
2789
2790     my $bootdisk = $conf->{bootdisk};
2791     return undef if !$bootdisk;
2792     return undef if !is_valid_drivename($bootdisk);
2793
2794     return undef if !$conf->{$bootdisk};
2795
2796     my $drive = parse_drive($bootdisk, $conf->{$bootdisk});
2797     return undef if !defined($drive);
2798
2799     return undef if drive_is_cdrom($drive);
2800
2801     my $volid = $drive->{file};
2802     return undef if !$volid;
2803
2804     return $drive->{size};
2805 }
2806
2807 our $vmstatus_return_properties = {
2808     vmid => get_standard_option('pve-vmid'),
2809     status => {
2810         description => "Qemu process status.",
2811         type => 'string',
2812         enum => ['stopped', 'running'],
2813     },
2814     maxmem => {
2815         description => "Maximum memory in bytes.",
2816         type => 'integer',
2817         optional => 1,
2818         renderer => 'bytes',
2819     },
2820     maxdisk => {
2821         description => "Root disk size in bytes.",
2822         type => 'integer',
2823         optional => 1,
2824         renderer => 'bytes',
2825     },
2826     name => {
2827         description => "VM name.",
2828         type => 'string',
2829         optional => 1,
2830     },
2831     qmpstatus => {
2832         description => "Qemu QMP agent status.",
2833         type => 'string',
2834         optional => 1,
2835     },
2836     pid => {
2837         description => "PID of running qemu process.",
2838         type => 'integer',
2839         optional => 1,
2840     },
2841     uptime => {
2842         description => "Uptime.",
2843         type => 'integer',
2844         optional => 1,
2845         renderer => 'duration',
2846     },
2847     cpus => {
2848         description => "Maximum usable CPUs.",
2849         type => 'number',
2850         optional => 1,
2851     },
2852 };
2853
2854 my $last_proc_pid_stat;
2855
2856 # get VM status information
2857 # This must be fast and should not block ($full == false)
2858 # We only query KVM using QMP if $full == true (this can be slow)
2859 sub vmstatus {
2860     my ($opt_vmid, $full) = @_;
2861
2862     my $res = {};
2863
2864     my $storecfg = PVE::Storage::config();
2865
2866     my $list = vzlist();
2867     my $defaults = load_defaults();
2868
2869     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
2870
2871     my $cpucount = $cpuinfo->{cpus} || 1;
2872
2873     foreach my $vmid (keys %$list) {
2874         next if $opt_vmid && ($vmid ne $opt_vmid);
2875
2876         my $cfspath = PVE::QemuConfig->cfs_config_path($vmid);
2877         my $conf = PVE::Cluster::cfs_read_file($cfspath) || {};
2878
2879         my $d = { vmid => $vmid };
2880         $d->{pid} = $list->{$vmid}->{pid};
2881
2882         # fixme: better status?
2883         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
2884
2885         my $size = disksize($storecfg, $conf);
2886         if (defined($size)) {
2887             $d->{disk} = 0; # no info available
2888             $d->{maxdisk} = $size;
2889         } else {
2890             $d->{disk} = 0;
2891             $d->{maxdisk} = 0;
2892         }
2893
2894         $d->{cpus} = ($conf->{sockets} || $defaults->{sockets})
2895             * ($conf->{cores} || $defaults->{cores});
2896         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
2897         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
2898
2899         $d->{name} = $conf->{name} || "VM $vmid";
2900         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024)
2901             : $defaults->{memory}*(1024*1024);
2902
2903         if ($conf->{balloon}) {
2904             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
2905             $d->{shares} = defined($conf->{shares}) ? $conf->{shares}
2906                 : $defaults->{shares};
2907         }
2908
2909         $d->{uptime} = 0;
2910         $d->{cpu} = 0;
2911         $d->{mem} = 0;
2912
2913         $d->{netout} = 0;
2914         $d->{netin} = 0;
2915
2916         $d->{diskread} = 0;
2917         $d->{diskwrite} = 0;
2918
2919         $d->{template} = PVE::QemuConfig->is_template($conf);
2920
2921         $d->{serial} = 1 if conf_has_serial($conf);
2922
2923         $res->{$vmid} = $d;
2924     }
2925
2926     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
2927     foreach my $dev (keys %$netdev) {
2928         next if $dev !~ m/^tap([1-9]\d*)i/;
2929         my $vmid = $1;
2930         my $d = $res->{$vmid};
2931         next if !$d;
2932
2933         $d->{netout} += $netdev->{$dev}->{receive};
2934         $d->{netin} += $netdev->{$dev}->{transmit};
2935
2936         if ($full) {
2937             $d->{nics}->{$dev}->{netout} = $netdev->{$dev}->{receive};
2938             $d->{nics}->{$dev}->{netin} = $netdev->{$dev}->{transmit};
2939         }
2940
2941     }
2942
2943     my $ctime = gettimeofday;
2944
2945     foreach my $vmid (keys %$list) {
2946
2947         my $d = $res->{$vmid};
2948         my $pid = $d->{pid};
2949         next if !$pid;
2950
2951         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
2952         next if !$pstat; # not running
2953
2954         my $used = $pstat->{utime} + $pstat->{stime};
2955
2956         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
2957
2958         if ($pstat->{vsize}) {
2959             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
2960         }
2961
2962         my $old = $last_proc_pid_stat->{$pid};
2963         if (!$old) {
2964             $last_proc_pid_stat->{$pid} = {
2965                 time => $ctime,
2966                 used => $used,
2967                 cpu => 0,
2968             };
2969             next;
2970         }
2971
2972         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
2973
2974         if ($dtime > 1000) {
2975             my $dutime = $used -  $old->{used};
2976
2977             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
2978             $last_proc_pid_stat->{$pid} = {
2979                 time => $ctime,
2980                 used => $used,
2981                 cpu => $d->{cpu},
2982             };
2983         } else {
2984             $d->{cpu} = $old->{cpu};
2985         }
2986     }
2987
2988     return $res if !$full;
2989
2990     my $qmpclient = PVE::QMPClient->new();
2991
2992     my $ballooncb = sub {
2993         my ($vmid, $resp) = @_;
2994
2995         my $info = $resp->{'return'};
2996         return if !$info->{max_mem};
2997
2998         my $d = $res->{$vmid};
2999
3000         # use memory assigned to VM
3001         $d->{maxmem} = $info->{max_mem};
3002         $d->{balloon} = $info->{actual};
3003
3004         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
3005             $d->{mem} = $info->{total_mem} - $info->{free_mem};
3006             $d->{freemem} = $info->{free_mem};
3007         }
3008
3009         $d->{ballooninfo} = $info;
3010     };
3011
3012     my $blockstatscb = sub {
3013         my ($vmid, $resp) = @_;
3014         my $data = $resp->{'return'} || [];
3015         my $totalrdbytes = 0;
3016         my $totalwrbytes = 0;
3017
3018         for my $blockstat (@$data) {
3019             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
3020             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
3021
3022             $blockstat->{device} =~ s/drive-//;
3023             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
3024         }
3025         $res->{$vmid}->{diskread} = $totalrdbytes;
3026         $res->{$vmid}->{diskwrite} = $totalwrbytes;
3027     };
3028
3029     my $statuscb = sub {
3030         my ($vmid, $resp) = @_;
3031
3032         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
3033         # this fails if ballon driver is not loaded, so this must be
3034         # the last commnand (following command are aborted if this fails).
3035         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
3036
3037         my $status = 'unknown';
3038         if (!defined($status = $resp->{'return'}->{status})) {
3039             warn "unable to get VM status\n";
3040             return;
3041         }
3042
3043         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
3044     };
3045
3046     foreach my $vmid (keys %$list) {
3047         next if $opt_vmid && ($vmid ne $opt_vmid);
3048         next if !$res->{$vmid}->{pid}; # not running
3049         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
3050     }
3051
3052     $qmpclient->queue_execute(undef, 2);
3053
3054     foreach my $vmid (keys %$list) {
3055         next if $opt_vmid && ($vmid ne $opt_vmid);
3056         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
3057     }
3058
3059     return $res;
3060 }
3061
3062 sub foreach_drive {
3063     my ($conf, $func, @param) = @_;
3064
3065     foreach my $ds (valid_drive_names()) {
3066         next if !defined($conf->{$ds});
3067
3068         my $drive = parse_drive($ds, $conf->{$ds});
3069         next if !$drive;
3070
3071         &$func($ds, $drive, @param);
3072     }
3073 }
3074
3075 sub foreach_volid {
3076     my ($conf, $func, @param) = @_;
3077
3078     my $volhash = {};
3079
3080     my $test_volid = sub {
3081         my ($volid, $is_cdrom, $replicate, $shared, $snapname) = @_;
3082
3083         return if !$volid;
3084
3085         $volhash->{$volid}->{cdrom} //= 1;
3086         $volhash->{$volid}->{cdrom} = 0 if !$is_cdrom;
3087
3088         $volhash->{$volid}->{replicate} //= 0;
3089         $volhash->{$volid}->{replicate} = 1 if $replicate;
3090
3091         $volhash->{$volid}->{shared} //= 0;
3092         $volhash->{$volid}->{shared} = 1 if $shared;
3093
3094         $volhash->{$volid}->{referenced_in_config} //= 0;
3095         $volhash->{$volid}->{referenced_in_config} = 1 if !defined($snapname);
3096
3097         $volhash->{$volid}->{referenced_in_snapshot}->{$snapname} = 1
3098             if defined($snapname);
3099     };
3100
3101     foreach_drive($conf, sub {
3102         my ($ds, $drive) = @_;
3103         $test_volid->($drive->{file}, drive_is_cdrom($drive), $drive->{replicate} // 1, $drive->{shared}, undef);
3104     });
3105
3106     foreach my $snapname (keys %{$conf->{snapshots}}) {
3107         my $snap = $conf->{snapshots}->{$snapname};
3108         $test_volid->($snap->{vmstate}, 0, 1, $snapname);
3109         foreach_drive($snap, sub {
3110             my ($ds, $drive) = @_;
3111             $test_volid->($drive->{file}, drive_is_cdrom($drive), $drive->{replicate} // 1, $drive->{shared}, $snapname);
3112         });
3113     }
3114
3115     foreach my $volid (keys %$volhash) {
3116         &$func($volid, $volhash->{$volid}, @param);
3117     }
3118 }
3119
3120 sub conf_has_serial {
3121     my ($conf) = @_;
3122
3123     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3124         if ($conf->{"serial$i"}) {
3125             return 1;
3126         }
3127     }
3128
3129     return 0;
3130 }
3131
3132 sub vga_conf_has_spice {
3133     my ($vga) = @_;
3134
3135     return 0 if !$vga || $vga !~ m/^qxl([234])?$/;
3136
3137     return $1 || 1;
3138 }
3139
3140 sub config_to_command {
3141     my ($storecfg, $vmid, $conf, $defaults, $forcemachine) = @_;
3142
3143     my $cmd = [];
3144     my $globalFlags = [];
3145     my $machineFlags = [];
3146     my $rtcFlags = [];
3147     my $cpuFlags = [];
3148     my $devices = [];
3149     my $pciaddr = '';
3150     my $bridges = {};
3151     my $kvmver = kvm_user_version();
3152     my $vernum = 0; # unknown
3153     my $ostype = $conf->{ostype};
3154     my $winversion = windows_version($ostype);
3155     my $kvm = $conf->{kvm} // 1;
3156
3157     die "KVM virtualisation configured, but not available. Either disable in VM configuration or enable in BIOS.\n" if (!$cpuinfo->{hvm} && $kvm);
3158
3159     if ($kvmver =~ m/^(\d+)\.(\d+)$/) {
3160         $vernum = $1*1000000+$2*1000;
3161     } elsif ($kvmver =~ m/^(\d+)\.(\d+)\.(\d+)$/) {
3162         $vernum = $1*1000000+$2*1000+$3;
3163     }
3164
3165     die "detected old qemu-kvm binary ($kvmver)\n" if $vernum < 15000;
3166
3167     my $have_ovz = -f '/proc/vz/vestat';
3168
3169     my $q35 = machine_type_is_q35($conf);
3170     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
3171     my $machine_type = $forcemachine || $conf->{machine};
3172     my $use_old_bios_files = undef;
3173     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3174
3175     my $cpuunits = defined($conf->{cpuunits}) ?
3176             $conf->{cpuunits} : $defaults->{cpuunits};
3177
3178     push @$cmd, '/usr/bin/kvm';
3179
3180     push @$cmd, '-id', $vmid;
3181
3182     my $vmname = $conf->{name} || "vm$vmid";
3183
3184     push @$cmd, '-name', $vmname;
3185
3186     my $use_virtio = 0;
3187
3188     my $qmpsocket = qmp_socket($vmid);
3189     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server,nowait";
3190     push @$cmd, '-mon', "chardev=qmp,mode=control";
3191
3192
3193     push @$cmd, '-pidfile' , pidfile_name($vmid);
3194
3195     push @$cmd, '-daemonize';
3196
3197     if ($conf->{smbios1}) {
3198         push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
3199     }
3200
3201     if ($conf->{vmgenid}) {
3202         push @$devices, '-device', 'vmgenid,guid='.$conf->{vmgenid};
3203     }
3204
3205     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3206         die "uefi base image not found\n" if ! -f $OVMF_CODE;
3207
3208         my $path;
3209         my $format;
3210         if (my $efidisk = $conf->{efidisk0}) {
3211             my $d = PVE::JSONSchema::parse_property_string($efidisk_fmt, $efidisk);
3212             my ($storeid, $volname) = PVE::Storage::parse_volume_id($d->{file}, 1);
3213             $format = $d->{format};
3214             if ($storeid) {
3215                 $path = PVE::Storage::path($storecfg, $d->{file});
3216                 if (!defined($format)) {
3217                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
3218                     $format = qemu_img_format($scfg, $volname);
3219                 }
3220             } else {
3221                 $path = $d->{file};
3222                 die "efidisk format must be specified\n"
3223                     if !defined($format);
3224             }
3225         } else {
3226             warn "no efidisk configured! Using temporary efivars disk.\n";
3227             $path = "/tmp/$vmid-ovmf.fd";
3228             PVE::Tools::file_copy($OVMF_VARS, $path, -s $OVMF_VARS);
3229             $format = 'raw';
3230         }
3231
3232         push @$cmd, '-drive', "if=pflash,unit=0,format=raw,readonly,file=$OVMF_CODE";
3233         push @$cmd, '-drive', "if=pflash,unit=1,format=$format,id=drive-efidisk0,file=$path";
3234     }
3235
3236
3237     # add usb controllers
3238     my @usbcontrollers = PVE::QemuServer::USB::get_usb_controllers($conf, $bridges, $q35, $usbdesc->{format}, $MAX_USB_DEVICES);
3239     push @$devices, @usbcontrollers if @usbcontrollers;
3240     my $vga = $conf->{vga};
3241
3242     my $qxlnum = vga_conf_has_spice($vga);
3243     $vga = 'qxl' if $qxlnum;
3244
3245     if (!$vga) {
3246         if (qemu_machine_feature_enabled($machine_type, $kvmver, 2, 9)) {
3247             $vga = (!$winversion || $winversion >= 6) ? 'std' : 'cirrus';
3248         } else {
3249             $vga = ($winversion >= 6) ? 'std' : 'cirrus';
3250         }
3251     }
3252
3253     # enable absolute mouse coordinates (needed by vnc)
3254     my $tablet;
3255     if (defined($conf->{tablet})) {
3256         $tablet = $conf->{tablet};
3257     } else {
3258         $tablet = $defaults->{tablet};
3259         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
3260         $tablet = 0 if $vga =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
3261     }
3262
3263     push @$devices, '-device', print_tabletdevice_full($conf) if $tablet;
3264
3265     my $kvm_off = 0;
3266     my $gpu_passthrough;
3267
3268     # host pci devices
3269     for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
3270         my $d = parse_hostpci($conf->{"hostpci$i"});
3271         next if !$d;
3272
3273         my $pcie = $d->{pcie};
3274         if($pcie){
3275             die "q35 machine model is not enabled" if !$q35;
3276             $pciaddr = print_pcie_addr("hostpci$i");
3277         }else{
3278             $pciaddr = print_pci_addr("hostpci$i", $bridges);
3279         }
3280
3281         my $rombar = defined($d->{rombar}) && !$d->{rombar} ? ',rombar=0' : '';
3282         my $romfile = $d->{romfile};
3283
3284         my $xvga = '';
3285         if ($d->{'x-vga'}) {
3286             $xvga = ',x-vga=on';
3287             $kvm_off = 1;
3288             $vga = 'none';
3289             $gpu_passthrough = 1;
3290
3291             if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3292                 $xvga = "";
3293             }
3294         }
3295         my $pcidevices = $d->{pciid};
3296         my $multifunction = 1 if @$pcidevices > 1;
3297
3298         my $j=0;
3299         foreach my $pcidevice (@$pcidevices) {
3300
3301             my $id = "hostpci$i";
3302             $id .= ".$j" if $multifunction;
3303             my $addr = $pciaddr;
3304             $addr .= ".$j" if $multifunction;
3305             my $devicestr = "vfio-pci,host=$pcidevice->{id}.$pcidevice->{function},id=$id$addr";
3306
3307             if($j == 0){
3308                 $devicestr .= "$rombar$xvga";
3309                 $devicestr .= ",multifunction=on" if $multifunction;
3310                 $devicestr .= ",romfile=/usr/share/kvm/$romfile" if $romfile;
3311             }
3312
3313             push @$devices, '-device', $devicestr;
3314             $j++;
3315         }
3316     }
3317
3318     # usb devices
3319     my @usbdevices = PVE::QemuServer::USB::get_usb_devices($conf, $usbdesc->{format}, $MAX_USB_DEVICES);
3320     push @$devices, @usbdevices if @usbdevices;
3321     # serial devices
3322     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3323         if (my $path = $conf->{"serial$i"}) {
3324             if ($path eq 'socket') {
3325                 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
3326                 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server,nowait";
3327                 push @$devices, '-device', "isa-serial,chardev=serial$i";
3328             } else {
3329                 die "no such serial device\n" if ! -c $path;
3330                 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
3331                 push @$devices, '-device', "isa-serial,chardev=serial$i";
3332             }
3333         }
3334     }
3335
3336     # parallel devices
3337     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
3338         if (my $path = $conf->{"parallel$i"}) {
3339             die "no such parallel device\n" if ! -c $path;
3340             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
3341             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
3342             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
3343         }
3344     }
3345
3346
3347     my $sockets = 1;
3348     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3349     $sockets = $conf->{sockets} if  $conf->{sockets};
3350
3351     my $cores = $conf->{cores} || 1;
3352
3353     my $maxcpus = $sockets * $cores;
3354
3355     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
3356
3357     my $allowed_vcpus = $cpuinfo->{cpus};
3358
3359     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n"
3360         if ($allowed_vcpus < $maxcpus);
3361
3362     if($hotplug_features->{cpu} && qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 7)) {
3363
3364         push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3365         for (my $i = 2; $i <= $vcpus; $i++)  {
3366             my $cpustr = print_cpu_device($conf,$i);
3367             push @$cmd, '-device', $cpustr;
3368         }
3369
3370     } else {
3371
3372         push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3373     }
3374     push @$cmd, '-nodefaults';
3375
3376     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
3377
3378     my $bootindex_hash = {};
3379     my $i = 1;
3380     foreach my $o (split(//, $bootorder)) {
3381         $bootindex_hash->{$o} = $i*100;
3382         $i++;
3383     }
3384
3385     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
3386
3387     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
3388
3389     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
3390
3391     push @$cmd, '-vga', $vga if $vga && $vga !~ m/^serial\d+$/; # for kvm 77 and later
3392
3393     if ($vga && $vga !~ m/^serial\d+$/ && $vga ne 'none'){
3394         my $socket = vnc_socket($vmid);
3395         push @$cmd,  '-vnc', "unix:$socket,x509,password";
3396     } else {
3397         push @$cmd, '-nographic';
3398     }
3399
3400     # time drift fix
3401     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
3402
3403     my $useLocaltime = $conf->{localtime};
3404
3405     if ($winversion >= 5) { # windows
3406         $useLocaltime = 1 if !defined($conf->{localtime});
3407
3408         # use time drift fix when acpi is enabled
3409         if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
3410             $tdf = 1 if !defined($conf->{tdf});
3411         }
3412     }
3413
3414     if ($winversion >= 6) {
3415         push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3416         push @$cmd, '-no-hpet';
3417     }
3418
3419     push @$rtcFlags, 'driftfix=slew' if $tdf;
3420
3421     if (!$kvm) {
3422         push @$machineFlags, 'accel=tcg';
3423     }
3424
3425     if ($machine_type) {
3426         push @$machineFlags, "type=${machine_type}";
3427     }
3428
3429     if ($conf->{startdate}) {
3430         push @$rtcFlags, "base=$conf->{startdate}";
3431     } elsif ($useLocaltime) {
3432         push @$rtcFlags, 'base=localtime';
3433     }
3434
3435     my $cpu = $kvm ? "kvm64" : "qemu64";
3436     if (my $cputype = $conf->{cpu}) {
3437         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
3438             or die "Cannot parse cpu description: $cputype\n";
3439         $cpu = $cpuconf->{cputype};
3440         $kvm_off = 1 if $cpuconf->{hidden};
3441
3442         if (defined(my $flags = $cpuconf->{flags})) {
3443             push @$cpuFlags, split(";", $flags);
3444         }
3445     }
3446
3447     push @$cpuFlags , '+lahf_lm' if $cpu eq 'kvm64';
3448
3449     push @$cpuFlags , '-x2apic'
3450         if $conf->{ostype} && $conf->{ostype} eq 'solaris';
3451
3452     push @$cpuFlags, '+sep' if $cpu eq 'kvm64' || $cpu eq 'kvm32';
3453
3454     push @$cpuFlags, '-rdtscp' if $cpu =~ m/^Opteron/;
3455
3456     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3457
3458         push @$cpuFlags , '+kvm_pv_unhalt' if $kvm;
3459         push @$cpuFlags , '+kvm_pv_eoi' if $kvm;
3460     }
3461
3462     add_hyperv_enlightenments($cpuFlags, $winversion, $machine_type, $kvmver, $conf->{bios}, $gpu_passthrough) if $kvm;
3463
3464     push @$cpuFlags, 'enforce' if $cpu ne 'host' && $kvm;
3465
3466     push @$cpuFlags, 'kvm=off' if $kvm_off;
3467
3468     my $cpu_vendor = $cpu_vendor_list->{$cpu} ||
3469         die "internal error"; # should not happen
3470
3471     push @$cpuFlags, "vendor=${cpu_vendor}"
3472         if $cpu_vendor ne 'default';
3473
3474     $cpu .= "," . join(',', @$cpuFlags) if scalar(@$cpuFlags);
3475
3476     push @$cmd, '-cpu', $cpu;
3477
3478     PVE::QemuServer::Memory::config($conf, $vmid, $sockets, $cores, $defaults, $hotplug_features, $cmd);
3479
3480     push @$cmd, '-S' if $conf->{freeze};
3481
3482     push @$cmd, '-k', $conf->{keyboard} if defined($conf->{keyboard});
3483
3484     # enable sound
3485     #my $soundhw = $conf->{soundhw} || $defaults->{soundhw};
3486     #push @$cmd, '-soundhw', 'es1370';
3487     #push @$cmd, '-soundhw', $soundhw if $soundhw;
3488
3489     if (parse_guest_agent($conf)->{enabled}) {
3490         my $qgasocket = qmp_socket($vmid, 1);
3491         my $pciaddr = print_pci_addr("qga0", $bridges);
3492         push @$devices, '-chardev', "socket,path=$qgasocket,server,nowait,id=qga0";
3493         push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3494         push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3495     }
3496
3497     my $spice_port;
3498
3499     if ($qxlnum) {
3500         if ($qxlnum > 1) {
3501             if ($winversion){
3502                 for(my $i = 1; $i < $qxlnum; $i++){
3503                     my $pciaddr = print_pci_addr("vga$i", $bridges);
3504                     push @$devices, '-device', "qxl,id=vga$i,ram_size=67108864,vram_size=33554432$pciaddr";
3505                 }
3506             } else {
3507                 # assume other OS works like Linux
3508                 push @$cmd, '-global', 'qxl-vga.ram_size=134217728';
3509                 push @$cmd, '-global', 'qxl-vga.vram_size=67108864';
3510             }
3511         }
3512
3513         my $pciaddr = print_pci_addr("spice", $bridges);
3514
3515         my $nodename = PVE::INotify::nodename();
3516         my $pfamily = PVE::Tools::get_host_address_family($nodename);
3517         my @nodeaddrs = PVE::Tools::getaddrinfo_all('localhost', family => $pfamily);
3518         die "failed to get an ip address of type $pfamily for 'localhost'\n" if !@nodeaddrs;
3519         my $localhost = PVE::Network::addr_to_ip($nodeaddrs[0]->{addr});
3520         $spice_port = PVE::Tools::next_spice_port($pfamily, $localhost);
3521
3522         push @$devices, '-spice', "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";
3523
3524         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3525         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3526         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3527     }
3528
3529     # enable balloon by default, unless explicitly disabled
3530     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3531         $pciaddr = print_pci_addr("balloon0", $bridges);
3532         push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
3533     }
3534
3535     if ($conf->{watchdog}) {
3536         my $wdopts = parse_watchdog($conf->{watchdog});
3537         $pciaddr = print_pci_addr("watchdog", $bridges);
3538         my $watchdog = $wdopts->{model} || 'i6300esb';
3539         push @$devices, '-device', "$watchdog$pciaddr";
3540         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3541     }
3542
3543     my $vollist = [];
3544     my $scsicontroller = {};
3545     my $ahcicontroller = {};
3546     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3547
3548     # Add iscsi initiator name if available
3549     if (my $initiator = get_initiator_name()) {
3550         push @$devices, '-iscsi', "initiator-name=$initiator";
3551     }
3552
3553     foreach_drive($conf, sub {
3554         my ($ds, $drive) = @_;
3555
3556         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3557             push @$vollist, $drive->{file};
3558         }
3559
3560         # ignore efidisk here, already added in bios/fw handling code above
3561         return if $drive->{interface} eq 'efidisk';
3562
3563         $use_virtio = 1 if $ds =~ m/^virtio/;
3564
3565         if (drive_is_cdrom ($drive)) {
3566             if ($bootindex_hash->{d}) {
3567                 $drive->{bootindex} = $bootindex_hash->{d};
3568                 $bootindex_hash->{d} += 1;
3569             }
3570         } else {
3571             if ($bootindex_hash->{c}) {
3572                 $drive->{bootindex} = $bootindex_hash->{c} if $conf->{bootdisk} && ($conf->{bootdisk} eq $ds);
3573                 $bootindex_hash->{c} += 1;
3574             }
3575         }
3576
3577         if($drive->{interface} eq 'virtio'){
3578            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3579         }
3580
3581         if ($drive->{interface} eq 'scsi') {
3582
3583             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
3584
3585             $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges);
3586             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
3587
3588             my $iothread = '';
3589             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
3590                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3591                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3592             } elsif ($drive->{iothread}) {
3593                 warn "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n";
3594             }
3595
3596             my $queues = '';
3597             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
3598                 $queues = ",num_queues=$drive->{queues}";
3599             }
3600
3601             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues" if !$scsicontroller->{$controller};
3602             $scsicontroller->{$controller}=1;
3603         }
3604
3605         if ($drive->{interface} eq 'sata') {
3606            my $controller = int($drive->{index} / $MAX_SATA_DISKS);
3607            $pciaddr = print_pci_addr("ahci$controller", $bridges);
3608            push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr" if !$ahcicontroller->{$controller};
3609            $ahcicontroller->{$controller}=1;
3610         }
3611
3612         my $drive_cmd = print_drive_full($storecfg, $vmid, $drive);
3613         push @$devices, '-drive',$drive_cmd;
3614         push @$devices, '-device', print_drivedevice_full($storecfg, $conf, $vmid, $drive, $bridges);
3615     });
3616
3617     for (my $i = 0; $i < $MAX_NETS; $i++) {
3618          next if !$conf->{"net$i"};
3619          my $d = parse_net($conf->{"net$i"});
3620          next if !$d;
3621
3622          $use_virtio = 1 if $d->{model} eq 'virtio';
3623
3624          if ($bootindex_hash->{n}) {
3625             $d->{bootindex} = $bootindex_hash->{n};
3626             $bootindex_hash->{n} += 1;
3627          }
3628
3629          my $netdevfull = print_netdev_full($vmid,$conf,$d,"net$i");
3630          push @$devices, '-netdev', $netdevfull;
3631
3632          my $netdevicefull = print_netdevice_full($vmid, $conf, $d, "net$i", $bridges, $use_old_bios_files);
3633          push @$devices, '-device', $netdevicefull;
3634     }
3635
3636     if (!$q35) {
3637         # add pci bridges
3638         if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3639            $bridges->{1} = 1;
3640            $bridges->{2} = 1;
3641         }
3642
3643         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
3644
3645         while (my ($k, $v) = each %$bridges) {
3646             $pciaddr = print_pci_addr("pci.$k");
3647             unshift @$devices, '-device', "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr" if $k > 0;
3648         }
3649     }
3650
3651     # add custom args
3652     if ($conf->{args}) {
3653         my $aa = PVE::Tools::split_args($conf->{args});
3654         push @$cmd, @$aa;
3655     }
3656
3657     push @$cmd, @$devices;
3658     push @$cmd, '-rtc', join(',', @$rtcFlags)
3659         if scalar(@$rtcFlags);
3660     push @$cmd, '-machine', join(',', @$machineFlags)
3661         if scalar(@$machineFlags);
3662     push @$cmd, '-global', join(',', @$globalFlags)
3663         if scalar(@$globalFlags);
3664
3665     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
3666 }
3667
3668 sub vnc_socket {
3669     my ($vmid) = @_;
3670     return "${var_run_tmpdir}/$vmid.vnc";
3671 }
3672
3673 sub spice_port {
3674     my ($vmid) = @_;
3675
3676     my $res = vm_mon_cmd($vmid, 'query-spice');
3677
3678     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
3679 }
3680
3681 sub qmp_socket {
3682     my ($vmid, $qga) = @_;
3683     my $sockettype = $qga ? 'qga' : 'qmp';
3684     return "${var_run_tmpdir}/$vmid.$sockettype";
3685 }
3686
3687 sub pidfile_name {
3688     my ($vmid) = @_;
3689     return "${var_run_tmpdir}/$vmid.pid";
3690 }
3691
3692 sub vm_devices_list {
3693     my ($vmid) = @_;
3694
3695     my $res = vm_mon_cmd($vmid, 'query-pci');
3696     my $devices_to_check = [];
3697     my $devices = {};
3698     foreach my $pcibus (@$res) {
3699         push @$devices_to_check, @{$pcibus->{devices}},
3700     }
3701
3702     while (@$devices_to_check) {
3703         my $to_check = [];
3704         for my $d (@$devices_to_check) {
3705             $devices->{$d->{'qdev_id'}} = 1 if $d->{'qdev_id'};
3706             next if !$d->{'pci_bridge'};
3707
3708             $devices->{$d->{'qdev_id'}} += scalar(@{$d->{'pci_bridge'}->{devices}});
3709             push @$to_check, @{$d->{'pci_bridge'}->{devices}};
3710         }
3711         $devices_to_check = $to_check;
3712     }
3713
3714     my $resblock = vm_mon_cmd($vmid, 'query-block');
3715     foreach my $block (@$resblock) {
3716         if($block->{device} =~ m/^drive-(\S+)/){
3717                 $devices->{$1} = 1;
3718         }
3719     }
3720
3721     my $resmice = vm_mon_cmd($vmid, 'query-mice');
3722     foreach my $mice (@$resmice) {
3723         if ($mice->{name} eq 'QEMU HID Tablet') {
3724             $devices->{tablet} = 1;
3725             last;
3726         }
3727     }
3728
3729     # for usb devices there is no query-usb
3730     # but we can iterate over the entries in
3731     # qom-list path=/machine/peripheral
3732     my $resperipheral = vm_mon_cmd($vmid, 'qom-list', path => '/machine/peripheral');
3733     foreach my $per (@$resperipheral) {
3734         if ($per->{name} =~ m/^usb\d+$/) {
3735             $devices->{$per->{name}} = 1;
3736         }
3737     }
3738
3739     return $devices;
3740 }
3741
3742 sub vm_deviceplug {
3743     my ($storecfg, $conf, $vmid, $deviceid, $device) = @_;
3744
3745     my $q35 = machine_type_is_q35($conf);
3746
3747     my $devices_list = vm_devices_list($vmid);
3748     return 1 if defined($devices_list->{$deviceid});
3749
3750     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid); # add PCI bridge if we need it for the device
3751
3752     if ($deviceid eq 'tablet') {
3753
3754         qemu_deviceadd($vmid, print_tabletdevice_full($conf));
3755
3756     } elsif ($deviceid =~ m/^usb(\d+)$/) {
3757
3758         die "usb hotplug currently not reliable\n";
3759         # since we can't reliably hot unplug all added usb devices
3760         # and usb passthrough disables live migration
3761         # we disable usb hotplugging for now
3762         qemu_deviceadd($vmid, PVE::QemuServer::USB::print_usbdevice_full($conf, $deviceid, $device));
3763
3764     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3765
3766         qemu_iothread_add($vmid, $deviceid, $device);
3767
3768         qemu_driveadd($storecfg, $vmid, $device);
3769         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
3770
3771         qemu_deviceadd($vmid, $devicefull);
3772         eval { qemu_deviceaddverify($vmid, $deviceid); };
3773         if (my $err = $@) {
3774             eval { qemu_drivedel($vmid, $deviceid); };
3775             warn $@ if $@;
3776             die $err;
3777         }
3778
3779     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3780
3781
3782         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
3783         my $pciaddr = print_pci_addr($deviceid);
3784         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
3785
3786         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
3787
3788         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
3789             qemu_iothread_add($vmid, $deviceid, $device);
3790             $devicefull .= ",iothread=iothread-$deviceid";
3791         }
3792
3793         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
3794             $devicefull .= ",num_queues=$device->{queues}";
3795         }
3796
3797         qemu_deviceadd($vmid, $devicefull);
3798         qemu_deviceaddverify($vmid, $deviceid);
3799
3800     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3801
3802         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device);
3803         qemu_driveadd($storecfg, $vmid, $device);
3804
3805         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
3806         eval { qemu_deviceadd($vmid, $devicefull); };
3807         if (my $err = $@) {
3808             eval { qemu_drivedel($vmid, $deviceid); };
3809             warn $@ if $@;
3810             die $err;
3811         }
3812
3813     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3814
3815         return undef if !qemu_netdevadd($vmid, $conf, $device, $deviceid);
3816
3817         my $machine_type = PVE::QemuServer::qemu_machine_pxe($vmid, $conf);
3818         my $use_old_bios_files = undef;
3819         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3820
3821         my $netdevicefull = print_netdevice_full($vmid, $conf, $device, $deviceid, undef, $use_old_bios_files);
3822         qemu_deviceadd($vmid, $netdevicefull);
3823         eval { qemu_deviceaddverify($vmid, $deviceid); };
3824         if (my $err = $@) {
3825             eval { qemu_netdevdel($vmid, $deviceid); };
3826             warn $@ if $@;
3827             die $err;
3828         }
3829
3830     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
3831
3832         my $bridgeid = $2;
3833         my $pciaddr = print_pci_addr($deviceid);
3834         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
3835
3836         qemu_deviceadd($vmid, $devicefull);
3837         qemu_deviceaddverify($vmid, $deviceid);
3838
3839     } else {
3840         die "can't hotplug device '$deviceid'\n";
3841     }
3842
3843     return 1;
3844 }
3845
3846 # fixme: this should raise exceptions on error!
3847 sub vm_deviceunplug {
3848     my ($vmid, $conf, $deviceid) = @_;
3849
3850     my $devices_list = vm_devices_list($vmid);
3851     return 1 if !defined($devices_list->{$deviceid});
3852
3853     die "can't unplug bootdisk" if $conf->{bootdisk} && $conf->{bootdisk} eq $deviceid;
3854
3855     if ($deviceid eq 'tablet') {
3856
3857         qemu_devicedel($vmid, $deviceid);
3858
3859     } elsif ($deviceid =~ m/^usb\d+$/) {
3860
3861         die "usb hotplug currently not reliable\n";
3862         # when unplugging usb devices this way,
3863         # there may be remaining usb controllers/hubs
3864         # so we disable it for now
3865         qemu_devicedel($vmid, $deviceid);
3866         qemu_devicedelverify($vmid, $deviceid);
3867
3868     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3869
3870         qemu_devicedel($vmid, $deviceid);
3871         qemu_devicedelverify($vmid, $deviceid);
3872         qemu_drivedel($vmid, $deviceid);
3873         qemu_iothread_del($conf, $vmid, $deviceid);
3874
3875     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3876
3877         qemu_devicedel($vmid, $deviceid);
3878         qemu_devicedelverify($vmid, $deviceid);
3879         qemu_iothread_del($conf, $vmid, $deviceid);
3880
3881     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3882
3883         qemu_devicedel($vmid, $deviceid);
3884         qemu_drivedel($vmid, $deviceid);
3885         qemu_deletescsihw($conf, $vmid, $deviceid);
3886
3887     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3888
3889         qemu_devicedel($vmid, $deviceid);
3890         qemu_devicedelverify($vmid, $deviceid);
3891         qemu_netdevdel($vmid, $deviceid);
3892
3893     } else {
3894         die "can't unplug device '$deviceid'\n";
3895     }
3896
3897     return 1;
3898 }
3899
3900 sub qemu_deviceadd {
3901     my ($vmid, $devicefull) = @_;
3902
3903     $devicefull = "driver=".$devicefull;
3904     my %options =  split(/[=,]/, $devicefull);
3905
3906     vm_mon_cmd($vmid, "device_add" , %options);
3907 }
3908
3909 sub qemu_devicedel {
3910     my ($vmid, $deviceid) = @_;
3911
3912     my $ret = vm_mon_cmd($vmid, "device_del", id => $deviceid);
3913 }
3914
3915 sub qemu_iothread_add {
3916     my($vmid, $deviceid, $device) = @_;
3917
3918     if ($device->{iothread}) {
3919         my $iothreads = vm_iothreads_list($vmid);
3920         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
3921     }
3922 }
3923
3924 sub qemu_iothread_del {
3925     my($conf, $vmid, $deviceid) = @_;
3926
3927     my $device = parse_drive($deviceid, $conf->{$deviceid});
3928     if ($device->{iothread}) {
3929         my $iothreads = vm_iothreads_list($vmid);
3930         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
3931     }
3932 }
3933
3934 sub qemu_objectadd {
3935     my($vmid, $objectid, $qomtype) = @_;
3936
3937     vm_mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
3938
3939     return 1;
3940 }
3941
3942 sub qemu_objectdel {
3943     my($vmid, $objectid) = @_;
3944
3945     vm_mon_cmd($vmid, "object-del", id => $objectid);
3946
3947     return 1;
3948 }
3949
3950 sub qemu_driveadd {
3951     my ($storecfg, $vmid, $device) = @_;
3952
3953     my $drive = print_drive_full($storecfg, $vmid, $device);
3954     $drive =~ s/\\/\\\\/g;
3955     my $ret = vm_human_monitor_command($vmid, "drive_add auto \"$drive\"");
3956
3957     # If the command succeeds qemu prints: "OK"
3958     return 1 if $ret =~ m/OK/s;
3959
3960     die "adding drive failed: $ret\n";
3961 }
3962
3963 sub qemu_drivedel {
3964     my($vmid, $deviceid) = @_;
3965
3966     my $ret = vm_human_monitor_command($vmid, "drive_del drive-$deviceid");
3967     $ret =~ s/^\s+//;
3968
3969     return 1 if $ret eq "";
3970
3971     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
3972     return 1 if $ret =~ m/Device \'.*?\' not found/s;
3973
3974     die "deleting drive $deviceid failed : $ret\n";
3975 }
3976
3977 sub qemu_deviceaddverify {
3978     my ($vmid, $deviceid) = @_;
3979
3980     for (my $i = 0; $i <= 5; $i++) {
3981          my $devices_list = vm_devices_list($vmid);
3982          return 1 if defined($devices_list->{$deviceid});
3983          sleep 1;
3984     }
3985
3986     die "error on hotplug device '$deviceid'\n";
3987 }
3988
3989
3990 sub qemu_devicedelverify {
3991     my ($vmid, $deviceid) = @_;
3992
3993     # need to verify that the device is correctly removed as device_del
3994     # is async and empty return is not reliable
3995
3996     for (my $i = 0; $i <= 5; $i++) {
3997          my $devices_list = vm_devices_list($vmid);
3998          return 1 if !defined($devices_list->{$deviceid});
3999          sleep 1;
4000     }
4001
4002     die "error on hot-unplugging device '$deviceid'\n";
4003 }
4004
4005 sub qemu_findorcreatescsihw {
4006     my ($storecfg, $conf, $vmid, $device) = @_;
4007
4008     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4009
4010     my $scsihwid="$controller_prefix$controller";
4011     my $devices_list = vm_devices_list($vmid);
4012
4013     if(!defined($devices_list->{$scsihwid})) {
4014         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device);
4015     }
4016
4017     return 1;
4018 }
4019
4020 sub qemu_deletescsihw {
4021     my ($conf, $vmid, $opt) = @_;
4022
4023     my $device = parse_drive($opt, $conf->{$opt});
4024
4025     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
4026         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
4027         return 1;
4028     }
4029
4030     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4031
4032     my $devices_list = vm_devices_list($vmid);
4033     foreach my $opt (keys %{$devices_list}) {
4034         if (PVE::QemuServer::is_valid_drivename($opt)) {
4035             my $drive = PVE::QemuServer::parse_drive($opt, $conf->{$opt});
4036             if($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
4037                 return 1;
4038             }
4039         }
4040     }
4041
4042     my $scsihwid="scsihw$controller";
4043
4044     vm_deviceunplug($vmid, $conf, $scsihwid);
4045
4046     return 1;
4047 }
4048
4049 sub qemu_add_pci_bridge {
4050     my ($storecfg, $conf, $vmid, $device) = @_;
4051
4052     my $bridges = {};
4053
4054     my $bridgeid;
4055
4056     print_pci_addr($device, $bridges);
4057
4058     while (my ($k, $v) = each %$bridges) {
4059         $bridgeid = $k;
4060     }
4061     return 1 if !defined($bridgeid) || $bridgeid < 1;
4062
4063     my $bridge = "pci.$bridgeid";
4064     my $devices_list = vm_devices_list($vmid);
4065
4066     if (!defined($devices_list->{$bridge})) {
4067         vm_deviceplug($storecfg, $conf, $vmid, $bridge);
4068     }
4069
4070     return 1;
4071 }
4072
4073 sub qemu_set_link_status {
4074     my ($vmid, $device, $up) = @_;
4075
4076     vm_mon_cmd($vmid, "set_link", name => $device,
4077                up => $up ? JSON::true : JSON::false);
4078 }
4079
4080 sub qemu_netdevadd {
4081     my ($vmid, $conf, $device, $deviceid) = @_;
4082
4083     my $netdev = print_netdev_full($vmid, $conf, $device, $deviceid, 1);
4084     my %options =  split(/[=,]/, $netdev);
4085
4086     vm_mon_cmd($vmid, "netdev_add",  %options);
4087     return 1;
4088 }
4089
4090 sub qemu_netdevdel {
4091     my ($vmid, $deviceid) = @_;
4092
4093     vm_mon_cmd($vmid, "netdev_del", id => $deviceid);
4094 }
4095
4096 sub qemu_usb_hotplug {
4097     my ($storecfg, $conf, $vmid, $deviceid, $device) = @_;
4098
4099     return if !$device;
4100
4101     # remove the old one first
4102     vm_deviceunplug($vmid, $conf, $deviceid);
4103
4104     # check if xhci controller is necessary and available
4105     if ($device->{usb3}) {
4106
4107         my $devicelist = vm_devices_list($vmid);
4108
4109         if (!$devicelist->{xhci}) {
4110             my $pciaddr = print_pci_addr("xhci");
4111             qemu_deviceadd($vmid, "nec-usb-xhci,id=xhci$pciaddr");
4112         }
4113     }
4114     my $d = parse_usb_device($device->{host});
4115     $d->{usb3} = $device->{usb3};
4116
4117     # add the new one
4118     vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $d);
4119 }
4120
4121 sub qemu_cpu_hotplug {
4122     my ($vmid, $conf, $vcpus) = @_;
4123
4124     my $machine_type = PVE::QemuServer::get_current_qemu_machine($vmid);
4125
4126     my $sockets = 1;
4127     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
4128     $sockets = $conf->{sockets} if  $conf->{sockets};
4129     my $cores = $conf->{cores} || 1;
4130     my $maxcpus = $sockets * $cores;
4131
4132     $vcpus = $maxcpus if !$vcpus;
4133
4134     die "you can't add more vcpus than maxcpus\n"
4135         if $vcpus > $maxcpus;
4136
4137     my $currentvcpus = $conf->{vcpus} || $maxcpus;
4138
4139     if ($vcpus < $currentvcpus) {
4140
4141         if (qemu_machine_feature_enabled ($machine_type, undef, 2, 7)) {
4142
4143             for (my $i = $currentvcpus; $i > $vcpus; $i--) {
4144                 qemu_devicedel($vmid, "cpu$i");
4145                 my $retry = 0;
4146                 my $currentrunningvcpus = undef;
4147                 while (1) {
4148                     $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
4149                     last if scalar(@{$currentrunningvcpus}) == $i-1;
4150                     raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
4151                     $retry++;
4152                     sleep 1;
4153                 }
4154                 #update conf after each succesfull cpu unplug
4155                 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4156                 PVE::QemuConfig->write_config($vmid, $conf);
4157             }
4158         } else {
4159             die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
4160         }
4161
4162         return;
4163     }
4164
4165     my $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
4166     die "vcpus in running vm does not match its configuration\n"
4167         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
4168
4169     if (qemu_machine_feature_enabled ($machine_type, undef, 2, 7)) {
4170
4171         for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
4172             my $cpustr = print_cpu_device($conf, $i);
4173             qemu_deviceadd($vmid, $cpustr);
4174
4175             my $retry = 0;
4176             my $currentrunningvcpus = undef;
4177             while (1) {
4178                 $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
4179                 last if scalar(@{$currentrunningvcpus}) == $i;
4180                 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
4181                 sleep 1;
4182                 $retry++;
4183             }
4184             #update conf after each succesfull cpu hotplug
4185             $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4186             PVE::QemuConfig->write_config($vmid, $conf);
4187         }
4188     } else {
4189
4190         for (my $i = $currentvcpus; $i < $vcpus; $i++) {
4191             vm_mon_cmd($vmid, "cpu-add", id => int($i));
4192         }
4193     }
4194 }
4195
4196 sub qemu_block_set_io_throttle {
4197     my ($vmid, $deviceid,
4198         $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
4199         $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max,
4200         $bps_max_length, $bps_rd_max_length, $bps_wr_max_length,
4201         $iops_max_length, $iops_rd_max_length, $iops_wr_max_length) = @_;
4202
4203     return if !check_running($vmid) ;
4204
4205     vm_mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
4206         bps => int($bps),
4207         bps_rd => int($bps_rd),
4208         bps_wr => int($bps_wr),
4209         iops => int($iops),
4210         iops_rd => int($iops_rd),
4211         iops_wr => int($iops_wr),
4212         bps_max => int($bps_max),
4213         bps_rd_max => int($bps_rd_max),
4214         bps_wr_max => int($bps_wr_max),
4215         iops_max => int($iops_max),
4216         iops_rd_max => int($iops_rd_max),
4217         iops_wr_max => int($iops_wr_max),
4218         bps_max_length => int($bps_max_length),
4219         bps_rd_max_length => int($bps_rd_max_length),
4220         bps_wr_max_length => int($bps_wr_max_length),
4221         iops_max_length => int($iops_max_length),
4222         iops_rd_max_length => int($iops_rd_max_length),
4223         iops_wr_max_length => int($iops_wr_max_length),
4224     );
4225
4226 }
4227
4228 # old code, only used to shutdown old VM after update
4229 sub __read_avail {
4230     my ($fh, $timeout) = @_;
4231
4232     my $sel = new IO::Select;
4233     $sel->add($fh);
4234
4235     my $res = '';
4236     my $buf;
4237
4238     my @ready;
4239     while (scalar (@ready = $sel->can_read($timeout))) {
4240         my $count;
4241         if ($count = $fh->sysread($buf, 8192)) {
4242             if ($buf =~ /^(.*)\(qemu\) $/s) {
4243                 $res .= $1;
4244                 last;
4245             } else {
4246                 $res .= $buf;
4247             }
4248         } else {
4249             if (!defined($count)) {
4250                 die "$!\n";
4251             }
4252             last;
4253         }
4254     }
4255
4256     die "monitor read timeout\n" if !scalar(@ready);
4257
4258     return $res;
4259 }
4260
4261 sub qemu_block_resize {
4262     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4263
4264     my $running = check_running($vmid);
4265
4266     $size = 0 if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4267
4268     return if !$running;
4269
4270     vm_mon_cmd($vmid, "block_resize", device => $deviceid, size => int($size));
4271
4272 }
4273
4274 sub qemu_volume_snapshot {
4275     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4276
4277     my $running = check_running($vmid);
4278
4279     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4280         vm_mon_cmd($vmid, "snapshot-drive", device => $deviceid, name => $snap);
4281     } else {
4282         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4283     }
4284 }
4285
4286 sub qemu_volume_snapshot_delete {
4287     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4288
4289     my $running = check_running($vmid);
4290
4291     if($running) {
4292
4293         $running = undef;
4294         my $conf = PVE::QemuConfig->load_config($vmid);
4295         foreach_drive($conf, sub {
4296             my ($ds, $drive) = @_;
4297             $running = 1 if $drive->{file} eq $volid;
4298         });
4299     }
4300
4301     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4302         vm_mon_cmd($vmid, "delete-drive-snapshot", device => $deviceid, name => $snap);
4303     } else {
4304         PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4305     }
4306 }
4307
4308 sub set_migration_caps {
4309     my ($vmid) = @_;
4310
4311     my $cap_ref = [];
4312
4313     my $enabled_cap = {
4314         "auto-converge" => 1,
4315         "xbzrle" => 1,
4316         "x-rdma-pin-all" => 0,
4317         "zero-blocks" => 0,
4318         "compress" => 0
4319     };
4320
4321     my $supported_capabilities = vm_mon_cmd_nocheck($vmid, "query-migrate-capabilities");
4322
4323     for my $supported_capability (@$supported_capabilities) {
4324         push @$cap_ref, {
4325             capability => $supported_capability->{capability},
4326             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4327         };
4328     }
4329
4330     vm_mon_cmd_nocheck($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4331 }
4332
4333 my $fast_plug_option = {
4334     'lock' => 1,
4335     'name' => 1,
4336     'onboot' => 1,
4337     'shares' => 1,
4338     'startup' => 1,
4339     'description' => 1,
4340     'protection' => 1,
4341     'vmstatestorage' => 1,
4342 };
4343
4344 # hotplug changes in [PENDING]
4345 # $selection hash can be used to only apply specified options, for
4346 # example: { cores => 1 } (only apply changed 'cores')
4347 # $errors ref is used to return error messages
4348 sub vmconfig_hotplug_pending {
4349     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4350
4351     my $defaults = load_defaults();
4352
4353     # commit values which do not have any impact on running VM first
4354     # Note: those option cannot raise errors, we we do not care about
4355     # $selection and always apply them.
4356
4357     my $add_error = sub {
4358         my ($opt, $msg) = @_;
4359         $errors->{$opt} = "hotplug problem - $msg";
4360     };
4361
4362     my $changes = 0;
4363     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4364         if ($fast_plug_option->{$opt}) {
4365             $conf->{$opt} = $conf->{pending}->{$opt};
4366             delete $conf->{pending}->{$opt};
4367             $changes = 1;
4368         }
4369     }
4370
4371     if ($changes) {
4372         PVE::QemuConfig->write_config($vmid, $conf);
4373         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4374     }
4375
4376     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4377
4378     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4379     while (my ($opt, $force) = each %$pending_delete_hash) {
4380         next if $selection && !$selection->{$opt};
4381         eval {
4382             if ($opt eq 'hotplug') {
4383                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
4384             } elsif ($opt eq 'tablet') {
4385                 die "skip\n" if !$hotplug_features->{usb};
4386                 if ($defaults->{tablet}) {
4387                     vm_deviceplug($storecfg, $conf, $vmid, $opt);
4388                 } else {
4389                     vm_deviceunplug($vmid, $conf, $opt);
4390                 }
4391             } elsif ($opt =~ m/^usb\d+/) {
4392                 die "skip\n";
4393                 # since we cannot reliably hot unplug usb devices
4394                 # we are disabling it
4395                 die "skip\n" if !$hotplug_features->{usb} || $conf->{$opt} =~ m/spice/i;
4396                 vm_deviceunplug($vmid, $conf, $opt);
4397             } elsif ($opt eq 'vcpus') {
4398                 die "skip\n" if !$hotplug_features->{cpu};
4399                 qemu_cpu_hotplug($vmid, $conf, undef);
4400             } elsif ($opt eq 'balloon') {
4401                 # enable balloon device is not hotpluggable
4402                 die "skip\n" if defined($conf->{balloon}) && $conf->{balloon} == 0;
4403                 # here we reset the ballooning value to memory
4404                 my $balloon = $conf->{memory} || $defaults->{memory};
4405                 vm_mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4406             } elsif ($fast_plug_option->{$opt}) {
4407                 # do nothing
4408             } elsif ($opt =~ m/^net(\d+)$/) {
4409                 die "skip\n" if !$hotplug_features->{network};
4410                 vm_deviceunplug($vmid, $conf, $opt);
4411             } elsif (is_valid_drivename($opt)) {
4412                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4413                 vm_deviceunplug($vmid, $conf, $opt);
4414                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4415             } elsif ($opt =~ m/^memory$/) {
4416                 die "skip\n" if !$hotplug_features->{memory};
4417                 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
4418             } elsif ($opt eq 'cpuunits') {
4419                 cgroups_write("cpu", $vmid, "cpu.shares", $defaults->{cpuunits});
4420             } elsif ($opt eq 'cpulimit') {
4421                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", -1);
4422             } else {
4423                 die "skip\n";
4424             }
4425         };
4426         if (my $err = $@) {
4427             &$add_error($opt, $err) if $err ne "skip\n";
4428         } else {
4429             # save new config if hotplug was successful
4430             delete $conf->{$opt};
4431             vmconfig_undelete_pending_option($conf, $opt);
4432             PVE::QemuConfig->write_config($vmid, $conf);
4433             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4434         }
4435     }
4436
4437     my $apply_pending_cloudinit;
4438     $apply_pending_cloudinit = sub {
4439         my ($key, $value) = @_;
4440         $apply_pending_cloudinit = sub {}; # once is enough
4441
4442         my @cloudinit_opts = keys %$confdesc_cloudinit;
4443         foreach my $opt (keys %{$conf->{pending}}) {
4444             next if !grep { $_ eq $opt } @cloudinit_opts;
4445             $conf->{$opt} = delete $conf->{pending}->{$opt};
4446         }
4447
4448         my $new_conf = { %$conf };
4449         $new_conf->{$key} = $value;
4450         PVE::QemuServer::Cloudinit::generate_cloudinitconfig($new_conf, $vmid);
4451     };
4452
4453     foreach my $opt (keys %{$conf->{pending}}) {
4454         next if $selection && !$selection->{$opt};
4455         my $value = $conf->{pending}->{$opt};
4456         eval {
4457             if ($opt eq 'hotplug') {
4458                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
4459             } elsif ($opt eq 'tablet') {
4460                 die "skip\n" if !$hotplug_features->{usb};
4461                 if ($value == 1) {
4462                     vm_deviceplug($storecfg, $conf, $vmid, $opt);
4463                 } elsif ($value == 0) {
4464                     vm_deviceunplug($vmid, $conf, $opt);
4465                 }
4466             } elsif ($opt =~ m/^usb\d+$/) {
4467                 die "skip\n";
4468                 # since we cannot reliably hot unplug usb devices
4469                 # we are disabling it
4470                 die "skip\n" if !$hotplug_features->{usb} || $value =~ m/spice/i;
4471                 my $d = eval { PVE::JSONSchema::parse_property_string($usbdesc->{format}, $value) };
4472                 die "skip\n" if !$d;
4473                 qemu_usb_hotplug($storecfg, $conf, $vmid, $opt, $d);
4474             } elsif ($opt eq 'vcpus') {
4475                 die "skip\n" if !$hotplug_features->{cpu};
4476                 qemu_cpu_hotplug($vmid, $conf, $value);
4477             } elsif ($opt eq 'balloon') {
4478                 # enable/disable balloning device is not hotpluggable
4479                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
4480                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
4481                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
4482
4483                 # allow manual ballooning if shares is set to zero
4484                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
4485                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
4486                     vm_mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4487                 }
4488             } elsif ($opt =~ m/^net(\d+)$/) {
4489                 # some changes can be done without hotplug
4490                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
4491                                     $vmid, $opt, $value);
4492             } elsif (is_valid_drivename($opt)) {
4493                 # some changes can be done without hotplug
4494                 my $drive = parse_drive($opt, $value);
4495                 if (drive_is_cloudinit($drive)) {
4496                     &$apply_pending_cloudinit($opt, $value);
4497                 }
4498                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
4499                                      $vmid, $opt, $value, 1);
4500             } elsif ($opt =~ m/^memory$/) { #dimms
4501                 die "skip\n" if !$hotplug_features->{memory};
4502                 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
4503             } elsif ($opt eq 'cpuunits') {
4504                 cgroups_write("cpu", $vmid, "cpu.shares", $conf->{pending}->{$opt});
4505             } elsif ($opt eq 'cpulimit') {
4506                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
4507                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", $cpulimit);
4508             } else {
4509                 die "skip\n";  # skip non-hot-pluggable options
4510             }
4511         };
4512         if (my $err = $@) {
4513             &$add_error($opt, $err) if $err ne "skip\n";
4514         } else {
4515             # save new config if hotplug was successful
4516             $conf->{$opt} = $value;
4517             delete $conf->{pending}->{$opt};
4518             PVE::QemuConfig->write_config($vmid, $conf);
4519             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4520         }
4521     }
4522 }
4523
4524 sub try_deallocate_drive {
4525     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
4526
4527     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
4528         my $volid = $drive->{file};
4529         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
4530             my $sid = PVE::Storage::parse_volume_id($volid);
4531             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
4532
4533             # check if the disk is really unused
4534             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
4535                 if is_volume_in_use($storecfg, $conf, $key, $volid);
4536             PVE::Storage::vdisk_free($storecfg, $volid);
4537             return 1;
4538         } else {
4539             # If vm is not owner of this disk remove from config
4540             return 1;
4541         }
4542     }
4543
4544     return undef;
4545 }
4546
4547 sub vmconfig_delete_or_detach_drive {
4548     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
4549
4550     my $drive = parse_drive($opt, $conf->{$opt});
4551
4552     my $rpcenv = PVE::RPCEnvironment::get();
4553     my $authuser = $rpcenv->get_user();
4554
4555     if ($force) {
4556         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
4557         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
4558     } else {
4559         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
4560     }
4561 }
4562
4563 sub vmconfig_apply_pending {
4564     my ($vmid, $conf, $storecfg) = @_;
4565
4566     # cold plug
4567
4568     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4569     while (my ($opt, $force) = each %$pending_delete_hash) {
4570         die "internal error" if $opt =~ m/^unused/;
4571         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4572         if (!defined($conf->{$opt})) {
4573             vmconfig_undelete_pending_option($conf, $opt);
4574             PVE::QemuConfig->write_config($vmid, $conf);
4575         } elsif (is_valid_drivename($opt)) {
4576             vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4577             vmconfig_undelete_pending_option($conf, $opt);
4578             delete $conf->{$opt};
4579             PVE::QemuConfig->write_config($vmid, $conf);
4580         } else {
4581             vmconfig_undelete_pending_option($conf, $opt);
4582             delete $conf->{$opt};
4583             PVE::QemuConfig->write_config($vmid, $conf);
4584         }
4585     }
4586
4587     $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4588
4589     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4590         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4591
4592         if (defined($conf->{$opt}) && ($conf->{$opt} eq $conf->{pending}->{$opt})) {
4593             # skip if nothing changed
4594         } elsif (is_valid_drivename($opt)) {
4595             vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
4596                 if defined($conf->{$opt});
4597             $conf->{$opt} = $conf->{pending}->{$opt};
4598         } else {
4599             $conf->{$opt} = $conf->{pending}->{$opt};
4600         }
4601
4602         delete $conf->{pending}->{$opt};
4603         PVE::QemuConfig->write_config($vmid, $conf);
4604     }
4605 }
4606
4607 my $safe_num_ne = sub {
4608     my ($a, $b) = @_;
4609
4610     return 0 if !defined($a) && !defined($b);
4611     return 1 if !defined($a);
4612     return 1 if !defined($b);
4613
4614     return $a != $b;
4615 };
4616
4617 my $safe_string_ne = sub {
4618     my ($a, $b) = @_;
4619
4620     return 0 if !defined($a) && !defined($b);
4621     return 1 if !defined($a);
4622     return 1 if !defined($b);
4623
4624     return $a ne $b;
4625 };
4626
4627 sub vmconfig_update_net {
4628     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value) = @_;
4629
4630     my $newnet = parse_net($value);
4631
4632     if ($conf->{$opt}) {
4633         my $oldnet = parse_net($conf->{$opt});
4634
4635         if (&$safe_string_ne($oldnet->{model}, $newnet->{model}) ||
4636             &$safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
4637             &$safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
4638             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
4639
4640             # for non online change, we try to hot-unplug
4641             die "skip\n" if !$hotplug;
4642             vm_deviceunplug($vmid, $conf, $opt);
4643         } else {
4644
4645             die "internal error" if $opt !~ m/net(\d+)/;
4646             my $iface = "tap${vmid}i$1";
4647
4648             if (&$safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
4649                 &$safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
4650                 &$safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
4651                 &$safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
4652                 PVE::Network::tap_unplug($iface);
4653                 PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
4654             } elsif (&$safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
4655                 # Rate can be applied on its own but any change above needs to
4656                 # include the rate in tap_plug since OVS resets everything.
4657                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
4658             }
4659
4660             if (&$safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
4661                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
4662             }
4663
4664             return 1;
4665         }
4666     }
4667
4668     if ($hotplug) {
4669         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet);
4670     } else {
4671         die "skip\n";
4672     }
4673 }
4674
4675 sub vmconfig_update_disk {
4676     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $force) = @_;
4677
4678     # fixme: do we need force?
4679
4680     my $drive = parse_drive($opt, $value);
4681
4682     if ($conf->{$opt}) {
4683
4684         if (my $old_drive = parse_drive($opt, $conf->{$opt}))  {
4685
4686             my $media = $drive->{media} || 'disk';
4687             my $oldmedia = $old_drive->{media} || 'disk';
4688             die "unable to change media type\n" if $media ne $oldmedia;
4689
4690             if (!drive_is_cdrom($old_drive)) {
4691
4692                 if ($drive->{file} ne $old_drive->{file}) {
4693
4694                     die "skip\n" if !$hotplug;
4695
4696                     # unplug and register as unused
4697                     vm_deviceunplug($vmid, $conf, $opt);
4698                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
4699
4700                 } else {
4701                     # update existing disk
4702
4703                     # skip non hotpluggable value
4704                     if (&$safe_string_ne($drive->{discard}, $old_drive->{discard}) ||
4705                         &$safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
4706                         &$safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
4707                         &$safe_string_ne($drive->{cache}, $old_drive->{cache})) {
4708                         die "skip\n";
4709                     }
4710
4711                     # apply throttle
4712                     if (&$safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
4713                         &$safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
4714                         &$safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
4715                         &$safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
4716                         &$safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
4717                         &$safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
4718                         &$safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
4719                         &$safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
4720                         &$safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
4721                         &$safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
4722                         &$safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
4723                         &$safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max}) ||
4724                         &$safe_num_ne($drive->{bps_max_length}, $old_drive->{bps_max_length}) ||
4725                         &$safe_num_ne($drive->{bps_rd_max_length}, $old_drive->{bps_rd_max_length}) ||
4726                         &$safe_num_ne($drive->{bps_wr_max_length}, $old_drive->{bps_wr_max_length}) ||
4727                         &$safe_num_ne($drive->{iops_max_length}, $old_drive->{iops_max_length}) ||
4728                         &$safe_num_ne($drive->{iops_rd_max_length}, $old_drive->{iops_rd_max_length}) ||
4729                         &$safe_num_ne($drive->{iops_wr_max_length}, $old_drive->{iops_wr_max_length})) {
4730
4731                         qemu_block_set_io_throttle($vmid,"drive-$opt",
4732                                                    ($drive->{mbps} || 0)*1024*1024,
4733                                                    ($drive->{mbps_rd} || 0)*1024*1024,
4734                                                    ($drive->{mbps_wr} || 0)*1024*1024,
4735                                                    $drive->{iops} || 0,
4736                                                    $drive->{iops_rd} || 0,
4737                                                    $drive->{iops_wr} || 0,
4738                                                    ($drive->{mbps_max} || 0)*1024*1024,
4739                                                    ($drive->{mbps_rd_max} || 0)*1024*1024,
4740                                                    ($drive->{mbps_wr_max} || 0)*1024*1024,
4741                                                    $drive->{iops_max} || 0,
4742                                                    $drive->{iops_rd_max} || 0,
4743                                                    $drive->{iops_wr_max} || 0,
4744                                                    $drive->{bps_max_length} || 1,
4745                                                    $drive->{bps_rd_max_length} || 1,
4746                                                    $drive->{bps_wr_max_length} || 1,
4747                                                    $drive->{iops_max_length} || 1,
4748                                                    $drive->{iops_rd_max_length} || 1,
4749                                                    $drive->{iops_wr_max_length} || 1);
4750
4751                     }
4752
4753                     return 1;
4754                 }
4755
4756             } else { # cdrom
4757
4758                 if ($drive->{file} eq 'none') {
4759                     vm_mon_cmd($vmid, "eject",force => JSON::true,device => "drive-$opt");
4760                     if (drive_is_cloudinit($old_drive)) {
4761                         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive);
4762                     }
4763                 } else {
4764                     my $path = get_iso_path($storecfg, $vmid, $drive->{file});
4765                     vm_mon_cmd($vmid, "eject", force => JSON::true,device => "drive-$opt"); # force eject if locked
4766                     vm_mon_cmd($vmid, "change", device => "drive-$opt",target => "$path") if $path;
4767                 }
4768
4769                 return 1;
4770             }
4771         }
4772     }
4773
4774     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
4775     # hotplug new disks
4776     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
4777     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive);
4778 }
4779
4780 sub vm_start {
4781     my ($storecfg, $vmid, $statefile, $skiplock, $migratedfrom, $paused,
4782         $forcemachine, $spice_ticket, $migration_network, $migration_type, $targetstorage) = @_;
4783
4784     PVE::QemuConfig->lock_config($vmid, sub {
4785         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
4786
4787         die "you can't start a vm if it's a template\n" if PVE::QemuConfig->is_template($conf);
4788
4789         PVE::QemuConfig->check_lock($conf) if !$skiplock;
4790
4791         die "VM $vmid already running\n" if check_running($vmid, undef, $migratedfrom);
4792
4793         if (!$statefile && scalar(keys %{$conf->{pending}})) {
4794             vmconfig_apply_pending($vmid, $conf, $storecfg);
4795             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4796         }
4797
4798         PVE::QemuServer::Cloudinit::generate_cloudinitconfig($conf, $vmid);
4799
4800         my $defaults = load_defaults();
4801
4802         # set environment variable useful inside network script
4803         $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
4804
4805         my $local_volumes = {};
4806
4807         if ($targetstorage) {
4808             foreach_drive($conf, sub {
4809                 my ($ds, $drive) = @_;
4810
4811                 return if drive_is_cdrom($drive);
4812
4813                 my $volid = $drive->{file};
4814
4815                 return if !$volid;
4816
4817                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
4818
4819                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
4820                 return if $scfg->{shared};
4821                 $local_volumes->{$ds} = [$volid, $storeid, $volname];
4822             });
4823
4824             my $format = undef;
4825
4826             foreach my $opt (sort keys %$local_volumes) {
4827
4828                 my ($volid, $storeid, $volname) = @{$local_volumes->{$opt}};
4829                 my $drive = parse_drive($opt, $conf->{$opt});
4830
4831                 #if remote storage is specified, use default format
4832                 if ($targetstorage && $targetstorage ne "1") {
4833                     $storeid = $targetstorage;
4834                     my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
4835                     $format = $defFormat;
4836                 } else {
4837                     #else we use same format than original
4838                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
4839                     $format = qemu_img_format($scfg, $volid);
4840                 }
4841
4842                 my $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $format, undef, ($drive->{size}/1024));
4843                 my $newdrive = $drive;
4844                 $newdrive->{format} = $format;
4845                 $newdrive->{file} = $newvolid;
4846                 my $drivestr = PVE::QemuServer::print_drive($vmid, $newdrive);
4847                 $local_volumes->{$opt} = $drivestr;
4848                 #pass drive to conf for command line
4849                 $conf->{$opt} = $drivestr;
4850             }
4851         }
4852
4853         my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine);
4854
4855         my $migrate_port = 0;
4856         my $migrate_uri;
4857         if ($statefile) {
4858             if ($statefile eq 'tcp') {
4859                 my $localip = "localhost";
4860                 my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
4861                 my $nodename = PVE::INotify::nodename();
4862
4863                 if (!defined($migration_type)) {
4864                     if (defined($datacenterconf->{migration}->{type})) {
4865                         $migration_type = $datacenterconf->{migration}->{type};
4866                     } else {
4867                         $migration_type = 'secure';
4868                     }
4869                 }
4870
4871                 if ($migration_type eq 'insecure') {
4872                     my $migrate_network_addr = PVE::Cluster::get_local_migration_ip($migration_network);
4873                     if ($migrate_network_addr) {
4874                         $localip = $migrate_network_addr;
4875                     } else {
4876                         $localip = PVE::Cluster::remote_node_ip($nodename, 1);
4877                     }
4878
4879                     $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
4880                 }
4881
4882                 my $pfamily = PVE::Tools::get_host_address_family($nodename);
4883                 $migrate_port = PVE::Tools::next_migrate_port($pfamily);
4884                 $migrate_uri = "tcp:${localip}:${migrate_port}";
4885                 push @$cmd, '-incoming', $migrate_uri;
4886                 push @$cmd, '-S';
4887
4888             } elsif ($statefile eq 'unix') {
4889                 # should be default for secure migrations as a ssh TCP forward
4890                 # tunnel is not deterministic reliable ready and fails regurarly
4891                 # to set up in time, so use UNIX socket forwards
4892                 my $socket_addr = "/run/qemu-server/$vmid.migrate";
4893                 unlink $socket_addr;
4894
4895                 $migrate_uri = "unix:$socket_addr";
4896
4897                 push @$cmd, '-incoming', $migrate_uri;
4898                 push @$cmd, '-S';
4899
4900             } else {
4901                 push @$cmd, '-loadstate', $statefile;
4902             }
4903         } elsif ($paused) {
4904             push @$cmd, '-S';
4905         }
4906
4907         # host pci devices
4908         for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
4909           my $d = parse_hostpci($conf->{"hostpci$i"});
4910           next if !$d;
4911           my $pcidevices = $d->{pciid};
4912           foreach my $pcidevice (@$pcidevices) {
4913                 my $pciid = $pcidevice->{id}.".".$pcidevice->{function};
4914
4915                 my $info = pci_device_info("0000:$pciid");
4916                 die "IOMMU not present\n" if !check_iommu_support();
4917                 die "no pci device info for device '$pciid'\n" if !$info;
4918                 die "can't unbind/bind pci group to vfio '$pciid'\n" if !pci_dev_group_bind_to_vfio($pciid);
4919                 die "can't reset pci device '$pciid'\n" if $info->{has_fl_reset} and !pci_dev_reset($info);
4920           }
4921         }
4922
4923         PVE::Storage::activate_volumes($storecfg, $vollist);
4924
4925         if (!check_running($vmid, 1)) {
4926             eval {
4927                 run_command(['/bin/systemctl', 'stop', "$vmid.scope"],
4928                     outfunc => sub {}, errfunc => sub {});
4929             };
4930         }
4931
4932         my $cpuunits = defined($conf->{cpuunits}) ? $conf->{cpuunits}
4933                                                   : $defaults->{cpuunits};
4934
4935         my $start_timeout = $conf->{hugepages} ? 300 : 30;
4936         my %run_params = (timeout => $statefile ? undef : $start_timeout, umask => 0077);
4937
4938         my %properties = (
4939             Slice => 'qemu.slice',
4940             KillMode => 'none',
4941             CPUShares => $cpuunits
4942         );
4943
4944         if (my $cpulimit = $conf->{cpulimit}) {
4945             $properties{CPUQuota} = int($cpulimit * 100);
4946         }
4947         $properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
4948
4949         my $run_qemu = sub {
4950             PVE::Tools::run_fork sub {
4951                 PVE::Systemd::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
4952                 run_command($cmd, %run_params);
4953             };
4954         };
4955
4956         if ($conf->{hugepages}) {
4957
4958             my $code = sub {
4959                 my $hugepages_topology = PVE::QemuServer::Memory::hugepages_topology($conf);
4960                 my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
4961
4962                 PVE::QemuServer::Memory::hugepages_mount();
4963                 PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
4964
4965                 eval { $run_qemu->() };
4966                 if (my $err = $@) {
4967                     PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology);
4968                     die $err;
4969                 }
4970
4971                 PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology);
4972             };
4973             eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
4974
4975         } else {
4976             eval { $run_qemu->() };
4977         }
4978
4979         if (my $err = $@) {
4980             # deactivate volumes if start fails
4981             eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
4982             die "start failed: $err";
4983         }
4984
4985         print "migration listens on $migrate_uri\n" if $migrate_uri;
4986
4987         if ($statefile && $statefile ne 'tcp' && $statefile ne 'unix')  {
4988             eval { vm_mon_cmd_nocheck($vmid, "cont"); };
4989             warn $@ if $@;
4990         }
4991
4992         #start nbd server for storage migration
4993         if ($targetstorage) {
4994             my $nodename = PVE::INotify::nodename();
4995             my $migrate_network_addr = PVE::Cluster::get_local_migration_ip($migration_network);
4996             my $localip = $migrate_network_addr ? $migrate_network_addr : PVE::Cluster::remote_node_ip($nodename, 1);
4997             my $pfamily = PVE::Tools::get_host_address_family($nodename);
4998             $migrate_port = PVE::Tools::next_migrate_port($pfamily);
4999
5000             vm_mon_cmd_nocheck($vmid, "nbd-server-start", addr => { type => 'inet', data => { host => "${localip}", port => "${migrate_port}" } } );
5001
5002             $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5003
5004             foreach my $opt (sort keys %$local_volumes) {
5005                 my $volid = $local_volumes->{$opt};
5006                 vm_mon_cmd_nocheck($vmid, "nbd-server-add", device => "drive-$opt", writable => JSON::true );
5007                 my $migrate_storage_uri = "nbd:${localip}:${migrate_port}:exportname=drive-$opt";
5008                 print "storage migration listens on $migrate_storage_uri volume:$volid\n";
5009             }
5010         }
5011
5012         if ($migratedfrom) {
5013             eval {
5014                 set_migration_caps($vmid);
5015             };
5016             warn $@ if $@;
5017
5018             if ($spice_port) {
5019                 print "spice listens on port $spice_port\n";
5020                 if ($spice_ticket) {
5021                     vm_mon_cmd_nocheck($vmid, "set_password", protocol => 'spice', password => $spice_ticket);
5022                     vm_mon_cmd_nocheck($vmid, "expire_password", protocol => 'spice', time => "+30");
5023                 }
5024             }
5025
5026         } else {
5027             vm_mon_cmd_nocheck($vmid, "balloon", value => $conf->{balloon}*1024*1024)
5028                 if !$statefile && $conf->{balloon};
5029
5030             foreach my $opt (keys %$conf) {
5031                 next if $opt !~  m/^net\d+$/;
5032                 my $nicconf = parse_net($conf->{$opt});
5033                 qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
5034             }
5035         }
5036
5037         vm_mon_cmd_nocheck($vmid, 'qom-set',
5038                     path => "machine/peripheral/balloon0",
5039                     property => "guest-stats-polling-interval",
5040                     value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
5041
5042     });
5043 }
5044
5045 sub vm_mon_cmd {
5046     my ($vmid, $execute, %params) = @_;
5047
5048     my $cmd = { execute => $execute, arguments => \%params };
5049     vm_qmp_command($vmid, $cmd);
5050 }
5051
5052 sub vm_mon_cmd_nocheck {
5053     my ($vmid, $execute, %params) = @_;
5054
5055     my $cmd = { execute => $execute, arguments => \%params };
5056     vm_qmp_command($vmid, $cmd, 1);
5057 }
5058
5059 sub vm_qmp_command {
5060     my ($vmid, $cmd, $nocheck) = @_;
5061
5062     my $res;
5063
5064     my $timeout;
5065     if ($cmd->{arguments} && $cmd->{arguments}->{timeout}) {
5066         $timeout = $cmd->{arguments}->{timeout};
5067         delete $cmd->{arguments}->{timeout};
5068     }
5069
5070     eval {
5071         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
5072         my $sname = qmp_socket($vmid);
5073         if (-e $sname) { # test if VM is reasonambe new and supports qmp/qga
5074             my $qmpclient = PVE::QMPClient->new();
5075
5076             $res = $qmpclient->cmd($vmid, $cmd, $timeout);
5077         } else {
5078             die "unable to open monitor socket\n";
5079         }
5080     };
5081     if (my $err = $@) {
5082         syslog("err", "VM $vmid qmp command failed - $err");
5083         die $err;
5084     }
5085
5086     return $res;
5087 }
5088
5089 sub vm_human_monitor_command {
5090     my ($vmid, $cmdline) = @_;
5091
5092     my $res;
5093
5094     my $cmd = {
5095         execute => 'human-monitor-command',
5096         arguments => { 'command-line' => $cmdline},
5097     };
5098
5099     return vm_qmp_command($vmid, $cmd);
5100 }
5101
5102 sub vm_commandline {
5103     my ($storecfg, $vmid) = @_;
5104
5105     my $conf = PVE::QemuConfig->load_config($vmid);
5106
5107     my $defaults = load_defaults();
5108
5109     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults);
5110
5111     return PVE::Tools::cmd2string($cmd);
5112 }
5113
5114 sub vm_reset {
5115     my ($vmid, $skiplock) = @_;
5116
5117     PVE::QemuConfig->lock_config($vmid, sub {
5118
5119         my $conf = PVE::QemuConfig->load_config($vmid);
5120
5121         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5122
5123         vm_mon_cmd($vmid, "system_reset");
5124     });
5125 }
5126
5127 sub get_vm_volumes {
5128     my ($conf) = @_;
5129
5130     my $vollist = [];
5131     foreach_volid($conf, sub {
5132         my ($volid, $attr) = @_;
5133
5134         return if $volid =~ m|^/|;
5135
5136         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5137         return if !$sid;
5138
5139         push @$vollist, $volid;
5140     });
5141
5142     return $vollist;
5143 }
5144
5145 sub vm_stop_cleanup {
5146     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
5147
5148     eval {
5149
5150         if (!$keepActive) {
5151             my $vollist = get_vm_volumes($conf);
5152             PVE::Storage::deactivate_volumes($storecfg, $vollist);
5153         }
5154
5155         foreach my $ext (qw(mon qmp pid vnc qga)) {
5156             unlink "/var/run/qemu-server/${vmid}.$ext";
5157         }
5158
5159         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
5160     };
5161     warn $@ if $@; # avoid errors - just warn
5162 }
5163
5164 # Note: use $nockeck to skip tests if VM configuration file exists.
5165 # We need that when migration VMs to other nodes (files already moved)
5166 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
5167 sub vm_stop {
5168     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
5169
5170     $force = 1 if !defined($force) && !$shutdown;
5171
5172     if ($migratedfrom){
5173         my $pid = check_running($vmid, $nocheck, $migratedfrom);
5174         kill 15, $pid if $pid;
5175         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
5176         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
5177         return;
5178     }
5179
5180     PVE::QemuConfig->lock_config($vmid, sub {
5181
5182         my $pid = check_running($vmid, $nocheck);
5183         return if !$pid;
5184
5185         my $conf;
5186         if (!$nocheck) {
5187             $conf = PVE::QemuConfig->load_config($vmid);
5188             PVE::QemuConfig->check_lock($conf) if !$skiplock;
5189             if (!defined($timeout) && $shutdown && $conf->{startup}) {
5190                 my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
5191                 $timeout = $opts->{down} if $opts->{down};
5192             }
5193         }
5194
5195         $timeout = 60 if !defined($timeout);
5196
5197         eval {
5198             if ($shutdown) {
5199                 if (defined($conf) && parse_guest_agent($conf)->{enabled}) {
5200                     vm_qmp_command($vmid, { execute => "guest-shutdown" }, $nocheck);
5201                 } else {
5202                     vm_qmp_command($vmid, { execute => "system_powerdown" }, $nocheck);
5203                 }
5204             } else {
5205                 vm_qmp_command($vmid, { execute => "quit" }, $nocheck);
5206             }
5207         };
5208         my $err = $@;
5209
5210         if (!$err) {
5211             my $count = 0;
5212             while (($count < $timeout) && check_running($vmid, $nocheck)) {
5213                 $count++;
5214                 sleep 1;
5215             }
5216
5217             if ($count >= $timeout) {
5218                 if ($force) {
5219                     warn "VM still running - terminating now with SIGTERM\n";
5220                     kill 15, $pid;
5221                 } else {
5222                     die "VM quit/powerdown failed - got timeout\n";
5223                 }
5224             } else {
5225                 vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5226                 return;
5227             }
5228         } else {
5229             if ($force) {
5230                 warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
5231                 kill 15, $pid;
5232             } else {
5233                 die "VM quit/powerdown failed\n";
5234             }
5235         }
5236
5237         # wait again
5238         $timeout = 10;
5239
5240         my $count = 0;
5241         while (($count < $timeout) && check_running($vmid, $nocheck)) {
5242             $count++;
5243             sleep 1;
5244         }
5245
5246         if ($count >= $timeout) {
5247             warn "VM still running - terminating now with SIGKILL\n";
5248             kill 9, $pid;
5249             sleep 1;
5250         }
5251
5252         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5253    });
5254 }
5255
5256 sub vm_suspend {
5257     my ($vmid, $skiplock) = @_;
5258
5259     PVE::QemuConfig->lock_config($vmid, sub {
5260
5261         my $conf = PVE::QemuConfig->load_config($vmid);
5262
5263         PVE::QemuConfig->check_lock($conf)
5264             if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
5265
5266         vm_mon_cmd($vmid, "stop");
5267     });
5268 }
5269
5270 sub vm_resume {
5271     my ($vmid, $skiplock, $nocheck) = @_;
5272
5273     PVE::QemuConfig->lock_config($vmid, sub {
5274
5275         my $res = vm_mon_cmd($vmid, 'query-status');
5276         my $resume_cmd = 'cont';
5277
5278         if ($res->{status} && $res->{status} eq 'suspended') {
5279             $resume_cmd = 'system_wakeup';
5280         }
5281
5282         if (!$nocheck) {
5283
5284             my $conf = PVE::QemuConfig->load_config($vmid);
5285
5286             PVE::QemuConfig->check_lock($conf)
5287                 if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
5288
5289             vm_mon_cmd($vmid, $resume_cmd);
5290
5291         } else {
5292             vm_mon_cmd_nocheck($vmid, $resume_cmd);
5293         }
5294     });
5295 }
5296
5297 sub vm_sendkey {
5298     my ($vmid, $skiplock, $key) = @_;
5299
5300     PVE::QemuConfig->lock_config($vmid, sub {
5301
5302         my $conf = PVE::QemuConfig->load_config($vmid);
5303
5304         # there is no qmp command, so we use the human monitor command
5305         vm_human_monitor_command($vmid, "sendkey $key");
5306     });
5307 }
5308
5309 sub vm_destroy {
5310     my ($storecfg, $vmid, $skiplock) = @_;
5311
5312     PVE::QemuConfig->lock_config($vmid, sub {
5313
5314         my $conf = PVE::QemuConfig->load_config($vmid);
5315
5316         if (!check_running($vmid)) {
5317             destroy_vm($storecfg, $vmid, undef, $skiplock);
5318         } else {
5319             die "VM $vmid is running - destroy failed\n";
5320         }
5321     });
5322 }
5323
5324 # pci helpers
5325
5326 sub file_write {
5327     my ($filename, $buf) = @_;
5328
5329     my $fh = IO::File->new($filename, "w");
5330     return undef if !$fh;
5331
5332     my $res = print $fh $buf;
5333
5334     $fh->close();
5335
5336     return $res;
5337 }
5338
5339 sub pci_device_info {
5340     my ($name) = @_;
5341
5342     my $res;
5343
5344     return undef if $name !~ m/^([a-f0-9]{4}):([a-f0-9]{2}):([a-f0-9]{2})\.([a-f0-9])$/;
5345     my ($domain, $bus, $slot, $func) = ($1, $2, $3, $4);
5346
5347     my $irq = file_read_firstline("$pcisysfs/devices/$name/irq");
5348     return undef if !defined($irq) || $irq !~ m/^\d+$/;
5349
5350     my $vendor = file_read_firstline("$pcisysfs/devices/$name/vendor");
5351     return undef if !defined($vendor) || $vendor !~ s/^0x//;
5352
5353     my $product = file_read_firstline("$pcisysfs/devices/$name/device");
5354     return undef if !defined($product) || $product !~ s/^0x//;
5355
5356     $res = {
5357         name => $name,
5358         vendor => $vendor,
5359         product => $product,
5360         domain => $domain,
5361         bus => $bus,
5362         slot => $slot,
5363         func => $func,
5364         irq => $irq,
5365         has_fl_reset => -f "$pcisysfs/devices/$name/reset" || 0,
5366     };
5367
5368     return $res;
5369 }
5370
5371 sub pci_dev_reset {
5372     my ($dev) = @_;
5373
5374     my $name = $dev->{name};
5375
5376     my $fn = "$pcisysfs/devices/$name/reset";
5377
5378     return file_write($fn, "1");
5379 }
5380
5381 sub pci_dev_bind_to_vfio {
5382     my ($dev) = @_;
5383
5384     my $name = $dev->{name};
5385
5386     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
5387
5388     if (!-d $vfio_basedir) {
5389         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
5390     }
5391     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
5392
5393     my $testdir = "$vfio_basedir/$name";
5394     return 1 if -d $testdir;
5395
5396     my $data = "$dev->{vendor} $dev->{product}";
5397     return undef if !file_write("$vfio_basedir/new_id", $data);
5398
5399     my $fn = "$pcisysfs/devices/$name/driver/unbind";
5400     if (!file_write($fn, $name)) {
5401         return undef if -f $fn;
5402     }
5403
5404     $fn = "$vfio_basedir/bind";
5405     if (! -d $testdir) {
5406         return undef if !file_write($fn, $name);
5407     }
5408
5409     return -d $testdir;
5410 }
5411
5412 sub pci_dev_group_bind_to_vfio {
5413     my ($pciid) = @_;
5414
5415     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
5416
5417     if (!-d $vfio_basedir) {
5418         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
5419     }
5420     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
5421
5422     # get IOMMU group devices
5423     opendir(my $D, "$pcisysfs/devices/0000:$pciid/iommu_group/devices/") || die "Cannot open iommu_group: $!\n";
5424       my @devs = grep /^0000:/, readdir($D);
5425     closedir($D);
5426
5427     foreach my $pciid (@devs) {
5428         $pciid =~ m/^([:\.\da-f]+)$/ or die "PCI ID $pciid not valid!\n";
5429
5430         # pci bridges, switches or root ports are not supported
5431         # they have a pci_bus subdirectory so skip them
5432         next if (-e "$pcisysfs/devices/$pciid/pci_bus");
5433
5434         my $info = pci_device_info($1);
5435         pci_dev_bind_to_vfio($info) || die "Cannot bind $pciid to vfio\n";
5436     }
5437
5438     return 1;
5439 }
5440
5441 # vzdump restore implementaion
5442
5443 sub tar_archive_read_firstfile {
5444     my $archive = shift;
5445
5446     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
5447
5448     # try to detect archive type first
5449     my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
5450         die "unable to open file '$archive'\n";
5451     my $firstfile = <$fh>;
5452     kill 15, $pid;
5453     close $fh;
5454
5455     die "ERROR: archive contaions no data\n" if !$firstfile;
5456     chomp $firstfile;
5457
5458     return $firstfile;
5459 }
5460
5461 sub tar_restore_cleanup {
5462     my ($storecfg, $statfile) = @_;
5463
5464     print STDERR "starting cleanup\n";
5465
5466     if (my $fd = IO::File->new($statfile, "r")) {
5467         while (defined(my $line = <$fd>)) {
5468             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5469                 my $volid = $2;
5470                 eval {
5471                     if ($volid =~ m|^/|) {
5472                         unlink $volid || die 'unlink failed\n';
5473                     } else {
5474                         PVE::Storage::vdisk_free($storecfg, $volid);
5475                     }
5476                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
5477                 };
5478                 print STDERR "unable to cleanup '$volid' - $@" if $@;
5479             } else {
5480                 print STDERR "unable to parse line in statfile - $line";
5481             }
5482         }
5483         $fd->close();
5484     }
5485 }
5486
5487 sub restore_archive {
5488     my ($archive, $vmid, $user, $opts) = @_;
5489
5490     my $format = $opts->{format};
5491     my $comp;
5492
5493     if ($archive =~ m/\.tgz$/ || $archive =~ m/\.tar\.gz$/) {
5494         $format = 'tar' if !$format;
5495         $comp = 'gzip';
5496     } elsif ($archive =~ m/\.tar$/) {
5497         $format = 'tar' if !$format;
5498     } elsif ($archive =~ m/.tar.lzo$/) {
5499         $format = 'tar' if !$format;
5500         $comp = 'lzop';
5501     } elsif ($archive =~ m/\.vma$/) {
5502         $format = 'vma' if !$format;
5503     } elsif ($archive =~ m/\.vma\.gz$/) {
5504         $format = 'vma' if !$format;
5505         $comp = 'gzip';
5506     } elsif ($archive =~ m/\.vma\.lzo$/) {
5507         $format = 'vma' if !$format;
5508         $comp = 'lzop';
5509     } else {
5510         $format = 'vma' if !$format; # default
5511     }
5512
5513     # try to detect archive format
5514     if ($format eq 'tar') {
5515         return restore_tar_archive($archive, $vmid, $user, $opts);
5516     } else {
5517         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
5518     }
5519 }
5520
5521 sub restore_update_config_line {
5522     my ($outfd, $cookie, $vmid, $map, $line, $unique) = @_;
5523
5524     return if $line =~ m/^\#qmdump\#/;
5525     return if $line =~ m/^\#vzdump\#/;
5526     return if $line =~ m/^lock:/;
5527     return if $line =~ m/^unused\d+:/;
5528     return if $line =~ m/^parent:/;
5529     return if $line =~ m/^template:/; # restored VM is never a template
5530
5531     my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
5532     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
5533         # try to convert old 1.X settings
5534         my ($id, $ind, $ethcfg) = ($1, $2, $3);
5535         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
5536             my ($model, $macaddr) = split(/\=/, $devconfig);
5537             $macaddr = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if !$macaddr || $unique;
5538             my $net = {
5539                 model => $model,
5540                 bridge => "vmbr$ind",
5541                 macaddr => $macaddr,
5542             };
5543             my $netstr = print_net($net);
5544
5545             print $outfd "net$cookie->{netcount}: $netstr\n";
5546             $cookie->{netcount}++;
5547         }
5548     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
5549         my ($id, $netstr) = ($1, $2);
5550         my $net = parse_net($netstr);
5551         $net->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if $net->{macaddr};
5552         $netstr = print_net($net);
5553         print $outfd "$id: $netstr\n";
5554     } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk)\d+):\s*(\S+)\s*$/) {
5555         my $virtdev = $1;
5556         my $value = $3;
5557         my $di = parse_drive($virtdev, $value);
5558         if (defined($di->{backup}) && !$di->{backup}) {
5559             print $outfd "#$line";
5560         } elsif ($map->{$virtdev}) {
5561             delete $di->{format}; # format can change on restore
5562             $di->{file} = $map->{$virtdev};
5563             $value = print_drive($vmid, $di);
5564             print $outfd "$virtdev: $value\n";
5565         } else {
5566             print $outfd $line;
5567         }
5568     } elsif (($line =~ m/^vmgenid: (.*)/)) {
5569         my $vmgenid = $2;
5570         if ($vmgenid ne '0') {
5571             # always generate a new vmgenid if there was a valid one setup
5572             $vmgenid = generate_uuid();
5573         }
5574         print $outfd "vmgenid: $vmgenid\n";
5575     } elsif (($line =~ m/^(smbios1: )(.*)/) && $unique) {
5576         my ($uuid, $uuid_str);
5577         UUID::generate($uuid);
5578         UUID::unparse($uuid, $uuid_str);
5579         my $smbios1 = parse_smbios1($2);
5580         $smbios1->{uuid} = $uuid_str;
5581         print $outfd $1.print_smbios1($smbios1)."\n";
5582     } else {
5583         print $outfd $line;
5584     }
5585 }
5586
5587 sub scan_volids {
5588     my ($cfg, $vmid) = @_;
5589
5590     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid);
5591
5592     my $volid_hash = {};
5593     foreach my $storeid (keys %$info) {
5594         foreach my $item (@{$info->{$storeid}}) {
5595             next if !($item->{volid} && $item->{size});
5596             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
5597             $volid_hash->{$item->{volid}} = $item;
5598         }
5599     }
5600
5601     return $volid_hash;
5602 }
5603
5604 sub is_volume_in_use {
5605     my ($storecfg, $conf, $skip_drive, $volid) = @_;
5606
5607     my $path = PVE::Storage::path($storecfg, $volid);
5608
5609     my $scan_config = sub {
5610         my ($cref, $snapname) = @_;
5611
5612         foreach my $key (keys %$cref) {
5613             my $value = $cref->{$key};
5614             if (is_valid_drivename($key)) {
5615                 next if $skip_drive && $key eq $skip_drive;
5616                 my $drive = parse_drive($key, $value);
5617                 next if !$drive || !$drive->{file} || drive_is_cdrom($drive);
5618                 return 1 if $volid eq $drive->{file};
5619                 if ($drive->{file} =~ m!^/!) {
5620                     return 1 if $drive->{file} eq $path;
5621                 } else {
5622                     my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file}, 1);
5623                     next if !$storeid;
5624                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid, 1);
5625                     next if !$scfg;
5626                     return 1 if $path eq PVE::Storage::path($storecfg, $drive->{file}, $snapname);
5627                 }
5628             }
5629         }
5630
5631         return 0;
5632     };
5633
5634     return 1 if &$scan_config($conf);
5635
5636     undef $skip_drive;
5637
5638     foreach my $snapname (keys %{$conf->{snapshots}}) {
5639         return 1 if &$scan_config($conf->{snapshots}->{$snapname}, $snapname);
5640     }
5641
5642     return 0;
5643 }
5644
5645 sub update_disksize {
5646     my ($vmid, $conf, $volid_hash) = @_;
5647
5648     my $changes;
5649     my $prefix = "VM $vmid:";
5650
5651     # used and unused disks
5652     my $referenced = {};
5653
5654     # Note: it is allowed to define multiple storages with same path (alias), so
5655     # we need to check both 'volid' and real 'path' (two different volid can point
5656     # to the same path).
5657
5658     my $referencedpath = {};
5659
5660     # update size info
5661     foreach my $opt (keys %$conf) {
5662         if (is_valid_drivename($opt)) {
5663             my $drive = parse_drive($opt, $conf->{$opt});
5664             my $volid = $drive->{file};
5665             next if !$volid;
5666
5667             $referenced->{$volid} = 1;
5668             if ($volid_hash->{$volid} &&
5669                 (my $path = $volid_hash->{$volid}->{path})) {
5670                 $referencedpath->{$path} = 1;
5671             }
5672
5673             next if drive_is_cdrom($drive);
5674             next if !$volid_hash->{$volid};
5675
5676             $drive->{size} = $volid_hash->{$volid}->{size};
5677             my $new = print_drive($vmid, $drive);
5678             if ($new ne $conf->{$opt}) {
5679                 $changes = 1;
5680                 $conf->{$opt} = $new;
5681                 print "$prefix update disk '$opt' information.\n";
5682             }
5683         }
5684     }
5685
5686     # remove 'unusedX' entry if volume is used
5687     foreach my $opt (keys %$conf) {
5688         next if $opt !~ m/^unused\d+$/;
5689         my $volid = $conf->{$opt};
5690         my $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
5691         if ($referenced->{$volid} || ($path && $referencedpath->{$path})) {
5692             print "$prefix remove entry '$opt', its volume '$volid' is in use.\n";
5693             $changes = 1;
5694             delete $conf->{$opt};
5695         }
5696
5697         $referenced->{$volid} = 1;
5698         $referencedpath->{$path} = 1 if $path;
5699     }
5700
5701     foreach my $volid (sort keys %$volid_hash) {
5702         next if $volid =~ m/vm-$vmid-state-/;
5703         next if $referenced->{$volid};
5704         my $path = $volid_hash->{$volid}->{path};
5705         next if !$path; # just to be sure
5706         next if $referencedpath->{$path};
5707         $changes = 1;
5708         my $key = PVE::QemuConfig->add_unused_volume($conf, $volid);
5709         print "$prefix add unreferenced volume '$volid' as '$key' to config.\n";
5710         $referencedpath->{$path} = 1; # avoid to add more than once (aliases)
5711     }
5712
5713     return $changes;
5714 }
5715
5716 sub rescan {
5717     my ($vmid, $nolock, $dryrun) = @_;
5718
5719     my $cfg = PVE::Storage::config();
5720
5721     # FIXME: Remove once our RBD plugin can handle CT and VM on a single storage
5722     # see: https://pve.proxmox.com/pipermail/pve-devel/2018-July/032900.html
5723     foreach my $stor (keys %{$cfg->{ids}}) {
5724         delete($cfg->{ids}->{$stor}) if ! $cfg->{ids}->{$stor}->{content}->{images};
5725     }
5726
5727     print "rescan volumes...\n";
5728     my $volid_hash = scan_volids($cfg, $vmid);
5729
5730     my $updatefn =  sub {
5731         my ($vmid) = @_;
5732
5733         my $conf = PVE::QemuConfig->load_config($vmid);
5734
5735         PVE::QemuConfig->check_lock($conf);
5736
5737         my $vm_volids = {};
5738         foreach my $volid (keys %$volid_hash) {
5739             my $info = $volid_hash->{$volid};
5740             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
5741         }
5742
5743         my $changes = update_disksize($vmid, $conf, $vm_volids);
5744
5745         PVE::QemuConfig->write_config($vmid, $conf) if $changes && !$dryrun;
5746     };
5747
5748     if (defined($vmid)) {
5749         if ($nolock) {
5750             &$updatefn($vmid);
5751         } else {
5752             PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
5753         }
5754     } else {
5755         my $vmlist = config_list();
5756         foreach my $vmid (keys %$vmlist) {
5757             if ($nolock) {
5758                 &$updatefn($vmid);
5759             } else {
5760                 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
5761             }
5762         }
5763     }
5764 }
5765
5766 sub restore_vma_archive {
5767     my ($archive, $vmid, $user, $opts, $comp) = @_;
5768
5769     my $readfrom = $archive;
5770
5771     my $cfg = PVE::Storage::config();
5772     my $commands = [];
5773     my $bwlimit = $opts->{bwlimit};
5774
5775     my $dbg_cmdstring = '';
5776     my $add_pipe = sub {
5777         my ($cmd) = @_;
5778         push @$commands, $cmd;
5779         $dbg_cmdstring .= ' | ' if length($dbg_cmdstring);
5780         $dbg_cmdstring .= PVE::Tools::cmd2string($cmd);
5781         $readfrom = '-';
5782     };
5783
5784     my $input = undef;
5785     if ($archive eq '-') {
5786         $input = '<&STDIN';
5787     } else {
5788         # If we use a backup from a PVE defined storage we also consider that
5789         # storage's rate limit:
5790         my (undef, $volid) = PVE::Storage::path_to_volume_id($cfg, $archive);
5791         if (defined($volid)) {
5792             my ($sid, undef) = PVE::Storage::parse_volume_id($volid);
5793             my $readlimit = PVE::Storage::get_bandwidth_limit('restore', [$sid], $bwlimit);
5794             if ($readlimit) {
5795                 print STDERR "applying read rate limit: $readlimit\n";
5796                 my $cstream = ['cstream', '-t', $readlimit*1024, '--', $readfrom];
5797                 $add_pipe->($cstream);
5798             }
5799         }
5800     }
5801
5802     if ($comp) {
5803         my $cmd;
5804         if ($comp eq 'gzip') {
5805             $cmd = ['zcat', $readfrom];
5806         } elsif ($comp eq 'lzop') {
5807             $cmd = ['lzop', '-d', '-c', $readfrom];
5808         } else {
5809             die "unknown compression method '$comp'\n";
5810         }
5811         $add_pipe->($cmd);
5812     }
5813
5814     my $tmpdir = "/var/tmp/vzdumptmp$$";
5815     rmtree $tmpdir;
5816
5817     # disable interrupts (always do cleanups)
5818     local $SIG{INT} =
5819         local $SIG{TERM} =
5820         local $SIG{QUIT} =
5821         local $SIG{HUP} = sub { warn "got interrupt - ignored\n"; };
5822
5823     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
5824     POSIX::mkfifo($mapfifo, 0600);
5825     my $fifofh;
5826
5827     my $openfifo = sub {
5828         open($fifofh, '>', $mapfifo) || die $!;
5829     };
5830
5831     $add_pipe->(['vma', 'extract', '-v', '-r', $mapfifo, $readfrom, $tmpdir]);
5832
5833     my $oldtimeout;
5834     my $timeout = 5;
5835
5836     my $devinfo = {};
5837
5838     my $rpcenv = PVE::RPCEnvironment::get();
5839
5840     my $conffile = PVE::QemuConfig->config_file($vmid);
5841     my $tmpfn = "$conffile.$$.tmp";
5842
5843     # Note: $oldconf is undef if VM does not exists
5844     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
5845     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
5846
5847     my %storage_limits;
5848
5849     my $print_devmap = sub {
5850         my $virtdev_hash = {};
5851
5852         my $cfgfn = "$tmpdir/qemu-server.conf";
5853
5854         # we can read the config - that is already extracted
5855         my $fh = IO::File->new($cfgfn, "r") ||
5856             "unable to read qemu-server.conf - $!\n";
5857
5858         my $fwcfgfn = "$tmpdir/qemu-server.fw";
5859         if (-f $fwcfgfn) {
5860             my $pve_firewall_dir = '/etc/pve/firewall';
5861             mkdir $pve_firewall_dir; # make sure the dir exists
5862             PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
5863         }
5864
5865         while (defined(my $line = <$fh>)) {
5866             if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
5867                 my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
5868                 die "archive does not contain data for drive '$virtdev'\n"
5869                     if !$devinfo->{$devname};
5870                 if (defined($opts->{storage})) {
5871                     $storeid = $opts->{storage} || 'local';
5872                 } elsif (!$storeid) {
5873                     $storeid = 'local';
5874                 }
5875                 $format = 'raw' if !$format;
5876                 $devinfo->{$devname}->{devname} = $devname;
5877                 $devinfo->{$devname}->{virtdev} = $virtdev;
5878                 $devinfo->{$devname}->{format} = $format;
5879                 $devinfo->{$devname}->{storeid} = $storeid;
5880
5881                 # check permission on storage
5882                 my $pool = $opts->{pool}; # todo: do we need that?
5883                 if ($user ne 'root@pam') {
5884                     $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
5885                 }
5886
5887                 $storage_limits{$storeid} = $bwlimit;
5888
5889                 $virtdev_hash->{$virtdev} = $devinfo->{$devname};
5890             }
5891         }
5892
5893         foreach my $key (keys %storage_limits) {
5894             my $limit = PVE::Storage::get_bandwidth_limit('restore', [$key], $bwlimit);
5895             next if !$limit;
5896             print STDERR "rate limit for storage $key: $limit KiB/s\n";
5897             $storage_limits{$key} = $limit * 1024;
5898         }
5899
5900         foreach my $devname (keys %$devinfo) {
5901             die "found no device mapping information for device '$devname'\n"
5902                 if !$devinfo->{$devname}->{virtdev};
5903         }
5904
5905         # create empty/temp config
5906         if ($oldconf) {
5907             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
5908             foreach_drive($oldconf, sub {
5909                 my ($ds, $drive) = @_;
5910
5911                 return if drive_is_cdrom($drive);
5912
5913                 my $volid = $drive->{file};
5914
5915                 return if !$volid || $volid =~ m|^/|;
5916
5917                 my ($path, $owner) = PVE::Storage::path($cfg, $volid);
5918                 return if !$path || !$owner || ($owner != $vmid);
5919
5920                 # Note: only delete disk we want to restore
5921                 # other volumes will become unused
5922                 if ($virtdev_hash->{$ds}) {
5923                     eval { PVE::Storage::vdisk_free($cfg, $volid); };
5924                     if (my $err = $@) {
5925                         warn $err;
5926                     }
5927                 }
5928             });
5929
5930             # delete vmstate files
5931             # since after the restore we have no snapshots anymore
5932             foreach my $snapname (keys %{$oldconf->{snapshots}}) {
5933                 my $snap = $oldconf->{snapshots}->{$snapname};
5934                 if ($snap->{vmstate}) {
5935                     eval { PVE::Storage::vdisk_free($cfg, $snap->{vmstate}); };
5936                     if (my $err = $@) {
5937                         warn $err;
5938                     }
5939                 }
5940             }
5941         }
5942
5943         my $map = {};
5944         foreach my $virtdev (sort keys %$virtdev_hash) {
5945             my $d = $virtdev_hash->{$virtdev};
5946             my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
5947             my $storeid = $d->{storeid};
5948             my $scfg = PVE::Storage::storage_config($cfg, $storeid);
5949
5950             my $map_opts = '';
5951             if (my $limit = $storage_limits{$storeid}) {
5952                 $map_opts .= "throttling.bps=$limit:throttling.group=$storeid:";
5953             }
5954
5955             # test if requested format is supported
5956             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($cfg, $storeid);
5957             my $supported = grep { $_ eq $d->{format} } @$validFormats;
5958             $d->{format} = $defFormat if !$supported;
5959
5960             my $volid = PVE::Storage::vdisk_alloc($cfg, $storeid, $vmid,
5961                                                   $d->{format}, undef, $alloc_size);
5962             print STDERR "new volume ID is '$volid'\n";
5963             $d->{volid} = $volid;
5964             my $path = PVE::Storage::path($cfg, $volid);
5965
5966             PVE::Storage::activate_volumes($cfg,[$volid]);
5967
5968             my $write_zeros = 1;
5969             if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
5970                 $write_zeros = 0;
5971             }
5972
5973             print $fifofh "${map_opts}format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
5974
5975             print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
5976             $map->{$virtdev} = $volid;
5977         }
5978
5979         $fh->seek(0, 0) || die "seek failed - $!\n";
5980
5981         my $outfd = new IO::File ($tmpfn, "w") ||
5982             die "unable to write config for VM $vmid\n";
5983
5984         my $cookie = { netcount => 0 };
5985         while (defined(my $line = <$fh>)) {
5986             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
5987         }
5988
5989         $fh->close();
5990         $outfd->close();
5991     };
5992
5993     eval {
5994         # enable interrupts
5995         local $SIG{INT} =
5996             local $SIG{TERM} =
5997             local $SIG{QUIT} =
5998             local $SIG{HUP} =
5999             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6000         local $SIG{ALRM} = sub { die "got timeout\n"; };
6001
6002         $oldtimeout = alarm($timeout);
6003
6004         my $parser = sub {
6005             my $line = shift;
6006
6007             print "$line\n";
6008
6009             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
6010                 my ($dev_id, $size, $devname) = ($1, $2, $3);
6011                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
6012             } elsif ($line =~ m/^CTIME: /) {
6013                 # we correctly received the vma config, so we can disable
6014                 # the timeout now for disk allocation (set to 10 minutes, so
6015                 # that we always timeout if something goes wrong)
6016                 alarm(600);
6017                 &$print_devmap();
6018                 print $fifofh "done\n";
6019                 my $tmp = $oldtimeout || 0;
6020                 $oldtimeout = undef;
6021                 alarm($tmp);
6022                 close($fifofh);
6023             }
6024         };
6025
6026         print "restore vma archive: $dbg_cmdstring\n";
6027         run_command($commands, input => $input, outfunc => $parser, afterfork => $openfifo);
6028     };
6029     my $err = $@;
6030
6031     alarm($oldtimeout) if $oldtimeout;
6032
6033     my $vollist = [];
6034     foreach my $devname (keys %$devinfo) {
6035         my $volid = $devinfo->{$devname}->{volid};
6036         push @$vollist, $volid if $volid;
6037     }
6038
6039     PVE::Storage::deactivate_volumes($cfg, $vollist);
6040
6041     unlink $mapfifo;
6042
6043     if ($err) {
6044         rmtree $tmpdir;
6045         unlink $tmpfn;
6046
6047         foreach my $devname (keys %$devinfo) {
6048             my $volid = $devinfo->{$devname}->{volid};
6049             next if !$volid;
6050             eval {
6051                 if ($volid =~ m|^/|) {
6052                     unlink $volid || die 'unlink failed\n';
6053                 } else {
6054                     PVE::Storage::vdisk_free($cfg, $volid);
6055                 }
6056                 print STDERR "temporary volume '$volid' sucessfuly removed\n";
6057             };
6058             print STDERR "unable to cleanup '$volid' - $@" if $@;
6059         }
6060         die $err;
6061     }
6062
6063     rmtree $tmpdir;
6064
6065     rename($tmpfn, $conffile) ||
6066         die "unable to commit configuration file '$conffile'\n";
6067
6068     PVE::Cluster::cfs_update(); # make sure we read new file
6069
6070     eval { rescan($vmid, 1); };
6071     warn $@ if $@;
6072 }
6073
6074 sub restore_tar_archive {
6075     my ($archive, $vmid, $user, $opts) = @_;
6076
6077     if ($archive ne '-') {
6078         my $firstfile = tar_archive_read_firstfile($archive);
6079         die "ERROR: file '$archive' dos not lock like a QemuServer vzdump backup\n"
6080             if $firstfile ne 'qemu-server.conf';
6081     }
6082
6083     my $storecfg = PVE::Storage::config();
6084
6085     # destroy existing data - keep empty config
6086     my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
6087     destroy_vm($storecfg, $vmid, 1) if -f $vmcfgfn;
6088
6089     my $tocmd = "/usr/lib/qemu-server/qmextract";
6090
6091     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
6092     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
6093     $tocmd .= ' --prealloc' if $opts->{prealloc};
6094     $tocmd .= ' --info' if $opts->{info};
6095
6096     # tar option "xf" does not autodetect compression when read from STDIN,
6097     # so we pipe to zcat
6098     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
6099         PVE::Tools::shellquote("--to-command=$tocmd");
6100
6101     my $tmpdir = "/var/tmp/vzdumptmp$$";
6102     mkpath $tmpdir;
6103
6104     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
6105     local $ENV{VZDUMP_VMID} = $vmid;
6106     local $ENV{VZDUMP_USER} = $user;
6107
6108     my $conffile = PVE::QemuConfig->config_file($vmid);
6109     my $tmpfn = "$conffile.$$.tmp";
6110
6111     # disable interrupts (always do cleanups)
6112     local $SIG{INT} =
6113         local $SIG{TERM} =
6114         local $SIG{QUIT} =
6115         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
6116
6117     eval {
6118         # enable interrupts
6119         local $SIG{INT} =
6120             local $SIG{TERM} =
6121             local $SIG{QUIT} =
6122             local $SIG{HUP} =
6123             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6124
6125         if ($archive eq '-') {
6126             print "extracting archive from STDIN\n";
6127             run_command($cmd, input => "<&STDIN");
6128         } else {
6129             print "extracting archive '$archive'\n";
6130             run_command($cmd);
6131         }
6132
6133         return if $opts->{info};
6134
6135         # read new mapping
6136         my $map = {};
6137         my $statfile = "$tmpdir/qmrestore.stat";
6138         if (my $fd = IO::File->new($statfile, "r")) {
6139             while (defined (my $line = <$fd>)) {
6140                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
6141                     $map->{$1} = $2 if $1;
6142                 } else {
6143                     print STDERR "unable to parse line in statfile - $line\n";
6144                 }
6145             }
6146             $fd->close();
6147         }
6148
6149         my $confsrc = "$tmpdir/qemu-server.conf";
6150
6151         my $srcfd = new IO::File($confsrc, "r") ||
6152             die "unable to open file '$confsrc'\n";
6153
6154         my $outfd = new IO::File ($tmpfn, "w") ||
6155             die "unable to write config for VM $vmid\n";
6156
6157         my $cookie = { netcount => 0 };
6158         while (defined (my $line = <$srcfd>)) {
6159             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
6160         }
6161
6162         $srcfd->close();
6163         $outfd->close();
6164     };
6165     my $err = $@;
6166
6167     if ($err) {
6168
6169         unlink $tmpfn;
6170
6171         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
6172
6173         die $err;
6174     }
6175
6176     rmtree $tmpdir;
6177
6178     rename $tmpfn, $conffile ||
6179         die "unable to commit configuration file '$conffile'\n";
6180
6181     PVE::Cluster::cfs_update(); # make sure we read new file
6182
6183     eval { rescan($vmid, 1); };
6184     warn $@ if $@;
6185 };
6186
6187 sub foreach_storage_used_by_vm {
6188     my ($conf, $func) = @_;
6189
6190     my $sidhash = {};
6191
6192     foreach_drive($conf, sub {
6193         my ($ds, $drive) = @_;
6194         return if drive_is_cdrom($drive);
6195
6196         my $volid = $drive->{file};
6197
6198         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
6199         $sidhash->{$sid} = $sid if $sid;
6200     });
6201
6202     foreach my $sid (sort keys %$sidhash) {
6203         &$func($sid);
6204     }
6205 }
6206
6207 sub do_snapshots_with_qemu {
6208     my ($storecfg, $volid) = @_;
6209
6210     my $storage_name = PVE::Storage::parse_volume_id($volid);
6211
6212     if ($qemu_snap_storage->{$storecfg->{ids}->{$storage_name}->{type}}
6213         && !$storecfg->{ids}->{$storage_name}->{krbd}){
6214         return 1;
6215     }
6216
6217     if ($volid =~ m/\.(qcow2|qed)$/){
6218         return 1;
6219     }
6220
6221     return undef;
6222 }
6223
6224 sub qga_check_running {
6225     my ($vmid, $nowarn) = @_;
6226
6227     eval { vm_mon_cmd($vmid, "guest-ping", timeout => 3); };
6228     if ($@) {
6229         warn "Qemu Guest Agent is not running - $@" if !$nowarn;
6230         return 0;
6231     }
6232     return 1;
6233 }
6234
6235 sub template_create {
6236     my ($vmid, $conf, $disk) = @_;
6237
6238     my $storecfg = PVE::Storage::config();
6239
6240     foreach_drive($conf, sub {
6241         my ($ds, $drive) = @_;
6242
6243         return if drive_is_cdrom($drive);
6244         return if $disk && $ds ne $disk;
6245
6246         my $volid = $drive->{file};
6247         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
6248
6249         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
6250         $drive->{file} = $voliddst;
6251         $conf->{$ds} = print_drive($vmid, $drive);
6252         PVE::QemuConfig->write_config($vmid, $conf);
6253     });
6254 }
6255
6256 sub qemu_img_convert {
6257     my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized) = @_;
6258
6259     my $storecfg = PVE::Storage::config();
6260     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
6261     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
6262
6263     if ($src_storeid && $dst_storeid) {
6264
6265         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
6266
6267         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
6268         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6269
6270         my $src_format = qemu_img_format($src_scfg, $src_volname);
6271         my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
6272
6273         my $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
6274         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6275
6276         my $cmd = [];
6277         push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
6278         push @$cmd, '-s', $snapname if($snapname && $src_format eq "qcow2");
6279         push @$cmd, '-t', 'none' if $dst_scfg->{type} eq 'zfspool';
6280         push @$cmd, '-T', 'none' if $src_scfg->{type} eq 'zfspool';
6281         push @$cmd, '-f', $src_format, '-O', $dst_format, $src_path;
6282         if ($is_zero_initialized) {
6283             push @$cmd, "zeroinit:$dst_path";
6284         } else {
6285             push @$cmd, $dst_path;
6286         }
6287
6288         my $parser = sub {
6289             my $line = shift;
6290             if($line =~ m/\((\S+)\/100\%\)/){
6291                 my $percent = $1;
6292                 my $transferred = int($size * $percent / 100);
6293                 my $remaining = $size - $transferred;
6294
6295                 print "transferred: $transferred bytes remaining: $remaining bytes total: $size bytes progression: $percent %\n";
6296             }
6297
6298         };
6299
6300         eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
6301         my $err = $@;
6302         die "copy failed: $err" if $err;
6303     }
6304 }
6305
6306 sub qemu_img_format {
6307     my ($scfg, $volname) = @_;
6308
6309     if ($scfg->{path} && $volname =~ m/\.($QEMU_FORMAT_RE)$/) {
6310         return $1;
6311     } else {
6312         return "raw";
6313     }
6314 }
6315
6316 sub qemu_drive_mirror {
6317     my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $skipcomplete, $qga) = @_;
6318
6319     $jobs = {} if !$jobs;
6320
6321     my $qemu_target;
6322     my $format;
6323     $jobs->{"drive-$drive"} = {};
6324
6325     if ($dst_volid =~ /^nbd:/) {
6326         $qemu_target = $dst_volid;
6327         $format = "nbd";
6328     } else {
6329         my $storecfg = PVE::Storage::config();
6330         my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
6331
6332         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6333
6334         $format = qemu_img_format($dst_scfg, $dst_volname);
6335
6336         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6337
6338         $qemu_target = $is_zero_initialized ? "zeroinit:$dst_path" : $dst_path;
6339     }
6340
6341     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $qemu_target };
6342     $opts->{format} = $format if $format;
6343
6344     print "drive mirror is starting for drive-$drive\n";
6345
6346     eval { vm_mon_cmd($vmid, "drive-mirror", %$opts); }; #if a job already run for this device,it's throw an error
6347
6348     if (my $err = $@) {
6349         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6350         die "mirroring error: $err";
6351     }
6352
6353     qemu_drive_mirror_monitor ($vmid, $vmiddst, $jobs, $skipcomplete, $qga);
6354 }
6355
6356 sub qemu_drive_mirror_monitor {
6357     my ($vmid, $vmiddst, $jobs, $skipcomplete, $qga) = @_;
6358
6359     eval {
6360         my $err_complete = 0;
6361
6362         while (1) {
6363             die "storage migration timed out\n" if $err_complete > 300;
6364
6365             my $stats = vm_mon_cmd($vmid, "query-block-jobs");
6366
6367             my $running_mirror_jobs = {};
6368             foreach my $stat (@$stats) {
6369                 next if $stat->{type} ne 'mirror';
6370                 $running_mirror_jobs->{$stat->{device}} = $stat;
6371             }
6372
6373             my $readycounter = 0;
6374
6375             foreach my $job (keys %$jobs) {
6376
6377                 if(defined($jobs->{$job}->{complete}) && !defined($running_mirror_jobs->{$job})) {
6378                     print "$job : finished\n";
6379                     delete $jobs->{$job};
6380                     next;
6381                 }
6382
6383                 die "$job: mirroring has been cancelled\n" if !defined($running_mirror_jobs->{$job});
6384
6385                 my $busy = $running_mirror_jobs->{$job}->{busy};
6386                 my $ready = $running_mirror_jobs->{$job}->{ready};
6387                 if (my $total = $running_mirror_jobs->{$job}->{len}) {
6388                     my $transferred = $running_mirror_jobs->{$job}->{offset} || 0;
6389                     my $remaining = $total - $transferred;
6390                     my $percent = sprintf "%.2f", ($transferred * 100 / $total);
6391
6392                     print "$job: transferred: $transferred bytes remaining: $remaining bytes total: $total bytes progression: $percent % busy: $busy ready: $ready \n";
6393                 }
6394
6395                 $readycounter++ if $running_mirror_jobs->{$job}->{ready};
6396             }
6397
6398             last if scalar(keys %$jobs) == 0;
6399
6400             if ($readycounter == scalar(keys %$jobs)) {
6401                 print "all mirroring jobs are ready \n";
6402                 last if $skipcomplete; #do the complete later
6403
6404                 if ($vmiddst && $vmiddst != $vmid) {
6405                     my $agent_running = $qga && qga_check_running($vmid);
6406                     if ($agent_running) {
6407                         print "freeze filesystem\n";
6408                         eval { PVE::QemuServer::vm_mon_cmd($vmid, "guest-fsfreeze-freeze"); };
6409                     } else {
6410                         print "suspend vm\n";
6411                         eval { PVE::QemuServer::vm_suspend($vmid, 1); };
6412                     }
6413
6414                     # if we clone a disk for a new target vm, we don't switch the disk
6415                     PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs);
6416
6417                     if ($agent_running) {
6418                         print "unfreeze filesystem\n";
6419                         eval { PVE::QemuServer::vm_mon_cmd($vmid, "guest-fsfreeze-thaw"); };
6420                     } else {
6421                         print "resume vm\n";
6422                         eval {  PVE::QemuServer::vm_resume($vmid, 1, 1); };
6423                     }
6424
6425                     last;
6426                 } else {
6427
6428                     foreach my $job (keys %$jobs) {
6429                         # try to switch the disk if source and destination are on the same guest
6430                         print "$job: Completing block job...\n";
6431
6432                         eval { vm_mon_cmd($vmid, "block-job-complete", device => $job) };
6433                         if ($@ =~ m/cannot be completed/) {
6434                             print "$job: Block job cannot be completed, try again.\n";
6435                             $err_complete++;
6436                         }else {
6437                             print "$job: Completed successfully.\n";
6438                             $jobs->{$job}->{complete} = 1;
6439                         }
6440                     }
6441                 }
6442             }
6443             sleep 1;
6444         }
6445     };
6446     my $err = $@;
6447
6448     if ($err) {
6449         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6450         die "mirroring error: $err";
6451     }
6452
6453 }
6454
6455 sub qemu_blockjobs_cancel {
6456     my ($vmid, $jobs) = @_;
6457
6458     foreach my $job (keys %$jobs) {
6459         print "$job: Cancelling block job\n";
6460         eval { vm_mon_cmd($vmid, "block-job-cancel", device => $job); };
6461         $jobs->{$job}->{cancel} = 1;
6462     }
6463
6464     while (1) {
6465         my $stats = vm_mon_cmd($vmid, "query-block-jobs");
6466
6467         my $running_jobs = {};
6468         foreach my $stat (@$stats) {
6469             $running_jobs->{$stat->{device}} = $stat;
6470         }
6471
6472         foreach my $job (keys %$jobs) {
6473
6474             if (defined($jobs->{$job}->{cancel}) && !defined($running_jobs->{$job})) {
6475                 print "$job: Done.\n";
6476                 delete $jobs->{$job};
6477             }
6478         }
6479
6480         last if scalar(keys %$jobs) == 0;
6481
6482         sleep 1;
6483     }
6484 }
6485
6486 sub clone_disk {
6487     my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
6488         $newvmid, $storage, $format, $full, $newvollist, $jobs, $skipcomplete, $qga) = @_;
6489
6490     my $newvolid;
6491
6492     if (!$full) {
6493         print "create linked clone of drive $drivename ($drive->{file})\n";
6494         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
6495         push @$newvollist, $newvolid;
6496     } else {
6497
6498         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6499         $storeid = $storage if $storage;
6500
6501         my $dst_format = resolve_dst_disk_format($storecfg, $storeid, $volname, $format);
6502         my ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 3);
6503
6504         print "create full clone of drive $drivename ($drive->{file})\n";
6505         my $name = undef;
6506         if (drive_is_cloudinit($drive)) {
6507             $name = "vm-$newvmid-cloudinit";
6508             # cloudinit only supports raw and qcow2 atm:
6509             if ($dst_format eq 'qcow2') {
6510                 $name .= '.qcow2';
6511             } elsif ($dst_format ne 'raw') {
6512                 die "clone: unhandled format for cloudinit image\n";
6513             }
6514         }
6515         $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $newvmid, $dst_format, $name, ($size/1024));
6516         push @$newvollist, $newvolid;
6517
6518         PVE::Storage::activate_volumes($storecfg, [$newvolid]);
6519
6520         my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
6521         if (!$running || $snapname) {
6522             qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit);
6523         } else {
6524
6525             my $kvmver = get_running_qemu_version ($vmid);
6526             if (!qemu_machine_feature_enabled (undef, $kvmver, 2, 7)) {
6527                 die "drive-mirror with iothread requires qemu version 2.7 or higher\n"
6528                     if $drive->{iothread};
6529             }
6530
6531             qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid, $sparseinit, $jobs, $skipcomplete, $qga);
6532         }
6533     }
6534
6535     my ($size) = PVE::Storage::volume_size_info($storecfg, $newvolid, 3);
6536
6537     my $disk = $drive;
6538     $disk->{format} = undef;
6539     $disk->{file} = $newvolid;
6540     $disk->{size} = $size;
6541
6542     return $disk;
6543 }
6544
6545 # this only works if VM is running
6546 sub get_current_qemu_machine {
6547     my ($vmid) = @_;
6548
6549     my $cmd = { execute => 'query-machines', arguments => {} };
6550     my $res = vm_qmp_command($vmid, $cmd);
6551
6552     my ($current, $default);
6553     foreach my $e (@$res) {
6554         $default = $e->{name} if $e->{'is-default'};
6555         $current = $e->{name} if $e->{'is-current'};
6556     }
6557
6558     # fallback to the default machine if current is not supported by qemu
6559     return $current || $default || 'pc';
6560 }
6561
6562 sub get_running_qemu_version {
6563     my ($vmid) = @_;
6564     my $cmd = { execute => 'query-version', arguments => {} };
6565     my $res = vm_qmp_command($vmid, $cmd);
6566     return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
6567 }
6568
6569 sub qemu_machine_feature_enabled {
6570     my ($machine, $kvmver, $version_major, $version_minor) = @_;
6571
6572     my $current_major;
6573     my $current_minor;
6574
6575     if ($machine && $machine =~ m/^(pc(-i440fx|-q35)?-(\d+)\.(\d+))/) {
6576
6577         $current_major = $3;
6578         $current_minor = $4;
6579
6580     } elsif ($kvmver =~ m/^(\d+)\.(\d+)/) {
6581
6582         $current_major = $1;
6583         $current_minor = $2;
6584     }
6585
6586     return 1 if $current_major > $version_major ||
6587                 ($current_major == $version_major &&
6588                  $current_minor >= $version_minor);
6589 }
6590
6591 sub qemu_machine_pxe {
6592     my ($vmid, $conf, $machine) = @_;
6593
6594     $machine =  PVE::QemuServer::get_current_qemu_machine($vmid) if !$machine;
6595
6596     if ($conf->{machine} && $conf->{machine} =~ m/\.pxe$/) {
6597         $machine .= '.pxe';
6598     }
6599
6600     return $machine;
6601 }
6602
6603 sub qemu_use_old_bios_files {
6604     my ($machine_type) = @_;
6605
6606     return if !$machine_type;
6607
6608     my $use_old_bios_files = undef;
6609
6610     if ($machine_type =~ m/^(\S+)\.pxe$/) {
6611         $machine_type = $1;
6612         $use_old_bios_files = 1;
6613     } else {
6614         my $kvmver = kvm_user_version();
6615         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
6616         # load new efi bios files on migration. So this hack is required to allow
6617         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
6618         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
6619         $use_old_bios_files = !qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 4);
6620     }
6621
6622     return ($use_old_bios_files, $machine_type);
6623 }
6624
6625 sub create_efidisk {
6626     my ($storecfg, $storeid, $vmid, $fmt) = @_;
6627
6628     die "EFI vars default image not found\n" if ! -f $OVMF_VARS;
6629
6630     my $vars_size = PVE::Tools::convert_size(-s $OVMF_VARS, 'b' => 'kb');
6631     my $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $fmt, undef, $vars_size);
6632     PVE::Storage::activate_volumes($storecfg, [$volid]);
6633
6634     my $path = PVE::Storage::path($storecfg, $volid);
6635     eval {
6636         run_command(['/usr/bin/qemu-img', 'convert', '-n', '-f', 'raw', '-O', $fmt, $OVMF_VARS, $path]);
6637     };
6638     die "Copying EFI vars image failed: $@" if $@;
6639
6640     return ($volid, $vars_size);
6641 }
6642
6643 sub lspci {
6644
6645     my $devices = {};
6646
6647     dir_glob_foreach("$pcisysfs/devices", '[a-f0-9]{4}:([a-f0-9]{2}:[a-f0-9]{2})\.([0-9])', sub {
6648             my (undef, $id, $function) = @_;
6649             my $res = { id => $id, function => $function};
6650             push @{$devices->{$id}}, $res;
6651     });
6652
6653     # Entries should be sorted by functions.
6654     foreach my $id (keys %$devices) {
6655         my $dev = $devices->{$id};
6656         $devices->{$id} = [ sort { $a->{function} <=> $b->{function} } @$dev ];
6657     }
6658
6659     return $devices;
6660 }
6661
6662 sub vm_iothreads_list {
6663     my ($vmid) = @_;
6664
6665     my $res = vm_mon_cmd($vmid, 'query-iothreads');
6666
6667     my $iothreads = {};
6668     foreach my $iothread (@$res) {
6669         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
6670     }
6671
6672     return $iothreads;
6673 }
6674
6675 sub scsihw_infos {
6676     my ($conf, $drive) = @_;
6677
6678     my $maxdev = 0;
6679
6680     if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)) {
6681         $maxdev = 7;
6682     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
6683         $maxdev = 1;
6684     } else {
6685         $maxdev = 256;
6686     }
6687
6688     my $controller = int($drive->{index} / $maxdev);
6689     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single') ? "virtioscsi" : "scsihw";
6690
6691     return ($maxdev, $controller, $controller_prefix);
6692 }
6693
6694 sub add_hyperv_enlightenments {
6695     my ($cpuFlags, $winversion, $machine_type, $kvmver, $bios, $gpu_passthrough) = @_;
6696
6697     return if $winversion < 6;
6698     return if $bios && $bios eq 'ovmf' && $winversion < 8;
6699
6700     push @$cpuFlags , 'hv_vendor_id=proxmox' if $gpu_passthrough;
6701
6702     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
6703         push @$cpuFlags , 'hv_spinlocks=0x1fff';
6704         push @$cpuFlags , 'hv_vapic';
6705         push @$cpuFlags , 'hv_time';
6706     } else {
6707         push @$cpuFlags , 'hv_spinlocks=0xffff';
6708     }
6709
6710     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 6)) {
6711         push @$cpuFlags , 'hv_reset';
6712         push @$cpuFlags , 'hv_vpindex';
6713         push @$cpuFlags , 'hv_runtime';
6714     }
6715
6716     if ($winversion >= 7) {
6717         push @$cpuFlags , 'hv_relaxed';
6718
6719         if (qemu_machine_feature_enabled ($machine_type, $kvmver, 3, 0)) {
6720             push @$cpuFlags , 'hv_synic';
6721             push @$cpuFlags , 'hv_stimer';
6722         }
6723     }
6724 }
6725
6726 sub windows_version {
6727     my ($ostype) = @_;
6728
6729     return 0 if !$ostype;
6730
6731     my $winversion = 0;
6732
6733     if($ostype eq 'wxp' || $ostype eq 'w2k3' || $ostype eq 'w2k') {
6734         $winversion = 5;
6735     } elsif($ostype eq 'w2k8' || $ostype eq 'wvista') {
6736         $winversion = 6;
6737     } elsif ($ostype =~ m/^win(\d+)$/) {
6738         $winversion = $1;
6739     }
6740
6741     return $winversion;
6742 }
6743
6744 sub resolve_dst_disk_format {
6745         my ($storecfg, $storeid, $src_volname, $format) = @_;
6746         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
6747
6748         if (!$format) {
6749             # if no target format is specified, use the source disk format as hint
6750             if ($src_volname) {
6751                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6752                 $format = qemu_img_format($scfg, $src_volname);
6753             } else {
6754                 return $defFormat;
6755             }
6756         }
6757
6758         # test if requested format is supported - else use default
6759         my $supported = grep { $_ eq $format } @$validFormats;
6760         $format = $defFormat if !$supported;
6761         return $format;
6762 }
6763
6764 sub resolve_first_disk {
6765     my $conf = shift;
6766     my @disks = PVE::QemuServer::valid_drive_names();
6767     my $firstdisk;
6768     foreach my $ds (reverse @disks) {
6769         next if !$conf->{$ds};
6770         my $disk = PVE::QemuServer::parse_drive($ds, $conf->{$ds});
6771         next if PVE::QemuServer::drive_is_cdrom($disk);
6772         $firstdisk = $ds;
6773     }
6774     return $firstdisk;
6775 }
6776
6777 sub generate_uuid {
6778     my ($uuid, $uuid_str);
6779     UUID::generate($uuid);
6780     UUID::unparse($uuid, $uuid_str);
6781     return $uuid_str;
6782 }
6783
6784 sub generate_smbios1_uuid {
6785     return "uuid=".generate_uuid();
6786 }
6787
6788 # bash completion helper
6789
6790 sub complete_backup_archives {
6791     my ($cmdname, $pname, $cvalue) = @_;
6792
6793     my $cfg = PVE::Storage::config();
6794
6795     my $storeid;
6796
6797     if ($cvalue =~ m/^([^:]+):/) {
6798         $storeid = $1;
6799     }
6800
6801     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
6802
6803     my $res = [];
6804     foreach my $id (keys %$data) {
6805         foreach my $item (@{$data->{$id}}) {
6806             next if $item->{format} !~ m/^vma\.(gz|lzo)$/;
6807             push @$res, $item->{volid} if defined($item->{volid});
6808         }
6809     }
6810
6811     return $res;
6812 }
6813
6814 my $complete_vmid_full = sub {
6815     my ($running) = @_;
6816
6817     my $idlist = vmstatus();
6818
6819     my $res = [];
6820
6821     foreach my $id (keys %$idlist) {
6822         my $d = $idlist->{$id};
6823         if (defined($running)) {
6824             next if $d->{template};
6825             next if $running && $d->{status} ne 'running';
6826             next if !$running && $d->{status} eq 'running';
6827         }
6828         push @$res, $id;
6829
6830     }
6831     return $res;
6832 };
6833
6834 sub complete_vmid {
6835     return &$complete_vmid_full();
6836 }
6837
6838 sub complete_vmid_stopped {
6839     return &$complete_vmid_full(0);
6840 }
6841
6842 sub complete_vmid_running {
6843     return &$complete_vmid_full(1);
6844 }
6845
6846 sub complete_storage {
6847
6848     my $cfg = PVE::Storage::config();
6849     my $ids = $cfg->{ids};
6850
6851     my $res = [];
6852     foreach my $sid (keys %$ids) {
6853         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
6854         next if !$ids->{$sid}->{content}->{images};
6855         push @$res, $sid;
6856     }
6857
6858     return $res;
6859 }
6860
6861 sub nbd_stop {
6862     my ($vmid) = @_;
6863
6864     vm_mon_cmd($vmid, 'nbd-server-stop');
6865 }
6866
6867 1;