PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5 use POSIX;
   6 use IO::Handle;
   7 use IO::Select;
   8 use IO::File;
   9 use IO::Dir;
  10 use IO::Socket::UNIX;
  11 use File::Basename;
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use Digest::SHA;
  16 use Fcntl ':flock';
  17 use Cwd 'abs_path';
  18 use IPC::Open3;
  19 use JSON;
  20 use Fcntl;
  21 use PVE::SafeSyslog;
  22 use Storable qw(dclone);
  23 use PVE::Exception qw(raise raise_param_exc);
  24 use PVE::Storage;
  25 use PVE::Tools qw(run_command lock_file lock_file_full file_read_firstline dir_glob_foreach $IPV6RE);
  26 use PVE::JSONSchema qw(get_standard_option);
  27 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file cfs_lock_file);
  28 use PVE::INotify;
  29 use PVE::ProcFSTools;
  30 use PVE::QemuConfig;
  31 use PVE::QMPClient;
  32 use PVE::RPCEnvironment;
  33 use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr);
  34 use PVE::QemuServer::Memory;
  35 use PVE::QemuServer::USB qw(parse_usb_device);
  36 use PVE::QemuServer::Cloudinit;
  37 use PVE::Systemd;
  38 use Time::HiRes qw(gettimeofday);
  39 use File::Copy qw(copy);
  40 use URI::Escape;
  41
  42 my $EDK2_FW_BASE = '/usr/share/pve-edk2-firmware/';
  43 my $OVMF_CODE = "$EDK2_FW_BASE/OVMF_CODE.fd";
  44 my $OVMF_VARS = "$EDK2_FW_BASE/OVMF_VARS.fd";
  45
  46 my $qemu_snap_storage = {rbd => 1, sheepdog => 1};
  47
  48 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  49
  50 my $QEMU_FORMAT_RE = qr/raw|cow|qcow|qcow2|qed|vmdk|cloop/;
  51
  52 # Note about locking: we use flock on the config file protect
  53 # against concurent actions.
  54 # Aditionaly, we have a 'lock' setting in the config file. This
  55 # can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
  56 # allowed when such lock is set. But you can ignore this kind of
  57 # lock with the --skiplock flag.
  58
  59 cfs_register_file('/qemu-server/',
  60                   \&parse_vm_config,
  61                   \&write_vm_config);
  62
  63 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
  64     description => "Some command save/restore state from this location.",
  65     type => 'string',
  66     maxLength => 128,
  67     optional => 1,
  68 });
  69
  70 PVE::JSONSchema::register_standard_option('pve-snapshot-name', {
  71     description => "The name of the snapshot.",
  72     type => 'string', format => 'pve-configid',
  73     maxLength => 40,
  74 });
  75
  76 PVE::JSONSchema::register_standard_option('pve-qm-image-format', {
  77     type => 'string',
  78     enum => [qw(raw cow qcow qed qcow2 vmdk cloop)],
  79     description => "The drive's backing file's data format.",
  80     optional => 1,
  81 });
  82
  83 PVE::JSONSchema::register_standard_option('pve-qemu-machine', {
  84         description => "Specifies the Qemu machine type.",
  85         type => 'string',
  86         pattern => '(pc|pc(-i440fx)?-\d+\.\d+(\.pxe)?|q35|pc-q35-\d+\.\d+(\.pxe)?)',
  87         maxLength => 40,
  88         optional => 1,
  89 });
  90
  91 #no warnings 'redefine';
  92
  93 sub cgroups_write {
  94    my ($controller, $vmid, $option, $value) = @_;
  95
  96    my $path = "/sys/fs/cgroup/$controller/qemu.slice/$vmid.scope/$option";
  97    PVE::ProcFSTools::write_proc_entry($path, $value);
  98
  99 }
 100
 101 my $nodename = PVE::INotify::nodename();
 102
 103 mkdir "/etc/pve/nodes/$nodename";
 104 my $confdir = "/etc/pve/nodes/$nodename/qemu-server";
 105 mkdir $confdir;
 106
 107 my $var_run_tmpdir = "/var/run/qemu-server";
 108 mkdir $var_run_tmpdir;
 109
 110 my $lock_dir = "/var/lock/qemu-server";
 111 mkdir $lock_dir;
 112
 113 my $pcisysfs = "/sys/bus/pci";
 114
 115 my $cpu_vendor_list = {
 116     # Intel CPUs
 117     486 => 'GenuineIntel',
 118     pentium => 'GenuineIntel',
 119     pentium2  => 'GenuineIntel',
 120     pentium3  => 'GenuineIntel',
 121     coreduo => 'GenuineIntel',
 122     core2duo => 'GenuineIntel',
 123     Conroe  => 'GenuineIntel',
 124     Penryn  => 'GenuineIntel',
 125     Nehalem  => 'GenuineIntel',
 126     'Nehalem-IBRS'  => 'GenuineIntel',
 127     Westmere => 'GenuineIntel',
 128     'Westmere-IBRS' => 'GenuineIntel',
 129     SandyBridge => 'GenuineIntel',
 130     'SandyBridge-IBRS' => 'GenuineIntel',
 131     IvyBridge => 'GenuineIntel',
 132     'IvyBridge-IBRS' => 'GenuineIntel',
 133     Haswell => 'GenuineIntel',
 134     'Haswell-IBRS' => 'GenuineIntel',
 135     'Haswell-noTSX' => 'GenuineIntel',
 136     'Haswell-noTSX-IBRS' => 'GenuineIntel',
 137     Broadwell => 'GenuineIntel',
 138     'Broadwell-IBRS' => 'GenuineIntel',
 139     'Broadwell-noTSX' => 'GenuineIntel',
 140     'Broadwell-noTSX-IBRS' => 'GenuineIntel',
 141     'Skylake-Client' => 'GenuineIntel',
 142     'Skylake-Client-IBRS' => 'GenuineIntel',
 143     'Skylake-Server' => 'GenuineIntel',
 144     'Skylake-Server-IBRS' => 'GenuineIntel',
 145
 146     # AMD CPUs
 147     athlon => 'AuthenticAMD',
 148     phenom  => 'AuthenticAMD',
 149     Opteron_G1  => 'AuthenticAMD',
 150     Opteron_G2  => 'AuthenticAMD',
 151     Opteron_G3  => 'AuthenticAMD',
 152     Opteron_G4  => 'AuthenticAMD',
 153     Opteron_G5  => 'AuthenticAMD',
 154     EPYC => 'AuthenticAMD',
 155     'EPYC-IBPB' => 'AuthenticAMD',
 156
 157     # generic types, use vendor from host node
 158     host => 'default',
 159     kvm32 => 'default',
 160     kvm64 => 'default',
 161     qemu32 => 'default',
 162     qemu64 => 'default',
 163     max => 'default',
 164 };
 165
 166 my $cpu_flag = qr/[+-](pcid|spec-ctrl|ibpb|ssbd|virt-ssbd|amd-ssbd|amd-no-ssb|pdpe1gb)/;
 167
 168 my $cpu_fmt = {
 169     cputype => {
 170         description => "Emulated CPU type.",
 171         type => 'string',
 172         enum => [ sort { "\L$a" cmp "\L$b" } keys %$cpu_vendor_list ],
 173         default => 'kvm64',
 174         default_key => 1,
 175     },
 176     hidden => {
 177         description => "Do not identify as a KVM virtual machine.",
 178         type => 'boolean',
 179         optional => 1,
 180         default => 0
 181     },
 182     flags => {
 183         description => "List of additional CPU flags separated by ';'."
 184                      . " Use '+FLAG' to enable, '-FLAG' to disable a flag."
 185                      . " Currently supported flags: 'pcid', 'spec-ctrl', 'ibpb', 'ssbd', 'virt-ssbd', 'amd-ssbd', 'amd-no-ssb', 'pdpe1gb'.",
 186         format_description => '+FLAG[;-FLAG...]',
 187         type => 'string',
 188         pattern => qr/$cpu_flag(;$cpu_flag)*/,
 189         optional => 1,
 190     },
 191 };
 192
 193 my $watchdog_fmt = {
 194     model => {
 195         default_key => 1,
 196         type => 'string',
 197         enum => [qw(i6300esb ib700)],
 198         description => "Watchdog type to emulate.",
 199         default => 'i6300esb',
 200         optional => 1,
 201     },
 202     action => {
 203         type => 'string',
 204         enum => [qw(reset shutdown poweroff pause debug none)],
 205         description => "The action to perform if after activation the guest fails to poll the watchdog in time.",
 206         optional => 1,
 207     },
 208 };
 209 PVE::JSONSchema::register_format('pve-qm-watchdog', $watchdog_fmt);
 210
 211 my $agent_fmt = {
 212     enabled => {
 213         description => "Enable/disable Qemu GuestAgent.",
 214         type => 'boolean',
 215         default => 0,
 216         default_key => 1,
 217     },
 218     fstrim_cloned_disks => {
 219         description => "Run fstrim after cloning/moving a disk.",
 220         type => 'boolean',
 221         optional => 1,
 222         default => 0
 223     },
 224 };
 225
 226 my $confdesc = {
 227     onboot => {
 228         optional => 1,
 229         type => 'boolean',
 230         description => "Specifies whether a VM will be started during system bootup.",
 231         default => 0,
 232     },
 233     autostart => {
 234         optional => 1,
 235         type => 'boolean',
 236         description => "Automatic restart after crash (currently ignored).",
 237         default => 0,
 238     },
 239     hotplug => {
 240         optional => 1,
 241         type => 'string', format => 'pve-hotplug-features',
 242         description => "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
 243         default => 'network,disk,usb',
 244     },
 245     reboot => {
 246         optional => 1,
 247         type => 'boolean',
 248         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 249         default => 1,
 250     },
 251     lock => {
 252         optional => 1,
 253         type => 'string',
 254         description => "Lock/unlock the VM.",
 255         enum => [qw(migrate backup snapshot rollback)],
 256     },
 257     cpulimit => {
 258         optional => 1,
 259         type => 'number',
 260         description => "Limit of CPU usage.",
 261         verbose_description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
 262         minimum => 0,
 263         maximum => 128,
 264         default => 0,
 265     },
 266     cpuunits => {
 267         optional => 1,
 268         type => 'integer',
 269         description => "CPU weight for a VM.",
 270         verbose_description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.",
 271         minimum => 2,
 272         maximum => 262144,
 273         default => 1024,
 274     },
 275     memory => {
 276         optional => 1,
 277         type => 'integer',
 278         description => "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
 279         minimum => 16,
 280         default => 512,
 281     },
 282     balloon => {
 283         optional => 1,
 284         type => 'integer',
 285         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 286         minimum => 0,
 287     },
 288     shares => {
 289         optional => 1,
 290         type => 'integer',
 291         description => "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning. Auto-ballooning is done by pvestatd.",
 292         minimum => 0,
 293         maximum => 50000,
 294         default => 1000,
 295     },
 296     keyboard => {
 297         optional => 1,
 298         type => 'string',
 299         description => "Keybord layout for vnc server. Default is read from the '/etc/pve/datacenter.cfg' configuration file.".
 300                        "It should not be necessary to set it.",
 301         enum => PVE::Tools::kvmkeymaplist(),
 302         default => undef,
 303     },
 304     name => {
 305         optional => 1,
 306         type => 'string', format => 'dns-name',
 307         description => "Set a name for the VM. Only used on the configuration web interface.",
 308     },
 309     scsihw => {
 310         optional => 1,
 311         type => 'string',
 312         description => "SCSI controller model",
 313         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 314         default => 'lsi',
 315     },
 316     description => {
 317         optional => 1,
 318         type => 'string',
 319         description => "Description for the VM. Only used on the configuration web interface. This is saved as comment inside the configuration file.",
 320     },
 321     ostype => {
 322         optional => 1,
 323         type => 'string',
 324         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 l24 l26 solaris)],
 325         description => "Specify guest operating system.",
 326         verbose_description => <<EODESC,
 327 Specify guest operating system. This is used to enable special
 328 optimization/features for specific operating systems:
 329
 330 [horizontal]
 331 other;; unspecified OS
 332 wxp;; Microsoft Windows XP
 333 w2k;; Microsoft Windows 2000
 334 w2k3;; Microsoft Windows 2003
 335 w2k8;; Microsoft Windows 2008
 336 wvista;; Microsoft Windows Vista
 337 win7;; Microsoft Windows 7
 338 win8;; Microsoft Windows 8/2012/2012r2
 339 win10;; Microsoft Windows 10/2016
 340 l24;; Linux 2.4 Kernel
 341 l26;; Linux 2.6/3.X Kernel
 342 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
 343 EODESC
 344     },
 345     boot => {
 346         optional => 1,
 347         type => 'string',
 348         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n).",
 349         pattern => '[acdn]{1,4}',
 350         default => 'cdn',
 351     },
 352     bootdisk => {
 353         optional => 1,
 354         type => 'string', format => 'pve-qm-bootdisk',
 355         description => "Enable booting from specified disk.",
 356         pattern => '(ide|sata|scsi|virtio)\d+',
 357     },
 358     smp => {
 359         optional => 1,
 360         type => 'integer',
 361         description => "The number of CPUs. Please use option -sockets instead.",
 362         minimum => 1,
 363         default => 1,
 364     },
 365     sockets => {
 366         optional => 1,
 367         type => 'integer',
 368         description => "The number of CPU sockets.",
 369         minimum => 1,
 370         default => 1,
 371     },
 372     cores => {
 373         optional => 1,
 374         type => 'integer',
 375         description => "The number of cores per socket.",
 376         minimum => 1,
 377         default => 1,
 378     },
 379     numa => {
 380         optional => 1,
 381         type => 'boolean',
 382         description => "Enable/disable NUMA.",
 383         default => 0,
 384     },
 385     hugepages => {
 386         optional => 1,
 387         type => 'string',
 388         description => "Enable/disable hugepages memory.",
 389         enum => [qw(any 2 1024)],
 390     },
 391     vcpus => {
 392         optional => 1,
 393         type => 'integer',
 394         description => "Number of hotplugged vcpus.",
 395         minimum => 1,
 396         default => 0,
 397     },
 398     acpi => {
 399         optional => 1,
 400         type => 'boolean',
 401         description => "Enable/disable ACPI.",
 402         default => 1,
 403     },
 404     agent => {
 405         optional => 1,
 406         description => "Enable/disable Qemu GuestAgent and its properties.",
 407         type => 'string',
 408         format => $agent_fmt,
 409     },
 410     kvm => {
 411         optional => 1,
 412         type => 'boolean',
 413         description => "Enable/disable KVM hardware virtualization.",
 414         default => 1,
 415     },
 416     tdf => {
 417         optional => 1,
 418         type => 'boolean',
 419         description => "Enable/disable time drift fix.",
 420         default => 0,
 421     },
 422     localtime => {
 423         optional => 1,
 424         type => 'boolean',
 425         description => "Set the real time clock to local time. This is enabled by default if ostype indicates a Microsoft OS.",
 426     },
 427     freeze => {
 428         optional => 1,
 429         type => 'boolean',
 430         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 431     },
 432     vga => {
 433         optional => 1,
 434         type => 'string',
 435         description => "Select the VGA type.",
 436         verbose_description => "Select the VGA type. If you want to use high resolution" .
 437             " modes (>= 1280x1024x16) then you should use the options " .
 438             "'std' or 'vmware'. Default is 'std' for win8/win7/w2k8, and " .
 439             "'cirrus' for other OS types. The 'qxl' option enables the SPICE " .
 440             "display sever. For win* OS you can select how many independent " .
 441             "displays you want, Linux guests can add displays them self. " .
 442             "You can also run without any graphic card, using a serial device" .
 443             " as terminal.",
 444         enum => [qw(std cirrus vmware qxl serial0 serial1 serial2 serial3 qxl2 qxl3 qxl4)],
 445     },
 446     watchdog => {
 447         optional => 1,
 448         type => 'string', format => 'pve-qm-watchdog',
 449         description => "Create a virtual hardware watchdog device.",
 450         verbose_description => "Create a virtual hardware watchdog device. Once enabled" .
 451             " (by a guest action), the watchdog must be periodically polled " .
 452             "by an agent inside the guest or else the watchdog will reset " .
 453             "the guest (or execute the respective action specified)",
 454     },
 455     startdate => {
 456         optional => 1,
 457         type => 'string',
 458         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 459         description => "Set the initial date of the real time clock. Valid format for date are: 'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 460         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 461         default => 'now',
 462     },
 463     startup =>  get_standard_option('pve-startup-order'),
 464     template => {
 465         optional => 1,
 466         type => 'boolean',
 467         description => "Enable/disable Template.",
 468         default => 0,
 469     },
 470     args => {
 471         optional => 1,
 472         type => 'string',
 473         description => "Arbitrary arguments passed to kvm.",
 474         verbose_description => <<EODESCR,
 475 Arbitrary arguments passed to kvm, for example:
 476
 477 args: -no-reboot -no-hpet
 478
 479 NOTE: this option is for experts only.
 480 EODESCR
 481     },
 482     tablet => {
 483         optional => 1,
 484         type => 'boolean',
 485         default => 1,
 486         description => "Enable/disable the USB tablet device.",
 487         verbose_description => "Enable/disable the USB tablet device. This device is " .
 488             "usually needed to allow absolute mouse positioning with VNC. " .
 489             "Else the mouse runs out of sync with normal VNC clients. " .
 490             "If you're running lots of console-only guests on one host, " .
 491             "you may consider disabling this to save some context switches. " .
 492             "This is turned off by default if you use spice (-vga=qxl).",
 493     },
 494     migrate_speed => {
 495         optional => 1,
 496         type => 'integer',
 497         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 498         minimum => 0,
 499         default => 0,
 500     },
 501     migrate_downtime => {
 502         optional => 1,
 503         type => 'number',
 504         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 505         minimum => 0,
 506         default => 0.1,
 507     },
 508     cdrom => {
 509         optional => 1,
 510         type => 'string', format => 'pve-qm-ide',
 511         typetext => '<volume>',
 512         description => "This is an alias for option -ide2",
 513     },
 514     cpu => {
 515         optional => 1,
 516         description => "Emulated CPU type.",
 517         type => 'string',
 518         format => $cpu_fmt,
 519     },
 520     parent => get_standard_option('pve-snapshot-name', {
 521         optional => 1,
 522         description => "Parent snapshot name. This is used internally, and should not be modified.",
 523     }),
 524     snaptime => {
 525         optional => 1,
 526         description => "Timestamp for snapshots.",
 527         type => 'integer',
 528         minimum => 0,
 529     },
 530     vmstate => {
 531         optional => 1,
 532         type => 'string', format => 'pve-volume-id',
 533         description => "Reference to a volume which stores the VM state. This is used internally for snapshots.",
 534     },
 535     vmstatestorage => get_standard_option('pve-storage-id', {
 536         description => "Default storage for VM state volumes/files.",
 537         optional => 1,
 538     }),
 539     runningmachine => get_standard_option('pve-qemu-machine', {
 540         description => "Specifies the Qemu machine type of the running vm. This is used internally for snapshots.",
 541     }),
 542     machine => get_standard_option('pve-qemu-machine'),
 543     smbios1 => {
 544         description => "Specify SMBIOS type 1 fields.",
 545         type => 'string', format => 'pve-qm-smbios1',
 546         maxLength => 256,
 547         optional => 1,
 548     },
 549     protection => {
 550         optional => 1,
 551         type => 'boolean',
 552         description => "Sets the protection flag of the VM. This will disable the remove VM and remove disk operations.",
 553         default => 0,
 554     },
 555     bios => {
 556         optional => 1,
 557         type => 'string',
 558         enum => [ qw(seabios ovmf) ],
 559         description => "Select BIOS implementation.",
 560         default => 'seabios',
 561     },
 562 };
 563
 564 my $confdesc_cloudinit = {
 565     citype => {
 566         optional => 1,
 567         type => 'string',
 568         description => 'Specifies the cloud-init configuration format. The default depends on the configured operating system type (`ostype`. We use the `nocloud` format for Linux, and `configdrive2` for windows.',
 569         enum => ['configdrive2', 'nocloud'],
 570     },
 571     ciuser => {
 572         optional => 1,
 573         type => 'string',
 574         description => "cloud-init: User name to change ssh keys and password for instead of the image's configured default user.",
 575     },
 576     cipassword => {
 577         optional => 1,
 578         type => 'string',
 579         description => 'cloud-init: Password to assign the user. Using this is generally not recommended. Use ssh keys instead. Also note that older cloud-init versions do not support hashed passwords.',
 580     },
 581     searchdomain => {
 582         optional => 1,
 583         type => 'string',
 584         description => "cloud-init: Sets DNS search domains for a container. Create will automatically use the setting from the host if neither searchdomain nor nameserver are set.",
 585     },
 586     nameserver => {
 587         optional => 1,
 588         type => 'string', format => 'address-list',
 589         description => "cloud-init: Sets DNS server IP address for a container. Create will automatically use the setting from the host if neither searchdomain nor nameserver are set.",
 590     },
 591     sshkeys => {
 592         optional => 1,
 593         type => 'string',
 594         format => 'urlencoded',
 595         description => "cloud-init: Setup public SSH keys (one key per line, OpenSSH format).",
 596     },
 597 };
 598
 599 # what about other qemu settings ?
 600 #cpu => 'string',
 601 #machine => 'string',
 602 #fda => 'file',
 603 #fdb => 'file',
 604 #mtdblock => 'file',
 605 #sd => 'file',
 606 #pflash => 'file',
 607 #snapshot => 'bool',
 608 #bootp => 'file',
 609 ##tftp => 'dir',
 610 ##smb => 'dir',
 611 #kernel => 'file',
 612 #append => 'string',
 613 #initrd => 'file',
 614 ##soundhw => 'string',
 615
 616 while (my ($k, $v) = each %$confdesc) {
 617     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 618 }
 619
 620 my $MAX_IDE_DISKS = 4;
 621 my $MAX_SCSI_DISKS = 14;
 622 my $MAX_VIRTIO_DISKS = 16;
 623 my $MAX_SATA_DISKS = 6;
 624 my $MAX_USB_DEVICES = 5;
 625 my $MAX_NETS = 32;
 626 my $MAX_UNUSED_DISKS = 8;
 627 my $MAX_HOSTPCI_DEVICES = 4;
 628 my $MAX_SERIAL_PORTS = 4;
 629 my $MAX_PARALLEL_PORTS = 3;
 630 my $MAX_NUMA = 8;
 631
 632 my $numa_fmt = {
 633     cpus => {
 634         type => "string",
 635         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 636         description => "CPUs accessing this NUMA node.",
 637         format_description => "id[-id];...",
 638     },
 639     memory => {
 640         type => "number",
 641         description => "Amount of memory this NUMA node provides.",
 642         optional => 1,
 643     },
 644     hostnodes => {
 645         type => "string",
 646         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 647         description => "Host NUMA nodes to use.",
 648         format_description => "id[-id];...",
 649         optional => 1,
 650     },
 651     policy => {
 652         type => 'string',
 653         enum => [qw(preferred bind interleave)],
 654         description => "NUMA allocation policy.",
 655         optional => 1,
 656     },
 657 };
 658 PVE::JSONSchema::register_format('pve-qm-numanode', $numa_fmt);
 659 my $numadesc = {
 660     optional => 1,
 661     type => 'string', format => $numa_fmt,
 662     description => "NUMA topology.",
 663 };
 664 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 665
 666 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 667     $confdesc->{"numa$i"} = $numadesc;
 668 }
 669
 670 my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000',  'pcnet',  'virtio',
 671                       'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3',
 672                       'e1000-82540em', 'e1000-82544gc', 'e1000-82545em'];
 673 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 674
 675 my $net_fmt_bridge_descr = <<__EOD__;
 676 Bridge to attach the network device to. The Proxmox VE standard bridge
 677 is called 'vmbr0'.
 678
 679 If you do not specify a bridge, we create a kvm user (NATed) network
 680 device, which provides DHCP and DNS services. The following addresses
 681 are used:
 682
 683  10.0.2.2   Gateway
 684  10.0.2.3   DNS Server
 685  10.0.2.4   SMB Server
 686
 687 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 688 __EOD__
 689
 690 my $net_fmt = {
 691     macaddr => {
 692         type => 'string',
 693         pattern => qr/[0-9a-f]{2}(?::[0-9a-f]{2}){5}/i,
 694         description => "MAC address. That address must be unique withing your network. This is automatically generated if not specified.",
 695         format_description => "XX:XX:XX:XX:XX:XX",
 696         optional => 1,
 697     },
 698     model => {
 699         type => 'string',
 700         description => "Network Card Model. The 'virtio' model provides the best performance with very low CPU overhead. If your guest does not support this driver, it is usually best to use 'e1000'.",
 701         enum => $nic_model_list,
 702         default_key => 1,
 703     },
 704     (map { $_ => { keyAlias => 'model', alias => 'macaddr' }} @$nic_model_list),
 705     bridge => {
 706         type => 'string',
 707         description => $net_fmt_bridge_descr,
 708         format_description => 'bridge',
 709         optional => 1,
 710     },
 711     queues => {
 712         type => 'integer',
 713         minimum => 0, maximum => 16,
 714         description => 'Number of packet queues to be used on the device.',
 715         optional => 1,
 716     },
 717     rate => {
 718         type => 'number',
 719         minimum => 0,
 720         description => "Rate limit in mbps (megabytes per second) as floating point number.",
 721         optional => 1,
 722     },
 723     tag => {
 724         type => 'integer',
 725         minimum => 1, maximum => 4094,
 726         description => 'VLAN tag to apply to packets on this interface.',
 727         optional => 1,
 728     },
 729     trunks => {
 730         type => 'string',
 731         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 732         description => 'VLAN trunks to pass through this interface.',
 733         format_description => 'vlanid[;vlanid...]',
 734         optional => 1,
 735     },
 736     firewall => {
 737         type => 'boolean',
 738         description => 'Whether this interface should be protected by the firewall.',
 739         optional => 1,
 740     },
 741     link_down => {
 742         type => 'boolean',
 743         description => 'Whether this interface should be disconnected (like pulling the plug).',
 744         optional => 1,
 745     },
 746 };
 747
 748 my $netdesc = {
 749     optional => 1,
 750     type => 'string', format => $net_fmt,
 751     description => "Specify network devices.",
 752 };
 753
 754 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 755
 756 my $ipconfig_fmt = {
 757     ip => {
 758         type => 'string',
 759         format => 'pve-ipv4-config',
 760         format_description => 'IPv4Format/CIDR',
 761         description => 'IPv4 address in CIDR format.',
 762         optional => 1,
 763         default => 'dhcp',
 764     },
 765     gw => {
 766         type => 'string',
 767         format => 'ipv4',
 768         format_description => 'GatewayIPv4',
 769         description => 'Default gateway for IPv4 traffic.',
 770         optional => 1,
 771         requires => 'ip',
 772     },
 773     ip6 => {
 774         type => 'string',
 775         format => 'pve-ipv6-config',
 776         format_description => 'IPv6Format/CIDR',
 777         description => 'IPv6 address in CIDR format.',
 778         optional => 1,
 779         default => 'dhcp',
 780     },
 781     gw6 => {
 782         type => 'string',
 783         format => 'ipv6',
 784         format_description => 'GatewayIPv6',
 785         description => 'Default gateway for IPv6 traffic.',
 786         optional => 1,
 787         requires => 'ip6',
 788     },
 789 };
 790 PVE::JSONSchema::register_format('pve-qm-ipconfig', $ipconfig_fmt);
 791 my $ipconfigdesc = {
 792     optional => 1,
 793     type => 'string', format => 'pve-qm-ipconfig',
 794     description => <<'EODESCR',
 795 cloud-init: Specify IP addresses and gateways for the corresponding interface.
 796
 797 IP addresses use CIDR notation, gateways are optional but need an IP of the same type specified.
 798
 799 The special string 'dhcp' can be used for IP addresses to use DHCP, in which case no explicit gateway should be provided.
 800 For IPv6 the special string 'auto' can be used to use stateless autoconfiguration.
 801
 802 If cloud-init is enabled and neither an IPv4 nor an IPv6 address is specified, it defaults to using dhcp on IPv4.
 803 EODESCR
 804 };
 805 PVE::JSONSchema::register_standard_option("pve-qm-ipconfig", $netdesc);
 806
 807 for (my $i = 0; $i < $MAX_NETS; $i++)  {
 808     $confdesc->{"net$i"} = $netdesc;
 809     $confdesc_cloudinit->{"ipconfig$i"} = $ipconfigdesc;
 810 }
 811
 812 foreach my $key (keys %$confdesc_cloudinit) {
 813     $confdesc->{$key} = $confdesc_cloudinit->{$key};
 814 }
 815
 816 PVE::JSONSchema::register_format('pve-volume-id-or-qm-path', \&verify_volume_id_or_qm_path);
 817 sub verify_volume_id_or_qm_path {
 818     my ($volid, $noerr) = @_;
 819
 820     if ($volid eq 'none' || $volid eq 'cdrom' || $volid =~ m|^/|) {
 821         return $volid;
 822     }
 823
 824     # if its neither 'none' nor 'cdrom' nor a path, check if its a volume-id
 825     $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') };
 826     if ($@) {
 827         return undef if $noerr;
 828         die $@;
 829     }
 830     return $volid;
 831 }
 832
 833 my $drivename_hash;
 834
 835 my %drivedesc_base = (
 836     volume => { alias => 'file' },
 837     file => {
 838         type => 'string',
 839         format => 'pve-volume-id-or-qm-path',
 840         default_key => 1,
 841         format_description => 'volume',
 842         description => "The drive's backing volume.",
 843     },
 844     media => {
 845         type => 'string',
 846         enum => [qw(cdrom disk)],
 847         description => "The drive's media type.",
 848         default => 'disk',
 849         optional => 1
 850     },
 851     cyls => {
 852         type => 'integer',
 853         description => "Force the drive's physical geometry to have a specific cylinder count.",
 854         optional => 1
 855     },
 856     heads => {
 857         type => 'integer',
 858         description => "Force the drive's physical geometry to have a specific head count.",
 859         optional => 1
 860     },
 861     secs => {
 862         type => 'integer',
 863         description => "Force the drive's physical geometry to have a specific sector count.",
 864         optional => 1
 865     },
 866     trans => {
 867         type => 'string',
 868         enum => [qw(none lba auto)],
 869         description => "Force disk geometry bios translation mode.",
 870         optional => 1,
 871     },
 872     snapshot => {
 873         type => 'boolean',
 874         description => "Controls qemu's snapshot mode feature."
 875             . " If activated, changes made to the disk are temporary and will"
 876             . " be discarded when the VM is shutdown.",
 877         optional => 1,
 878     },
 879     cache => {
 880         type => 'string',
 881         enum => [qw(none writethrough writeback unsafe directsync)],
 882         description => "The drive's cache mode",
 883         optional => 1,
 884     },
 885     format => get_standard_option('pve-qm-image-format'),
 886     size => {
 887         type => 'string',
 888         format => 'disk-size',
 889         format_description => 'DiskSize',
 890         description => "Disk size. This is purely informational and has no effect.",
 891         optional => 1,
 892     },
 893     backup => {
 894         type => 'boolean',
 895         description => "Whether the drive should be included when making backups.",
 896         optional => 1,
 897     },
 898     replicate => {
 899         type => 'boolean',
 900         description => 'Whether the drive should considered for replication jobs.',
 901         optional => 1,
 902         default => 1,
 903     },
 904     rerror => {
 905         type => 'string',
 906         enum => [qw(ignore report stop)],
 907         description => 'Read error action.',
 908         optional => 1,
 909     },
 910     werror => {
 911         type => 'string',
 912         enum => [qw(enospc ignore report stop)],
 913         description => 'Write error action.',
 914         optional => 1,
 915     },
 916     aio => {
 917         type => 'string',
 918         enum => [qw(native threads)],
 919         description => 'AIO type to use.',
 920         optional => 1,
 921     },
 922     discard => {
 923         type => 'string',
 924         enum => [qw(ignore on)],
 925         description => 'Controls whether to pass discard/trim requests to the underlying storage.',
 926         optional => 1,
 927     },
 928     detect_zeroes => {
 929         type => 'boolean',
 930         description => 'Controls whether to detect and try to optimize writes of zeroes.',
 931         optional => 1,
 932     },
 933     serial => {
 934         type => 'string',
 935         format => 'urlencoded',
 936         format_description => 'serial',
 937         maxLength => 20*3, # *3 since it's %xx url enoded
 938         description => "The drive's reported serial number, url-encoded, up to 20 bytes long.",
 939         optional => 1,
 940     },
 941     shared => {
 942         type => 'boolean',
 943         description => 'Mark this locally-managed volume as available on all nodes',
 944         verbose_description => "Mark this locally-managed volume as available on all nodes.\n\nWARNING: This option does not share the volume automatically, it assumes it is shared already!",
 945         optional => 1,
 946         default => 0,
 947     }
 948 );
 949
 950 my %iothread_fmt = ( iothread => {
 951         type => 'boolean',
 952         description => "Whether to use iothreads for this drive",
 953         optional => 1,
 954 });
 955
 956 my %model_fmt = (
 957     model => {
 958         type => 'string',
 959         format => 'urlencoded',
 960         format_description => 'model',
 961         maxLength => 40*3, # *3 since it's %xx url enoded
 962         description => "The drive's reported model name, url-encoded, up to 40 bytes long.",
 963         optional => 1,
 964     },
 965 );
 966
 967 my %queues_fmt = (
 968     queues => {
 969         type => 'integer',
 970         description => "Number of queues.",
 971         minimum => 2,
 972         optional => 1
 973     }
 974 );
 975
 976 my %scsiblock_fmt = (
 977     scsiblock => {
 978         type => 'boolean',
 979         description => "whether to use scsi-block for full passthrough of host block device\n\nWARNING: can lead to I/O errors in combination with low memory or high memory fragmentation on host",
 980         optional => 1,
 981         default => 0,
 982     },
 983 );
 984
 985 my $add_throttle_desc = sub {
 986     my ($key, $type, $what, $unit, $longunit, $minimum) = @_;
 987     my $d = {
 988         type => $type,
 989         format_description => $unit,
 990         description => "Maximum $what in $longunit.",
 991         optional => 1,
 992     };
 993     $d->{minimum} = $minimum if defined($minimum);
 994     $drivedesc_base{$key} = $d;
 995 };
 996 # throughput: (leaky bucket)
 997 $add_throttle_desc->('bps',     'integer', 'r/w speed',   'bps',  'bytes per second');
 998 $add_throttle_desc->('bps_rd',  'integer', 'read speed',  'bps',  'bytes per second');
 999 $add_throttle_desc->('bps_wr',  'integer', 'write speed', 'bps',  'bytes per second');
1000 $add_throttle_desc->('mbps',    'number',  'r/w speed',   'mbps', 'megabytes per second');
1001 $add_throttle_desc->('mbps_rd', 'number',  'read speed',  'mbps', 'megabytes per second');
1002 $add_throttle_desc->('mbps_wr', 'number',  'write speed', 'mbps', 'megabytes per second');
1003 $add_throttle_desc->('iops',    'integer', 'r/w I/O',     'iops', 'operations per second');
1004 $add_throttle_desc->('iops_rd', 'integer', 'read I/O',    'iops', 'operations per second');
1005 $add_throttle_desc->('iops_wr', 'integer', 'write I/O',   'iops', 'operations per second');
1006
1007 # pools: (pool of IO before throttling starts taking effect)
1008 $add_throttle_desc->('mbps_max',    'number',  'unthrottled r/w pool',       'mbps', 'megabytes per second');
1009 $add_throttle_desc->('mbps_rd_max', 'number',  'unthrottled read pool',      'mbps', 'megabytes per second');
1010 $add_throttle_desc->('mbps_wr_max', 'number',  'unthrottled write pool',     'mbps', 'megabytes per second');
1011 $add_throttle_desc->('iops_max',    'integer', 'unthrottled r/w I/O pool',   'iops', 'operations per second');
1012 $add_throttle_desc->('iops_rd_max', 'integer', 'unthrottled read I/O pool',  'iops', 'operations per second');
1013 $add_throttle_desc->('iops_wr_max', 'integer', 'unthrottled write I/O pool', 'iops', 'operations per second');
1014
1015 # burst lengths
1016 $add_throttle_desc->('bps_max_length',     'integer', 'length of I/O bursts',       'seconds', 'seconds', 1);
1017 $add_throttle_desc->('bps_rd_max_length',  'integer', 'length of read I/O bursts',  'seconds', 'seconds', 1);
1018 $add_throttle_desc->('bps_wr_max_length',  'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
1019 $add_throttle_desc->('iops_max_length',    'integer', 'length of I/O bursts',       'seconds', 'seconds', 1);
1020 $add_throttle_desc->('iops_rd_max_length', 'integer', 'length of read I/O bursts',  'seconds', 'seconds', 1);
1021 $add_throttle_desc->('iops_wr_max_length', 'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
1022
1023 # legacy support
1024 $drivedesc_base{'bps_rd_length'} = { alias => 'bps_rd_max_length' };
1025 $drivedesc_base{'bps_wr_length'} = { alias => 'bps_wr_max_length' };
1026 $drivedesc_base{'iops_rd_length'} = { alias => 'iops_rd_max_length' };
1027 $drivedesc_base{'iops_wr_length'} = { alias => 'iops_wr_max_length' };
1028
1029 my $ide_fmt = {
1030     %drivedesc_base,
1031     %model_fmt,
1032 };
1033 PVE::JSONSchema::register_format("pve-qm-ide", $ide_fmt);
1034
1035 my $idedesc = {
1036     optional => 1,
1037     type => 'string', format => $ide_fmt,
1038     description => "Use volume as IDE hard disk or CD-ROM (n is 0 to " .($MAX_IDE_DISKS -1) . ").",
1039 };
1040 PVE::JSONSchema::register_standard_option("pve-qm-ide", $idedesc);
1041
1042 my $scsi_fmt = {
1043     %drivedesc_base,
1044     %iothread_fmt,
1045     %queues_fmt,
1046     %scsiblock_fmt,
1047 };
1048 my $scsidesc = {
1049     optional => 1,
1050     type => 'string', format => $scsi_fmt,
1051     description => "Use volume as SCSI hard disk or CD-ROM (n is 0 to " . ($MAX_SCSI_DISKS - 1) . ").",
1052 };
1053 PVE::JSONSchema::register_standard_option("pve-qm-scsi", $scsidesc);
1054
1055 my $sata_fmt = {
1056     %drivedesc_base,
1057 };
1058 my $satadesc = {
1059     optional => 1,
1060     type => 'string', format => $sata_fmt,
1061     description => "Use volume as SATA hard disk or CD-ROM (n is 0 to " . ($MAX_SATA_DISKS - 1). ").",
1062 };
1063 PVE::JSONSchema::register_standard_option("pve-qm-sata", $satadesc);
1064
1065 my $virtio_fmt = {
1066     %drivedesc_base,
1067     %iothread_fmt,
1068 };
1069 my $virtiodesc = {
1070     optional => 1,
1071     type => 'string', format => $virtio_fmt,
1072     description => "Use volume as VIRTIO hard disk (n is 0 to " . ($MAX_VIRTIO_DISKS - 1) . ").",
1073 };
1074 PVE::JSONSchema::register_standard_option("pve-qm-virtio", $virtiodesc);
1075
1076 my $alldrive_fmt = {
1077     %drivedesc_base,
1078     %iothread_fmt,
1079     %model_fmt,
1080     %queues_fmt,
1081     %scsiblock_fmt,
1082 };
1083
1084 my $efidisk_fmt = {
1085     volume => { alias => 'file' },
1086     file => {
1087         type => 'string',
1088         format => 'pve-volume-id-or-qm-path',
1089         default_key => 1,
1090         format_description => 'volume',
1091         description => "The drive's backing volume.",
1092     },
1093     format => get_standard_option('pve-qm-image-format'),
1094     size => {
1095         type => 'string',
1096         format => 'disk-size',
1097         format_description => 'DiskSize',
1098         description => "Disk size. This is purely informational and has no effect.",
1099         optional => 1,
1100     },
1101 };
1102
1103 my $efidisk_desc = {
1104     optional => 1,
1105     type => 'string', format => $efidisk_fmt,
1106     description => "Configure a Disk for storing EFI vars",
1107 };
1108
1109 PVE::JSONSchema::register_standard_option("pve-qm-efidisk", $efidisk_desc);
1110
1111 my $usb_fmt = {
1112     host => {
1113         default_key => 1,
1114         type => 'string', format => 'pve-qm-usb-device',
1115         format_description => 'HOSTUSBDEVICE|spice',
1116         description => <<EODESCR,
1117 The Host USB device or port or the value 'spice'. HOSTUSBDEVICE syntax is:
1118
1119  'bus-port(.port)*' (decimal numbers) or
1120  'vendor_id:product_id' (hexadeciaml numbers) or
1121  'spice'
1122
1123 You can use the 'lsusb -t' command to list existing usb devices.
1124
1125 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1126
1127 The value 'spice' can be used to add a usb redirection devices for spice.
1128 EODESCR
1129     },
1130     usb3 => {
1131         optional => 1,
1132         type => 'boolean',
1133         description => "Specifies whether if given host option is a USB3 device or port (this does currently not work reliably with spice redirection and is then ignored).",
1134         default => 0,
1135     },
1136 };
1137
1138 my $usbdesc = {
1139     optional => 1,
1140     type => 'string', format => $usb_fmt,
1141     description => "Configure an USB device (n is 0 to 4).",
1142 };
1143 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
1144
1145 # NOTE: the match-groups of this regex are used in parse_hostpci
1146 my $PCIRE = qr/([a-f0-9]{2}:[a-f0-9]{2})(?:\.([a-f0-9]))?/;
1147 my $hostpci_fmt = {
1148     host => {
1149         default_key => 1,
1150         type => 'string',
1151         pattern => qr/$PCIRE(;$PCIRE)*/,
1152         format_description => 'HOSTPCIID[;HOSTPCIID2...]',
1153         description => <<EODESCR,
1154 Host PCI device pass through. The PCI ID of a host's PCI device or a list
1155 of PCI virtual functions of the host. HOSTPCIID syntax is:
1156
1157 'bus:dev.func' (hexadecimal numbers)
1158
1159 You can us the 'lspci' command to list existing PCI devices.
1160 EODESCR
1161     },
1162     rombar => {
1163         type => 'boolean',
1164         description =>  "Specify whether or not the device's ROM will be visible in the guest's memory map.",
1165         optional => 1,
1166         default => 1,
1167     },
1168     romfile => {
1169         type => 'string',
1170         pattern => '[^,;]+',
1171         format_description => 'string',
1172         description => "Custom pci device rom filename (must be located in /usr/share/kvm/).",
1173         optional => 1,
1174     },
1175     pcie => {
1176         type => 'boolean',
1177         description =>  "Choose the PCI-express bus (needs the 'q35' machine model).",
1178         optional => 1,
1179         default => 0,
1180     },
1181     'x-vga' => {
1182         type => 'boolean',
1183         description =>  "Enable vfio-vga device support.",
1184         optional => 1,
1185         default => 0,
1186     },
1187 };
1188 PVE::JSONSchema::register_format('pve-qm-hostpci', $hostpci_fmt);
1189
1190 my $hostpcidesc = {
1191         optional => 1,
1192         type => 'string', format => 'pve-qm-hostpci',
1193         description => "Map host PCI devices into guest.",
1194         verbose_description =>  <<EODESCR,
1195 Map host PCI devices into guest.
1196
1197 NOTE: This option allows direct access to host hardware. So it is no longer
1198 possible to migrate such machines - use with special care.
1199
1200 CAUTION: Experimental! User reported problems with this option.
1201 EODESCR
1202 };
1203 PVE::JSONSchema::register_standard_option("pve-qm-hostpci", $hostpcidesc);
1204
1205 my $serialdesc = {
1206         optional => 1,
1207         type => 'string',
1208         pattern => '(/dev/.+|socket)',
1209         description =>  "Create a serial device inside the VM (n is 0 to 3)",
1210         verbose_description =>  <<EODESCR,
1211 Create a serial device inside the VM (n is 0 to 3), and pass through a
1212 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1213 host side (use 'qm terminal' to open a terminal connection).
1214
1215 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines - use with special care.
1216
1217 CAUTION: Experimental! User reported problems with this option.
1218 EODESCR
1219 };
1220
1221 my $paralleldesc= {
1222         optional => 1,
1223         type => 'string',
1224         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
1225         description =>  "Map host parallel devices (n is 0 to 2).",
1226         verbose_description =>  <<EODESCR,
1227 Map host parallel devices (n is 0 to 2).
1228
1229 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1230
1231 CAUTION: Experimental! User reported problems with this option.
1232 EODESCR
1233 };
1234
1235 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
1236     $confdesc->{"parallel$i"} = $paralleldesc;
1237 }
1238
1239 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
1240     $confdesc->{"serial$i"} = $serialdesc;
1241 }
1242
1243 for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
1244     $confdesc->{"hostpci$i"} = $hostpcidesc;
1245 }
1246
1247 for (my $i = 0; $i < $MAX_IDE_DISKS; $i++)  {
1248     $drivename_hash->{"ide$i"} = 1;
1249     $confdesc->{"ide$i"} = $idedesc;
1250 }
1251
1252 for (my $i = 0; $i < $MAX_SATA_DISKS; $i++)  {
1253     $drivename_hash->{"sata$i"} = 1;
1254     $confdesc->{"sata$i"} = $satadesc;
1255 }
1256
1257 for (my $i = 0; $i < $MAX_SCSI_DISKS; $i++)  {
1258     $drivename_hash->{"scsi$i"} = 1;
1259     $confdesc->{"scsi$i"} = $scsidesc ;
1260 }
1261
1262 for (my $i = 0; $i < $MAX_VIRTIO_DISKS; $i++)  {
1263     $drivename_hash->{"virtio$i"} = 1;
1264     $confdesc->{"virtio$i"} = $virtiodesc;
1265 }
1266
1267 $drivename_hash->{efidisk0} = 1;
1268 $confdesc->{efidisk0} = $efidisk_desc;
1269
1270 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
1271     $confdesc->{"usb$i"} = $usbdesc;
1272 }
1273
1274 my $unuseddesc = {
1275     optional => 1,
1276     type => 'string', format => 'pve-volume-id',
1277     description => "Reference to unused volumes. This is used internally, and should not be modified manually.",
1278 };
1279
1280 for (my $i = 0; $i < $MAX_UNUSED_DISKS; $i++)  {
1281     $confdesc->{"unused$i"} = $unuseddesc;
1282 }
1283
1284 my $kvm_api_version = 0;
1285
1286 sub kvm_version {
1287
1288     return $kvm_api_version if $kvm_api_version;
1289
1290     my $fh = IO::File->new("</dev/kvm") ||
1291         return 0;
1292
1293     if (my $v = $fh->ioctl(KVM_GET_API_VERSION(), 0)) {
1294         $kvm_api_version = $v;
1295     }
1296
1297     $fh->close();
1298
1299     return  $kvm_api_version;
1300 }
1301
1302 my $kvm_user_version;
1303
1304 sub kvm_user_version {
1305
1306     return $kvm_user_version if $kvm_user_version;
1307
1308     $kvm_user_version = 'unknown';
1309
1310     my $code = sub {
1311         my $line = shift;
1312         if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1313             $kvm_user_version = $2;
1314         }
1315     };
1316
1317     eval { run_command("kvm -version", outfunc => $code); };
1318     warn $@ if $@;
1319
1320     return $kvm_user_version;
1321
1322 }
1323
1324 my $kernel_has_vhost_net = -c '/dev/vhost-net';
1325
1326 sub valid_drive_names {
1327     # order is important - used to autoselect boot disk
1328     return ((map { "ide$_" } (0 .. ($MAX_IDE_DISKS - 1))),
1329             (map { "scsi$_" } (0 .. ($MAX_SCSI_DISKS - 1))),
1330             (map { "virtio$_" } (0 .. ($MAX_VIRTIO_DISKS - 1))),
1331             (map { "sata$_" } (0 .. ($MAX_SATA_DISKS - 1))),
1332             'efidisk0');
1333 }
1334
1335 sub is_valid_drivename {
1336     my $dev = shift;
1337
1338     return defined($drivename_hash->{$dev});
1339 }
1340
1341 sub option_exists {
1342     my $key = shift;
1343     return defined($confdesc->{$key});
1344 }
1345
1346 sub nic_models {
1347     return $nic_model_list;
1348 }
1349
1350 sub os_list_description {
1351
1352     return {
1353         other => 'Other',
1354         wxp => 'Windows XP',
1355         w2k => 'Windows 2000',
1356         w2k3 =>, 'Windows 2003',
1357         w2k8 => 'Windows 2008',
1358         wvista => 'Windows Vista',
1359         win7 => 'Windows 7',
1360         win8 => 'Windows 8/2012',
1361         win10 => 'Windows 10/2016',
1362         l24 => 'Linux 2.4',
1363         l26 => 'Linux 2.6',
1364     };
1365 }
1366
1367 my $cdrom_path;
1368
1369 sub get_cdrom_path {
1370
1371     return  $cdrom_path if $cdrom_path;
1372
1373     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
1374     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
1375     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
1376 }
1377
1378 sub get_iso_path {
1379     my ($storecfg, $vmid, $cdrom) = @_;
1380
1381     if ($cdrom eq 'cdrom') {
1382         return get_cdrom_path();
1383     } elsif ($cdrom eq 'none') {
1384         return '';
1385     } elsif ($cdrom =~ m|^/|) {
1386         return $cdrom;
1387     } else {
1388         return PVE::Storage::path($storecfg, $cdrom);
1389     }
1390 }
1391
1392 # try to convert old style file names to volume IDs
1393 sub filename_to_volume_id {
1394     my ($vmid, $file, $media) = @_;
1395
1396      if (!($file eq 'none' || $file eq 'cdrom' ||
1397           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1398
1399         return undef if $file =~ m|/|;
1400
1401         if ($media && $media eq 'cdrom') {
1402             $file = "local:iso/$file";
1403         } else {
1404             $file = "local:$vmid/$file";
1405         }
1406     }
1407
1408     return $file;
1409 }
1410
1411 sub verify_media_type {
1412     my ($opt, $vtype, $media) = @_;
1413
1414     return if !$media;
1415
1416     my $etype;
1417     if ($media eq 'disk') {
1418         $etype = 'images';
1419     } elsif ($media eq 'cdrom') {
1420         $etype = 'iso';
1421     } else {
1422         die "internal error";
1423     }
1424
1425     return if ($vtype eq $etype);
1426
1427     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1428 }
1429
1430 sub cleanup_drive_path {
1431     my ($opt, $storecfg, $drive) = @_;
1432
1433     # try to convert filesystem paths to volume IDs
1434
1435     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1436         ($drive->{file} !~ m|^/dev/.+|) &&
1437         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1438         ($drive->{file} !~ m/^\d+$/)) {
1439         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1440         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"}) if !$vtype;
1441         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1442         verify_media_type($opt, $vtype, $drive->{media});
1443         $drive->{file} = $volid;
1444     }
1445
1446     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1447 }
1448
1449 sub parse_hotplug_features {
1450     my ($data) = @_;
1451
1452     my $res = {};
1453
1454     return $res if $data eq '0';
1455
1456     $data = $confdesc->{hotplug}->{default} if $data eq '1';
1457
1458     foreach my $feature (PVE::Tools::split_list($data)) {
1459         if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
1460             $res->{$1} = 1;
1461         } else {
1462             die "invalid hotplug feature '$feature'\n";
1463         }
1464     }
1465     return $res;
1466 }
1467
1468 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1469 sub pve_verify_hotplug_features {
1470     my ($value, $noerr) = @_;
1471
1472     return $value if parse_hotplug_features($value);
1473
1474     return undef if $noerr;
1475
1476     die "unable to parse hotplug option\n";
1477 }
1478
1479 # ideX = [volume=]volume-id[,media=d][,cyls=c,heads=h,secs=s[,trans=t]]
1480 #        [,snapshot=on|off][,cache=on|off][,format=f][,backup=yes|no]
1481 #        [,rerror=ignore|report|stop][,werror=enospc|ignore|report|stop]
1482 #        [,aio=native|threads][,discard=ignore|on][,detect_zeroes=on|off]
1483 #        [,iothread=on][,serial=serial][,model=model]
1484
1485 sub parse_drive {
1486     my ($key, $data) = @_;
1487
1488     my ($interface, $index);
1489
1490     if ($key =~ m/^([^\d]+)(\d+)$/) {
1491         $interface = $1;
1492         $index = $2;
1493     } else {
1494         return undef;
1495     }
1496
1497     my $desc = $key =~ /^unused\d+$/ ? $alldrive_fmt
1498                                      : $confdesc->{$key}->{format};
1499     if (!$desc) {
1500         warn "invalid drive key: $key\n";
1501         return undef;
1502     }
1503     my $res = eval { PVE::JSONSchema::parse_property_string($desc, $data) };
1504     return undef if !$res;
1505     $res->{interface} = $interface;
1506     $res->{index} = $index;
1507
1508     my $error = 0;
1509     foreach my $opt (qw(bps bps_rd bps_wr)) {
1510         if (my $bps = defined(delete $res->{$opt})) {
1511             if (defined($res->{"m$opt"})) {
1512                 warn "both $opt and m$opt specified\n";
1513                 ++$error;
1514                 next;
1515             }
1516             $res->{"m$opt"} = sprintf("%.3f", $bps / (1024*1024.0));
1517         }
1518     }
1519
1520     # can't use the schema's 'requires' because of the mbps* => bps* "transforming aliases"
1521     for my $requirement (
1522         [mbps_max => 'mbps'],
1523         [mbps_rd_max => 'mbps_rd'],
1524         [mbps_wr_max => 'mbps_wr'],
1525         [miops_max => 'miops'],
1526         [miops_rd_max => 'miops_rd'],
1527         [miops_wr_max => 'miops_wr'],
1528         [bps_max_length => 'mbps_max'],
1529         [bps_rd_max_length => 'mbps_rd_max'],
1530         [bps_wr_max_length => 'mbps_wr_max'],
1531         [iops_max_length => 'iops_max'],
1532         [iops_rd_max_length => 'iops_rd_max'],
1533         [iops_wr_max_length => 'iops_wr_max']) {
1534         my ($option, $requires) = @$requirement;
1535         if ($res->{$option} && !$res->{$requires}) {
1536             warn "$option requires $requires\n";
1537             ++$error;
1538         }
1539     }
1540
1541     return undef if $error;
1542
1543     return undef if $res->{mbps_rd} && $res->{mbps};
1544     return undef if $res->{mbps_wr} && $res->{mbps};
1545     return undef if $res->{iops_rd} && $res->{iops};
1546     return undef if $res->{iops_wr} && $res->{iops};
1547
1548     if ($res->{media} && ($res->{media} eq 'cdrom')) {
1549         return undef if $res->{snapshot} || $res->{trans} || $res->{format};
1550         return undef if $res->{heads} || $res->{secs} || $res->{cyls};
1551         return undef if $res->{interface} eq 'virtio';
1552     }
1553
1554     if (my $size = $res->{size}) {
1555         return undef if !defined($res->{size} = PVE::JSONSchema::parse_size($size));
1556     }
1557
1558     return $res;
1559 }
1560
1561 sub print_drive {
1562     my ($vmid, $drive) = @_;
1563     my $data = { %$drive };
1564     delete $data->{$_} for qw(index interface);
1565     return PVE::JSONSchema::print_property_string($data, $alldrive_fmt);
1566 }
1567
1568 sub scsi_inquiry {
1569     my($fh, $noerr) = @_;
1570
1571     my $SG_IO = 0x2285;
1572     my $SG_GET_VERSION_NUM = 0x2282;
1573
1574     my $versionbuf = "\x00" x 8;
1575     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1576     if (!$ret) {
1577         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1578         return undef;
1579     }
1580     my $version = unpack("I", $versionbuf);
1581     if ($version < 30000) {
1582         die "scsi generic interface too old\n"  if !$noerr;
1583         return undef;
1584     }
1585
1586     my $buf = "\x00" x 36;
1587     my $sensebuf = "\x00" x 8;
1588     my $cmd = pack("C x3 C x1", 0x12, 36);
1589
1590     # see /usr/include/scsi/sg.h
1591     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1592
1593     my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1594                       length($sensebuf), 0, length($buf), $buf,
1595                       $cmd, $sensebuf, 6000);
1596
1597     $ret = ioctl($fh, $SG_IO, $packet);
1598     if (!$ret) {
1599         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1600         return undef;
1601     }
1602
1603     my @res = unpack($sg_io_hdr_t, $packet);
1604     if ($res[17] || $res[18]) {
1605         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1606         return undef;
1607     }
1608
1609     my $res = {};
1610     (my $byte0, my $byte1, $res->{vendor},
1611      $res->{product}, $res->{revision}) = unpack("C C x6 A8 A16 A4", $buf);
1612
1613     $res->{removable} = $byte1 & 128 ? 1 : 0;
1614     $res->{type} = $byte0 & 31;
1615
1616     return $res;
1617 }
1618
1619 sub path_is_scsi {
1620     my ($path) = @_;
1621
1622     my $fh = IO::File->new("+<$path") || return undef;
1623     my $res = scsi_inquiry($fh, 1);
1624     close($fh);
1625
1626     return $res;
1627 }
1628
1629 sub machine_type_is_q35 {
1630     my ($conf) = @_;
1631
1632     return $conf->{machine} && ($conf->{machine} =~ m/q35/) ? 1 : 0;
1633 }
1634
1635 sub print_tabletdevice_full {
1636     my ($conf) = @_;
1637
1638     my $q35 = machine_type_is_q35($conf);
1639
1640     # we use uhci for old VMs because tablet driver was buggy in older qemu
1641     my $usbbus = $q35 ? "ehci" : "uhci";
1642
1643     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1644 }
1645
1646 sub print_drivedevice_full {
1647     my ($storecfg, $conf, $vmid, $drive, $bridges) = @_;
1648
1649     my $device = '';
1650     my $maxdev = 0;
1651
1652     if ($drive->{interface} eq 'virtio') {
1653         my $pciaddr = print_pci_addr("$drive->{interface}$drive->{index}", $bridges);
1654         $device = "virtio-blk-pci,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}$pciaddr";
1655         $device .= ",iothread=iothread-$drive->{interface}$drive->{index}" if $drive->{iothread};
1656     } elsif ($drive->{interface} eq 'scsi') {
1657
1658         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1659         my $unit = $drive->{index} % $maxdev;
1660         my $devicetype = 'hd';
1661         my $path = '';
1662         if (drive_is_cdrom($drive)) {
1663             $devicetype = 'cd';
1664         } else {
1665             if ($drive->{file} =~ m|^/|) {
1666                 $path = $drive->{file};
1667                 if (my $info = path_is_scsi($path)) {
1668                     if ($info->{type} == 0 && $drive->{scsiblock}) {
1669                         $devicetype = 'block';
1670                     } elsif ($info->{type} == 1) { # tape
1671                         $devicetype = 'generic';
1672                     }
1673                 }
1674             } else {
1675                  $path = PVE::Storage::path($storecfg, $drive->{file});
1676             }
1677
1678             if($path =~ m/^iscsi\:\/\//){
1679                 $devicetype = 'generic';
1680             }
1681         }
1682
1683         if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)){
1684             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1685         } else {
1686             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0,lun=$drive->{index},drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1687         }
1688
1689     } elsif ($drive->{interface} eq 'ide'){
1690         $maxdev = 2;
1691         my $controller = int($drive->{index} / $maxdev);
1692         my $unit = $drive->{index} % $maxdev;
1693         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1694
1695         $device = "ide-$devicetype,bus=ide.$controller,unit=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1696         if ($devicetype eq 'hd' && (my $model = $drive->{model})) {
1697             $model = URI::Escape::uri_unescape($model);
1698             $device .= ",model=$model";
1699         }
1700     } elsif ($drive->{interface} eq 'sata'){
1701         my $controller = int($drive->{index} / $MAX_SATA_DISKS);
1702         my $unit = $drive->{index} % $MAX_SATA_DISKS;
1703         $device = "ide-drive,bus=ahci$controller.$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1704     } elsif ($drive->{interface} eq 'usb') {
1705         die "implement me";
1706         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1707     } else {
1708         die "unsupported interface type";
1709     }
1710
1711     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1712
1713     if (my $serial = $drive->{serial}) {
1714         $serial = URI::Escape::uri_unescape($serial);
1715         $device .= ",serial=$serial";
1716     }
1717
1718
1719     return $device;
1720 }
1721
1722 sub get_initiator_name {
1723     my $initiator;
1724
1725     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return undef;
1726     while (defined(my $line = <$fh>)) {
1727         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1728         $initiator = $1;
1729         last;
1730     }
1731     $fh->close();
1732
1733     return $initiator;
1734 }
1735
1736 sub print_drive_full {
1737     my ($storecfg, $vmid, $drive) = @_;
1738
1739     my $path;
1740     my $volid = $drive->{file};
1741     my $format;
1742
1743     if (drive_is_cdrom($drive)) {
1744         $path = get_iso_path($storecfg, $vmid, $volid);
1745     } else {
1746         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1747         if ($storeid) {
1748             $path = PVE::Storage::path($storecfg, $volid);
1749             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
1750             $format = qemu_img_format($scfg, $volname);
1751         } else {
1752             $path = $volid;
1753             $format = "raw";
1754         }
1755    }
1756
1757     my $opts = '';
1758     my @qemu_drive_options = qw(heads secs cyls trans media format cache rerror werror aio discard);
1759     foreach my $o (@qemu_drive_options) {
1760         $opts .= ",$o=$drive->{$o}" if defined($drive->{$o});
1761     }
1762
1763     # snapshot only accepts on|off
1764     if (defined($drive->{snapshot})) {
1765         my $v = $drive->{snapshot} ? 'on' : 'off';
1766         $opts .= ",snapshot=$v";
1767     }
1768
1769     foreach my $type (['', '-total'], [_rd => '-read'], [_wr => '-write']) {
1770         my ($dir, $qmpname) = @$type;
1771         if (my $v = $drive->{"mbps$dir"}) {
1772             $opts .= ",throttling.bps$qmpname=".int($v*1024*1024);
1773         }
1774         if (my $v = $drive->{"mbps${dir}_max"}) {
1775             $opts .= ",throttling.bps$qmpname-max=".int($v*1024*1024);
1776         }
1777         if (my $v = $drive->{"bps${dir}_max_length"}) {
1778             $opts .= ",throttling.bps$qmpname-max-length=$v";
1779         }
1780         if (my $v = $drive->{"iops${dir}"}) {
1781             $opts .= ",throttling.iops$qmpname=$v";
1782         }
1783         if (my $v = $drive->{"iops${dir}_max"}) {
1784             $opts .= ",throttling.iops$qmpname-max=$v";
1785         }
1786         if (my $v = $drive->{"iops${dir}_max_length"}) {
1787             $opts .= ",throttling.iops$qmpname-max-length=$v";
1788         }
1789     }
1790
1791     $opts .= ",format=$format" if $format && !$drive->{format};
1792
1793     my $cache_direct = 0;
1794
1795     if (my $cache = $drive->{cache}) {
1796         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1797     } elsif (!drive_is_cdrom($drive)) {
1798         $opts .= ",cache=none";
1799         $cache_direct = 1;
1800     }
1801
1802     # aio native works only with O_DIRECT
1803     if (!$drive->{aio}) {
1804         if($cache_direct) {
1805             $opts .= ",aio=native";
1806         } else {
1807             $opts .= ",aio=threads";
1808         }
1809     }
1810
1811     if (!drive_is_cdrom($drive)) {
1812         my $detectzeroes;
1813         if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
1814             $detectzeroes = 'off';
1815         } elsif ($drive->{discard}) {
1816             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
1817         } else {
1818             # This used to be our default with discard not being specified:
1819             $detectzeroes = 'on';
1820         }
1821         $opts .= ",detect-zeroes=$detectzeroes" if $detectzeroes;
1822     }
1823
1824     my $pathinfo = $path ? "file=$path," : '';
1825
1826     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1827 }
1828
1829 sub print_netdevice_full {
1830     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files) = @_;
1831
1832     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
1833
1834     my $device = $net->{model};
1835     if ($net->{model} eq 'virtio') {
1836          $device = 'virtio-net-pci';
1837      };
1838
1839     my $pciaddr = print_pci_addr("$netid", $bridges);
1840     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1841     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
1842         #Consider we have N queues, the number of vectors needed is 2*N + 2 (plus one config interrupt and control vq)
1843         my $vectors = $net->{queues} * 2 + 2;
1844         $tmpstr .= ",vectors=$vectors,mq=on";
1845     }
1846     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
1847
1848     if ($use_old_bios_files) {
1849         my $romfile;
1850         if ($device eq 'virtio-net-pci') {
1851             $romfile = 'pxe-virtio.rom';
1852         } elsif ($device eq 'e1000') {
1853             $romfile = 'pxe-e1000.rom';
1854         } elsif ($device eq 'ne2k') {
1855             $romfile = 'pxe-ne2k_pci.rom';
1856         } elsif ($device eq 'pcnet') {
1857             $romfile = 'pxe-pcnet.rom';
1858         } elsif ($device eq 'rtl8139') {
1859             $romfile = 'pxe-rtl8139.rom';
1860         }
1861         $tmpstr .= ",romfile=$romfile" if $romfile;
1862     }
1863
1864     return $tmpstr;
1865 }
1866
1867 sub print_netdev_full {
1868     my ($vmid, $conf, $net, $netid, $hotplug) = @_;
1869
1870     my $i = '';
1871     if ($netid =~ m/^net(\d+)$/) {
1872         $i = int($1);
1873     }
1874
1875     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
1876
1877     my $ifname = "tap${vmid}i$i";
1878
1879     # kvm uses TUNSETIFF ioctl, and that limits ifname length
1880     die "interface name '$ifname' is too long (max 15 character)\n"
1881         if length($ifname) >= 16;
1882
1883     my $vhostparam = '';
1884     $vhostparam = ',vhost=on' if $kernel_has_vhost_net && $net->{model} eq 'virtio';
1885
1886     my $vmname = $conf->{name} || "vm$vmid";
1887
1888     my $netdev = "";
1889     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
1890
1891     if ($net->{bridge}) {
1892         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script,downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1893     } else {
1894         $netdev = "type=user,id=$netid,hostname=$vmname";
1895     }
1896
1897     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
1898
1899     return $netdev;
1900 }
1901
1902
1903 sub print_cpu_device {
1904     my ($conf, $id) = @_;
1905
1906     my $kvm = $conf->{kvm} // 1;
1907     my $cpu = $kvm ? "kvm64" : "qemu64";
1908     if (my $cputype = $conf->{cpu}) {
1909         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
1910             or die "Cannot parse cpu description: $cputype\n";
1911         $cpu = $cpuconf->{cputype};
1912     }
1913
1914     my $cores = $conf->{cores} || 1;
1915
1916     my $current_core = ($id - 1) % $cores;
1917     my $current_socket = int(($id - 1 - $current_core)/$cores);
1918
1919     return "$cpu-x86_64-cpu,id=cpu$id,socket-id=$current_socket,core-id=$current_core,thread-id=0";
1920 }
1921
1922 sub drive_is_cloudinit {
1923     my ($drive) = @_;
1924     return $drive->{file} =~ m@[:/]vm-\d+-cloudinit(?:\.$QEMU_FORMAT_RE)?$@;
1925 }
1926
1927 sub drive_is_cdrom {
1928     my ($drive, $exclude_cloudinit) = @_;
1929
1930     return 0 if $exclude_cloudinit && drive_is_cloudinit($drive);
1931
1932     return $drive && $drive->{media} && ($drive->{media} eq 'cdrom');
1933
1934 }
1935
1936 sub parse_number_sets {
1937     my ($set) = @_;
1938     my $res = [];
1939     foreach my $part (split(/;/, $set)) {
1940         if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
1941             die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
1942             push @$res, [ $1, $2 ];
1943         } else {
1944             die "invalid range: $part\n";
1945         }
1946     }
1947     return $res;
1948 }
1949
1950 sub parse_numa {
1951     my ($data) = @_;
1952
1953     my $res = PVE::JSONSchema::parse_property_string($numa_fmt, $data);
1954     $res->{cpus} = parse_number_sets($res->{cpus}) if defined($res->{cpus});
1955     $res->{hostnodes} = parse_number_sets($res->{hostnodes}) if defined($res->{hostnodes});
1956     return $res;
1957 }
1958
1959 sub parse_hostpci {
1960     my ($value) = @_;
1961
1962     return undef if !$value;
1963
1964     my $res = PVE::JSONSchema::parse_property_string($hostpci_fmt, $value);
1965
1966     my @idlist = split(/;/, $res->{host});
1967     delete $res->{host};
1968     foreach my $id (@idlist) {
1969         if ($id =~ /^$PCIRE$/) {
1970             if (defined($2)) {
1971                 push @{$res->{pciid}}, { id => $1, function => $2 };
1972             } else {
1973                 my $pcidevices = lspci($1);
1974                 $res->{pciid} = $pcidevices->{$1};
1975             }
1976         } else {
1977             # should have been caught by parse_property_string already
1978             die "failed to parse PCI id: $id\n";
1979         }
1980     }
1981     return $res;
1982 }
1983
1984 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
1985 sub parse_net {
1986     my ($data) = @_;
1987
1988     my $res = eval { PVE::JSONSchema::parse_property_string($net_fmt, $data) };
1989     if ($@) {
1990         warn $@;
1991         return undef;
1992     }
1993     if (!defined($res->{macaddr})) {
1994         my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
1995         $res->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix});
1996     }
1997     return $res;
1998 }
1999
2000 # ipconfigX ip=cidr,gw=ip,ip6=cidr,gw6=ip
2001 sub parse_ipconfig {
2002     my ($data) = @_;
2003
2004     my $res = eval { PVE::JSONSchema::parse_property_string($ipconfig_fmt, $data) };
2005     if ($@) {
2006         warn $@;
2007         return undef;
2008     }
2009
2010     if ($res->{gw} && !$res->{ip}) {
2011         warn 'gateway specified without specifying an IP address';
2012         return undef;
2013     }
2014     if ($res->{gw6} && !$res->{ip6}) {
2015         warn 'IPv6 gateway specified without specifying an IPv6 address';
2016         return undef;
2017     }
2018     if ($res->{gw} && $res->{ip} eq 'dhcp') {
2019         warn 'gateway specified together with DHCP';
2020         return undef;
2021     }
2022     if ($res->{gw6} && $res->{ip6} !~ /^$IPV6RE/) {
2023         # gw6 + auto/dhcp
2024         warn "IPv6 gateway specified together with $res->{ip6} address";
2025         return undef;
2026     }
2027
2028     if (!$res->{ip} && !$res->{ip6}) {
2029         return { ip => 'dhcp', ip6 => 'dhcp' };
2030     }
2031
2032     return $res;
2033 }
2034
2035 sub print_net {
2036     my $net = shift;
2037
2038     return PVE::JSONSchema::print_property_string($net, $net_fmt);
2039 }
2040
2041 sub add_random_macs {
2042     my ($settings) = @_;
2043
2044     foreach my $opt (keys %$settings) {
2045         next if $opt !~ m/^net(\d+)$/;
2046         my $net = parse_net($settings->{$opt});
2047         next if !$net;
2048         $settings->{$opt} = print_net($net);
2049     }
2050 }
2051
2052 sub vm_is_volid_owner {
2053     my ($storecfg, $vmid, $volid) = @_;
2054
2055     if ($volid !~  m|^/|) {
2056         my ($path, $owner);
2057         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
2058         if ($owner && ($owner == $vmid)) {
2059             return 1;
2060         }
2061     }
2062
2063     return undef;
2064 }
2065
2066 sub split_flagged_list {
2067     my $text = shift || '';
2068     $text =~ s/[,;]/ /g;
2069     $text =~ s/^\s+//;
2070     return { map { /^(!?)(.*)$/ && ($2, $1) } ($text =~ /\S+/g) };
2071 }
2072
2073 sub join_flagged_list {
2074     my ($how, $lst) = @_;
2075     join $how, map { $lst->{$_} . $_ } keys %$lst;
2076 }
2077
2078 sub vmconfig_delete_pending_option {
2079     my ($conf, $key, $force) = @_;
2080
2081     delete $conf->{pending}->{$key};
2082     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
2083     $pending_delete_hash->{$key} = $force ? '!' : '';
2084     $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
2085 }
2086
2087 sub vmconfig_undelete_pending_option {
2088     my ($conf, $key) = @_;
2089
2090     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
2091     delete $pending_delete_hash->{$key};
2092
2093     if (%$pending_delete_hash) {
2094         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
2095     } else {
2096         delete $conf->{pending}->{delete};
2097     }
2098 }
2099
2100 sub vmconfig_register_unused_drive {
2101     my ($storecfg, $vmid, $conf, $drive) = @_;
2102
2103     if (drive_is_cloudinit($drive)) {
2104         eval { PVE::Storage::vdisk_free($storecfg, $drive->{file}) };
2105         warn $@ if $@;
2106     } elsif (!drive_is_cdrom($drive)) {
2107         my $volid = $drive->{file};
2108         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
2109             PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
2110         }
2111     }
2112 }
2113
2114 sub vmconfig_cleanup_pending {
2115     my ($conf) = @_;
2116
2117     # remove pending changes when nothing changed
2118     my $changes;
2119     foreach my $opt (keys %{$conf->{pending}}) {
2120         if (defined($conf->{$opt}) && ($conf->{pending}->{$opt} eq  $conf->{$opt})) {
2121             $changes = 1;
2122             delete $conf->{pending}->{$opt};
2123         }
2124     }
2125
2126     my $current_delete_hash = split_flagged_list($conf->{pending}->{delete});
2127     my $pending_delete_hash = {};
2128     while (my ($opt, $force) = each %$current_delete_hash) {
2129         if (defined($conf->{$opt})) {
2130             $pending_delete_hash->{$opt} = $force;
2131         } else {
2132             $changes = 1;
2133         }
2134     }
2135
2136     if (%$pending_delete_hash) {
2137         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
2138     } else {
2139         delete $conf->{pending}->{delete};
2140     }
2141
2142     return $changes;
2143 }
2144
2145 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str]
2146 my $smbios1_fmt = {
2147     uuid => {
2148         type => 'string',
2149         pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
2150         format_description => 'UUID',
2151         description => "Set SMBIOS1 UUID.",
2152         optional => 1,
2153     },
2154     version => {
2155         type => 'string',
2156         pattern => '\S+',
2157         format_description => 'string',
2158         description => "Set SMBIOS1 version.",
2159         optional => 1,
2160     },
2161     serial => {
2162         type => 'string',
2163         pattern => '\S+',
2164         format_description => 'string',
2165         description => "Set SMBIOS1 serial number.",
2166         optional => 1,
2167     },
2168     manufacturer => {
2169         type => 'string',
2170         pattern => '\S+',
2171         format_description => 'string',
2172         description => "Set SMBIOS1 manufacturer.",
2173         optional => 1,
2174     },
2175     product => {
2176         type => 'string',
2177         pattern => '\S+',
2178         format_description => 'string',
2179         description => "Set SMBIOS1 product ID.",
2180         optional => 1,
2181     },
2182     sku => {
2183         type => 'string',
2184         pattern => '\S+',
2185         format_description => 'string',
2186         description => "Set SMBIOS1 SKU string.",
2187         optional => 1,
2188     },
2189     family => {
2190         type => 'string',
2191         pattern => '\S+',
2192         format_description => 'string',
2193         description => "Set SMBIOS1 family string.",
2194         optional => 1,
2195     },
2196 };
2197
2198 sub parse_smbios1 {
2199     my ($data) = @_;
2200
2201     my $res = eval { PVE::JSONSchema::parse_property_string($smbios1_fmt, $data) };
2202     warn $@ if $@;
2203     return $res;
2204 }
2205
2206 sub print_smbios1 {
2207     my ($smbios1) = @_;
2208     return PVE::JSONSchema::print_property_string($smbios1, $smbios1_fmt);
2209 }
2210
2211 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_fmt);
2212
2213 PVE::JSONSchema::register_format('pve-qm-bootdisk', \&verify_bootdisk);
2214 sub verify_bootdisk {
2215     my ($value, $noerr) = @_;
2216
2217     return $value if is_valid_drivename($value);
2218
2219     return undef if $noerr;
2220
2221     die "invalid boot disk '$value'\n";
2222 }
2223
2224 sub parse_watchdog {
2225     my ($value) = @_;
2226
2227     return undef if !$value;
2228
2229     my $res = eval { PVE::JSONSchema::parse_property_string($watchdog_fmt, $value) };
2230     warn $@ if $@;
2231     return $res;
2232 }
2233
2234 sub parse_guest_agent {
2235     my ($value) = @_;
2236
2237     return {} if !defined($value->{agent});
2238
2239     my $res = eval { PVE::JSONSchema::parse_property_string($agent_fmt, $value->{agent}) };
2240     warn $@ if $@;
2241
2242     # if the agent is disabled ignore the other potentially set properties
2243     return {} if !$res->{enabled};
2244     return $res;
2245 }
2246
2247 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
2248 sub verify_usb_device {
2249     my ($value, $noerr) = @_;
2250
2251     return $value if parse_usb_device($value);
2252
2253     return undef if $noerr;
2254
2255     die "unable to parse usb device\n";
2256 }
2257
2258 # add JSON properties for create and set function
2259 sub json_config_properties {
2260     my $prop = shift;
2261
2262     foreach my $opt (keys %$confdesc) {
2263         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate' || $opt eq 'runningmachine';
2264         $prop->{$opt} = $confdesc->{$opt};
2265     }
2266
2267     return $prop;
2268 }
2269
2270 # return copy of $confdesc_cloudinit to generate documentation
2271 sub cloudinit_config_properties {
2272
2273     return dclone($confdesc_cloudinit);
2274 }
2275
2276 sub check_type {
2277     my ($key, $value) = @_;
2278
2279     die "unknown setting '$key'\n" if !$confdesc->{$key};
2280
2281     my $type = $confdesc->{$key}->{type};
2282
2283     if (!defined($value)) {
2284         die "got undefined value\n";
2285     }
2286
2287     if ($value =~ m/[\n\r]/) {
2288         die "property contains a line feed\n";
2289     }
2290
2291     if ($type eq 'boolean') {
2292         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
2293         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
2294         die "type check ('boolean') failed - got '$value'\n";
2295     } elsif ($type eq 'integer') {
2296         return int($1) if $value =~ m/^(\d+)$/;
2297         die "type check ('integer') failed - got '$value'\n";
2298     } elsif ($type eq 'number') {
2299         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
2300         die "type check ('number') failed - got '$value'\n";
2301     } elsif ($type eq 'string') {
2302         if (my $fmt = $confdesc->{$key}->{format}) {
2303             PVE::JSONSchema::check_format($fmt, $value);
2304             return $value;
2305         }
2306         $value =~ s/^\"(.*)\"$/$1/;
2307         return $value;
2308     } else {
2309         die "internal error"
2310     }
2311 }
2312
2313 sub check_iommu_support{
2314     #fixme : need to check IOMMU support
2315     #http://www.linux-kvm.org/page/How_to_assign_devices_with_VT-d_in_KVM
2316
2317     my $iommu=1;
2318     return $iommu;
2319
2320 }
2321
2322 sub touch_config {
2323     my ($vmid) = @_;
2324
2325     my $conf = PVE::QemuConfig->config_file($vmid);
2326     utime undef, undef, $conf;
2327 }
2328
2329 sub destroy_vm {
2330     my ($storecfg, $vmid, $keep_empty_config, $skiplock) = @_;
2331
2332     my $conffile = PVE::QemuConfig->config_file($vmid);
2333
2334     my $conf = PVE::QemuConfig->load_config($vmid);
2335
2336     PVE::QemuConfig->check_lock($conf) if !$skiplock;
2337
2338     if ($conf->{template}) {
2339         # check if any base image is still used by a linked clone
2340         foreach_drive($conf, sub {
2341                 my ($ds, $drive) = @_;
2342
2343                 return if drive_is_cdrom($drive);
2344
2345                 my $volid = $drive->{file};
2346
2347                 return if !$volid || $volid =~ m|^/|;
2348
2349                 die "base volume '$volid' is still in use by linked cloned\n"
2350                     if PVE::Storage::volume_is_base_and_used($storecfg, $volid);
2351
2352         });
2353     }
2354
2355     # only remove disks owned by this VM
2356     foreach_drive($conf, sub {
2357         my ($ds, $drive) = @_;
2358
2359         return if drive_is_cdrom($drive, 1);
2360
2361         my $volid = $drive->{file};
2362
2363         return if !$volid || $volid =~ m|^/|;
2364
2365         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
2366         return if !$path || !$owner || ($owner != $vmid);
2367
2368         eval {
2369             PVE::Storage::vdisk_free($storecfg, $volid);
2370         };
2371         warn "Could not remove disk '$volid', check manually: $@" if $@;
2372
2373     });
2374
2375     if ($keep_empty_config) {
2376         PVE::Tools::file_set_contents($conffile, "memory: 128\n");
2377     } else {
2378         unlink $conffile;
2379     }
2380
2381     # also remove unused disk
2382     eval {
2383         my $dl = PVE::Storage::vdisk_list($storecfg, undef, $vmid);
2384
2385         eval {
2386             PVE::Storage::foreach_volid($dl, sub {
2387                 my ($volid, $sid, $volname, $d) = @_;
2388                 PVE::Storage::vdisk_free($storecfg, $volid);
2389             });
2390         };
2391         warn $@ if $@;
2392
2393     };
2394     warn $@ if $@;
2395 }
2396
2397 sub parse_vm_config {
2398     my ($filename, $raw) = @_;
2399
2400     return undef if !defined($raw);
2401
2402     my $res = {
2403         digest => Digest::SHA::sha1_hex($raw),
2404         snapshots => {},
2405         pending => {},
2406     };
2407
2408     $filename =~ m|/qemu-server/(\d+)\.conf$|
2409         || die "got strange filename '$filename'";
2410
2411     my $vmid = $1;
2412
2413     my $conf = $res;
2414     my $descr;
2415     my $section = '';
2416
2417     my @lines = split(/\n/, $raw);
2418     foreach my $line (@lines) {
2419         next if $line =~ m/^\s*$/;
2420
2421         if ($line =~ m/^\[PENDING\]\s*$/i) {
2422             $section = 'pending';
2423             if (defined($descr)) {
2424                 $descr =~ s/\s+$//;
2425                 $conf->{description} = $descr;
2426             }
2427             $descr = undef;
2428             $conf = $res->{$section} = {};
2429             next;
2430
2431         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2432             $section = $1;
2433             if (defined($descr)) {
2434                 $descr =~ s/\s+$//;
2435                 $conf->{description} = $descr;
2436             }
2437             $descr = undef;
2438             $conf = $res->{snapshots}->{$section} = {};
2439             next;
2440         }
2441
2442         if ($line =~ m/^\#(.*)\s*$/) {
2443             $descr = '' if !defined($descr);
2444             $descr .= PVE::Tools::decode_text($1) . "\n";
2445             next;
2446         }
2447
2448         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2449             $descr = '' if !defined($descr);
2450             $descr .= PVE::Tools::decode_text($2);
2451         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2452             $conf->{snapstate} = $1;
2453         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2454             my $key = $1;
2455             my $value = $2;
2456             $conf->{$key} = $value;
2457         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2458             my $value = $1;
2459             if ($section eq 'pending') {
2460                 $conf->{delete} = $value; # we parse this later
2461             } else {
2462                 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
2463             }
2464         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(.+?)\s*$/) {
2465             my $key = $1;
2466             my $value = $2;
2467             eval { $value = check_type($key, $value); };
2468             if ($@) {
2469                 warn "vm $vmid - unable to parse value of '$key' - $@";
2470             } else {
2471                 $key = 'ide2' if $key eq 'cdrom';
2472                 my $fmt = $confdesc->{$key}->{format};
2473                 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2474                     my $v = parse_drive($key, $value);
2475                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2476                         $v->{file} = $volid;
2477                         $value = print_drive($vmid, $v);
2478                     } else {
2479                         warn "vm $vmid - unable to parse value of '$key'\n";
2480                         next;
2481                     }
2482                 }
2483
2484                 $conf->{$key} = $value;
2485             }
2486         }
2487     }
2488
2489     if (defined($descr)) {
2490         $descr =~ s/\s+$//;
2491         $conf->{description} = $descr;
2492     }
2493     delete $res->{snapstate}; # just to be sure
2494
2495     return $res;
2496 }
2497
2498 sub write_vm_config {
2499     my ($filename, $conf) = @_;
2500
2501     delete $conf->{snapstate}; # just to be sure
2502
2503     if ($conf->{cdrom}) {
2504         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2505         $conf->{ide2} = $conf->{cdrom};
2506         delete $conf->{cdrom};
2507     }
2508
2509     # we do not use 'smp' any longer
2510     if ($conf->{sockets}) {
2511         delete $conf->{smp};
2512     } elsif ($conf->{smp}) {
2513         $conf->{sockets} = $conf->{smp};
2514         delete $conf->{cores};
2515         delete $conf->{smp};
2516     }
2517
2518     my $used_volids = {};
2519
2520     my $cleanup_config = sub {
2521         my ($cref, $pending, $snapname) = @_;
2522
2523         foreach my $key (keys %$cref) {
2524             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2525                 $key eq 'snapstate' || $key eq 'pending';
2526             my $value = $cref->{$key};
2527             if ($key eq 'delete') {
2528                 die "propertry 'delete' is only allowed in [PENDING]\n"
2529                     if !$pending;
2530                 # fixme: check syntax?
2531                 next;
2532             }
2533             eval { $value = check_type($key, $value); };
2534             die "unable to parse value of '$key' - $@" if $@;
2535
2536             $cref->{$key} = $value;
2537
2538             if (!$snapname && is_valid_drivename($key)) {
2539                 my $drive = parse_drive($key, $value);
2540                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2541             }
2542         }
2543     };
2544
2545     &$cleanup_config($conf);
2546
2547     &$cleanup_config($conf->{pending}, 1);
2548
2549     foreach my $snapname (keys %{$conf->{snapshots}}) {
2550         die "internal error" if $snapname eq 'pending';
2551         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2552     }
2553
2554     # remove 'unusedX' settings if we re-add a volume
2555     foreach my $key (keys %$conf) {
2556         my $value = $conf->{$key};
2557         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2558             delete $conf->{$key};
2559         }
2560     }
2561
2562     my $generate_raw_config = sub {
2563         my ($conf, $pending) = @_;
2564
2565         my $raw = '';
2566
2567         # add description as comment to top of file
2568         if (defined(my $descr = $conf->{description})) {
2569             if ($descr) {
2570                 foreach my $cl (split(/\n/, $descr)) {
2571                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2572                 }
2573             } else {
2574                 $raw .= "#\n" if $pending;
2575             }
2576         }
2577
2578         foreach my $key (sort keys %$conf) {
2579             next if $key eq 'digest' || $key eq 'description' || $key eq 'pending' || $key eq 'snapshots';
2580             $raw .= "$key: $conf->{$key}\n";
2581         }
2582         return $raw;
2583     };
2584
2585     my $raw = &$generate_raw_config($conf);
2586
2587     if (scalar(keys %{$conf->{pending}})){
2588         $raw .= "\n[PENDING]\n";
2589         $raw .= &$generate_raw_config($conf->{pending}, 1);
2590     }
2591
2592     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2593         $raw .= "\n[$snapname]\n";
2594         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2595     }
2596
2597     return $raw;
2598 }
2599
2600 sub load_defaults {
2601
2602     my $res = {};
2603
2604     # we use static defaults from our JSON schema configuration
2605     foreach my $key (keys %$confdesc) {
2606         if (defined(my $default = $confdesc->{$key}->{default})) {
2607             $res->{$key} = $default;
2608         }
2609     }
2610
2611     return $res;
2612 }
2613
2614 sub config_list {
2615     my $vmlist = PVE::Cluster::get_vmlist();
2616     my $res = {};
2617     return $res if !$vmlist || !$vmlist->{ids};
2618     my $ids = $vmlist->{ids};
2619
2620     foreach my $vmid (keys %$ids) {
2621         my $d = $ids->{$vmid};
2622         next if !$d->{node} || $d->{node} ne $nodename;
2623         next if !$d->{type} || $d->{type} ne 'qemu';
2624         $res->{$vmid}->{exists} = 1;
2625     }
2626     return $res;
2627 }
2628
2629 # test if VM uses local resources (to prevent migration)
2630 sub check_local_resources {
2631     my ($conf, $noerr) = @_;
2632
2633     my $loc_res = 0;
2634
2635     $loc_res = 1 if $conf->{hostusb}; # old syntax
2636     $loc_res = 1 if $conf->{hostpci}; # old syntax
2637
2638     foreach my $k (keys %$conf) {
2639         next if $k =~ m/^usb/ && ($conf->{$k} eq 'spice');
2640         # sockets are safe: they will recreated be on the target side post-migrate
2641         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2642         $loc_res = 1 if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2643     }
2644
2645     die "VM uses local resources\n" if $loc_res && !$noerr;
2646
2647     return $loc_res;
2648 }
2649
2650 # check if used storages are available on all nodes (use by migrate)
2651 sub check_storage_availability {
2652     my ($storecfg, $conf, $node) = @_;
2653
2654     foreach_drive($conf, sub {
2655         my ($ds, $drive) = @_;
2656
2657         my $volid = $drive->{file};
2658         return if !$volid;
2659
2660         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2661         return if !$sid;
2662
2663         # check if storage is available on both nodes
2664         my $scfg = PVE::Storage::storage_check_node($storecfg, $sid);
2665         PVE::Storage::storage_check_node($storecfg, $sid, $node);
2666    });
2667 }
2668
2669 # list nodes where all VM images are available (used by has_feature API)
2670 sub shared_nodes {
2671     my ($conf, $storecfg) = @_;
2672
2673     my $nodelist = PVE::Cluster::get_nodelist();
2674     my $nodehash = { map { $_ => 1 } @$nodelist };
2675     my $nodename = PVE::INotify::nodename();
2676
2677     foreach_drive($conf, sub {
2678         my ($ds, $drive) = @_;
2679
2680         my $volid = $drive->{file};
2681         return if !$volid;
2682
2683         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2684         if ($storeid) {
2685             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2686             if ($scfg->{disable}) {
2687                 $nodehash = {};
2688             } elsif (my $avail = $scfg->{nodes}) {
2689                 foreach my $node (keys %$nodehash) {
2690                     delete $nodehash->{$node} if !$avail->{$node};
2691                 }
2692             } elsif (!$scfg->{shared}) {
2693                 foreach my $node (keys %$nodehash) {
2694                     delete $nodehash->{$node} if $node ne $nodename
2695                 }
2696             }
2697         }
2698     });
2699
2700     return $nodehash
2701 }
2702
2703 sub check_cmdline {
2704     my ($pidfile, $pid) = @_;
2705
2706     my $fh = IO::File->new("/proc/$pid/cmdline", "r");
2707     if (defined($fh)) {
2708         my $line = <$fh>;
2709         $fh->close;
2710         return undef if !$line;
2711         my @param = split(/\0/, $line);
2712
2713         my $cmd = $param[0];
2714         return if !$cmd || ($cmd !~ m|kvm$| && $cmd !~ m|qemu-system-x86_64$|);
2715
2716         for (my $i = 0; $i < scalar (@param); $i++) {
2717             my $p = $param[$i];
2718             next if !$p;
2719             if (($p eq '-pidfile') || ($p eq '--pidfile')) {
2720                 my $p = $param[$i+1];
2721                 return 1 if $p && ($p eq $pidfile);
2722                 return undef;
2723             }
2724         }
2725     }
2726     return undef;
2727 }
2728
2729 sub check_running {
2730     my ($vmid, $nocheck, $node) = @_;
2731
2732     my $filename = PVE::QemuConfig->config_file($vmid, $node);
2733
2734     die "unable to find configuration file for VM $vmid - no such machine\n"
2735         if !$nocheck && ! -f $filename;
2736
2737     my $pidfile = pidfile_name($vmid);
2738
2739     if (my $fd = IO::File->new("<$pidfile")) {
2740         my $st = stat($fd);
2741         my $line = <$fd>;
2742         close($fd);
2743
2744         my $mtime = $st->mtime;
2745         if ($mtime > time()) {
2746             warn "file '$filename' modified in future\n";
2747         }
2748
2749         if ($line =~ m/^(\d+)$/) {
2750             my $pid = $1;
2751             if (check_cmdline($pidfile, $pid)) {
2752                 if (my $pinfo = PVE::ProcFSTools::check_process_running($pid)) {
2753                     return $pid;
2754                 }
2755             }
2756         }
2757     }
2758
2759     return undef;
2760 }
2761
2762 sub vzlist {
2763
2764     my $vzlist = config_list();
2765
2766     my $fd = IO::Dir->new($var_run_tmpdir) || return $vzlist;
2767
2768     while (defined(my $de = $fd->read)) {
2769         next if $de !~ m/^(\d+)\.pid$/;
2770         my $vmid = $1;
2771         next if !defined($vzlist->{$vmid});
2772         if (my $pid = check_running($vmid)) {
2773             $vzlist->{$vmid}->{pid} = $pid;
2774         }
2775     }
2776
2777     return $vzlist;
2778 }
2779
2780 sub disksize {
2781     my ($storecfg, $conf) = @_;
2782
2783     my $bootdisk = $conf->{bootdisk};
2784     return undef if !$bootdisk;
2785     return undef if !is_valid_drivename($bootdisk);
2786
2787     return undef if !$conf->{$bootdisk};
2788
2789     my $drive = parse_drive($bootdisk, $conf->{$bootdisk});
2790     return undef if !defined($drive);
2791
2792     return undef if drive_is_cdrom($drive);
2793
2794     my $volid = $drive->{file};
2795     return undef if !$volid;
2796
2797     return $drive->{size};
2798 }
2799
2800 our $vmstatus_return_properties = {
2801     vmid => get_standard_option('pve-vmid'),
2802     status => {
2803         description => "Qemu process status.",
2804         type => 'string',
2805         enum => ['stopped', 'running'],
2806     },
2807     maxmem => {
2808         description => "Maximum memory in bytes.",
2809         type => 'integer',
2810         optional => 1,
2811         renderer => 'bytes',
2812     },
2813     maxdisk => {
2814         description => "Root disk size in bytes.",
2815         type => 'integer',
2816         optional => 1,
2817         renderer => 'bytes',
2818     },
2819     name => {
2820         description => "VM name.",
2821         type => 'string',
2822         optional => 1,
2823     },
2824     qmpstatus => {
2825         description => "Qemu QMP agent status.",
2826         type => 'string',
2827         optional => 1,
2828     },
2829     pid => {
2830         description => "PID of running qemu process.",
2831         type => 'integer',
2832         optional => 1,
2833     },
2834     uptime => {
2835         description => "Uptime.",
2836         type => 'integer',
2837         optional => 1,
2838         renderer => 'duration',
2839     },
2840     cpus => {
2841         description => "Maximum usable CPUs.",
2842         type => 'number',
2843         optional => 1,
2844     },
2845 };
2846
2847 my $last_proc_pid_stat;
2848
2849 # get VM status information
2850 # This must be fast and should not block ($full == false)
2851 # We only query KVM using QMP if $full == true (this can be slow)
2852 sub vmstatus {
2853     my ($opt_vmid, $full) = @_;
2854
2855     my $res = {};
2856
2857     my $storecfg = PVE::Storage::config();
2858
2859     my $list = vzlist();
2860     my $defaults = load_defaults();
2861
2862     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
2863
2864     my $cpucount = $cpuinfo->{cpus} || 1;
2865
2866     foreach my $vmid (keys %$list) {
2867         next if $opt_vmid && ($vmid ne $opt_vmid);
2868
2869         my $cfspath = PVE::QemuConfig->cfs_config_path($vmid);
2870         my $conf = PVE::Cluster::cfs_read_file($cfspath) || {};
2871
2872         my $d = { vmid => $vmid };
2873         $d->{pid} = $list->{$vmid}->{pid};
2874
2875         # fixme: better status?
2876         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
2877
2878         my $size = disksize($storecfg, $conf);
2879         if (defined($size)) {
2880             $d->{disk} = 0; # no info available
2881             $d->{maxdisk} = $size;
2882         } else {
2883             $d->{disk} = 0;
2884             $d->{maxdisk} = 0;
2885         }
2886
2887         $d->{cpus} = ($conf->{sockets} || $defaults->{sockets})
2888             * ($conf->{cores} || $defaults->{cores});
2889         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
2890         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
2891
2892         $d->{name} = $conf->{name} || "VM $vmid";
2893         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024)
2894             : $defaults->{memory}*(1024*1024);
2895
2896         if ($conf->{balloon}) {
2897             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
2898             $d->{shares} = defined($conf->{shares}) ? $conf->{shares}
2899                 : $defaults->{shares};
2900         }
2901
2902         $d->{uptime} = 0;
2903         $d->{cpu} = 0;
2904         $d->{mem} = 0;
2905
2906         $d->{netout} = 0;
2907         $d->{netin} = 0;
2908
2909         $d->{diskread} = 0;
2910         $d->{diskwrite} = 0;
2911
2912         $d->{template} = PVE::QemuConfig->is_template($conf);
2913
2914         $d->{serial} = 1 if conf_has_serial($conf);
2915
2916         $res->{$vmid} = $d;
2917     }
2918
2919     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
2920     foreach my $dev (keys %$netdev) {
2921         next if $dev !~ m/^tap([1-9]\d*)i/;
2922         my $vmid = $1;
2923         my $d = $res->{$vmid};
2924         next if !$d;
2925
2926         $d->{netout} += $netdev->{$dev}->{receive};
2927         $d->{netin} += $netdev->{$dev}->{transmit};
2928
2929         if ($full) {
2930             $d->{nics}->{$dev}->{netout} = $netdev->{$dev}->{receive};
2931             $d->{nics}->{$dev}->{netin} = $netdev->{$dev}->{transmit};
2932         }
2933
2934     }
2935
2936     my $ctime = gettimeofday;
2937
2938     foreach my $vmid (keys %$list) {
2939
2940         my $d = $res->{$vmid};
2941         my $pid = $d->{pid};
2942         next if !$pid;
2943
2944         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
2945         next if !$pstat; # not running
2946
2947         my $used = $pstat->{utime} + $pstat->{stime};
2948
2949         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
2950
2951         if ($pstat->{vsize}) {
2952             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
2953         }
2954
2955         my $old = $last_proc_pid_stat->{$pid};
2956         if (!$old) {
2957             $last_proc_pid_stat->{$pid} = {
2958                 time => $ctime,
2959                 used => $used,
2960                 cpu => 0,
2961             };
2962             next;
2963         }
2964
2965         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
2966
2967         if ($dtime > 1000) {
2968             my $dutime = $used -  $old->{used};
2969
2970             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
2971             $last_proc_pid_stat->{$pid} = {
2972                 time => $ctime,
2973                 used => $used,
2974                 cpu => $d->{cpu},
2975             };
2976         } else {
2977             $d->{cpu} = $old->{cpu};
2978         }
2979     }
2980
2981     return $res if !$full;
2982
2983     my $qmpclient = PVE::QMPClient->new();
2984
2985     my $ballooncb = sub {
2986         my ($vmid, $resp) = @_;
2987
2988         my $info = $resp->{'return'};
2989         return if !$info->{max_mem};
2990
2991         my $d = $res->{$vmid};
2992
2993         # use memory assigned to VM
2994         $d->{maxmem} = $info->{max_mem};
2995         $d->{balloon} = $info->{actual};
2996
2997         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
2998             $d->{mem} = $info->{total_mem} - $info->{free_mem};
2999             $d->{freemem} = $info->{free_mem};
3000         }
3001
3002         $d->{ballooninfo} = $info;
3003     };
3004
3005     my $blockstatscb = sub {
3006         my ($vmid, $resp) = @_;
3007         my $data = $resp->{'return'} || [];
3008         my $totalrdbytes = 0;
3009         my $totalwrbytes = 0;
3010
3011         for my $blockstat (@$data) {
3012             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
3013             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
3014
3015             $blockstat->{device} =~ s/drive-//;
3016             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
3017         }
3018         $res->{$vmid}->{diskread} = $totalrdbytes;
3019         $res->{$vmid}->{diskwrite} = $totalwrbytes;
3020     };
3021
3022     my $statuscb = sub {
3023         my ($vmid, $resp) = @_;
3024
3025         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
3026         # this fails if ballon driver is not loaded, so this must be
3027         # the last commnand (following command are aborted if this fails).
3028         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
3029
3030         my $status = 'unknown';
3031         if (!defined($status = $resp->{'return'}->{status})) {
3032             warn "unable to get VM status\n";
3033             return;
3034         }
3035
3036         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
3037     };
3038
3039     foreach my $vmid (keys %$list) {
3040         next if $opt_vmid && ($vmid ne $opt_vmid);
3041         next if !$res->{$vmid}->{pid}; # not running
3042         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
3043     }
3044
3045     $qmpclient->queue_execute(undef, 2);
3046
3047     foreach my $vmid (keys %$list) {
3048         next if $opt_vmid && ($vmid ne $opt_vmid);
3049         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
3050     }
3051
3052     return $res;
3053 }
3054
3055 sub foreach_drive {
3056     my ($conf, $func, @param) = @_;
3057
3058     foreach my $ds (valid_drive_names()) {
3059         next if !defined($conf->{$ds});
3060
3061         my $drive = parse_drive($ds, $conf->{$ds});
3062         next if !$drive;
3063
3064         &$func($ds, $drive, @param);
3065     }
3066 }
3067
3068 sub foreach_volid {
3069     my ($conf, $func, @param) = @_;
3070
3071     my $volhash = {};
3072
3073     my $test_volid = sub {
3074         my ($volid, $is_cdrom, $replicate, $shared, $snapname) = @_;
3075
3076         return if !$volid;
3077
3078         $volhash->{$volid}->{cdrom} //= 1;
3079         $volhash->{$volid}->{cdrom} = 0 if !$is_cdrom;
3080
3081         $volhash->{$volid}->{replicate} //= 0;
3082         $volhash->{$volid}->{replicate} = 1 if $replicate;
3083
3084         $volhash->{$volid}->{shared} //= 0;
3085         $volhash->{$volid}->{shared} = 1 if $shared;
3086
3087         $volhash->{$volid}->{referenced_in_config} //= 0;
3088         $volhash->{$volid}->{referenced_in_config} = 1 if !defined($snapname);
3089
3090         $volhash->{$volid}->{referenced_in_snapshot}->{$snapname} = 1
3091             if defined($snapname);
3092     };
3093
3094     foreach_drive($conf, sub {
3095         my ($ds, $drive) = @_;
3096         $test_volid->($drive->{file}, drive_is_cdrom($drive), $drive->{replicate} // 1, $drive->{shared}, undef);
3097     });
3098
3099     foreach my $snapname (keys %{$conf->{snapshots}}) {
3100         my $snap = $conf->{snapshots}->{$snapname};
3101         $test_volid->($snap->{vmstate}, 0, 1, $snapname);
3102         foreach_drive($snap, sub {
3103             my ($ds, $drive) = @_;
3104             $test_volid->($drive->{file}, drive_is_cdrom($drive), $drive->{replicate} // 1, $drive->{shared}, $snapname);
3105         });
3106     }
3107
3108     foreach my $volid (keys %$volhash) {
3109         &$func($volid, $volhash->{$volid}, @param);
3110     }
3111 }
3112
3113 sub conf_has_serial {
3114     my ($conf) = @_;
3115
3116     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3117         if ($conf->{"serial$i"}) {
3118             return 1;
3119         }
3120     }
3121
3122     return 0;
3123 }
3124
3125 sub vga_conf_has_spice {
3126     my ($vga) = @_;
3127
3128     return 0 if !$vga || $vga !~ m/^qxl([234])?$/;
3129
3130     return $1 || 1;
3131 }
3132
3133 sub config_to_command {
3134     my ($storecfg, $vmid, $conf, $defaults, $forcemachine) = @_;
3135
3136     my $cmd = [];
3137     my $globalFlags = [];
3138     my $machineFlags = [];
3139     my $rtcFlags = [];
3140     my $cpuFlags = [];
3141     my $devices = [];
3142     my $pciaddr = '';
3143     my $bridges = {};
3144     my $kvmver = kvm_user_version();
3145     my $vernum = 0; # unknown
3146     my $ostype = $conf->{ostype};
3147     my $winversion = windows_version($ostype);
3148     my $kvm = $conf->{kvm} // 1;
3149
3150     die "KVM virtualisation configured, but not available. Either disable in VM configuration or enable in BIOS.\n" if (!$cpuinfo->{hvm} && $kvm);
3151
3152     if ($kvmver =~ m/^(\d+)\.(\d+)$/) {
3153         $vernum = $1*1000000+$2*1000;
3154     } elsif ($kvmver =~ m/^(\d+)\.(\d+)\.(\d+)$/) {
3155         $vernum = $1*1000000+$2*1000+$3;
3156     }
3157
3158     die "detected old qemu-kvm binary ($kvmver)\n" if $vernum < 15000;
3159
3160     my $have_ovz = -f '/proc/vz/vestat';
3161
3162     my $q35 = machine_type_is_q35($conf);
3163     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
3164     my $machine_type = $forcemachine || $conf->{machine};
3165     my $use_old_bios_files = undef;
3166     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3167
3168     my $cpuunits = defined($conf->{cpuunits}) ?
3169             $conf->{cpuunits} : $defaults->{cpuunits};
3170
3171     push @$cmd, '/usr/bin/kvm';
3172
3173     push @$cmd, '-id', $vmid;
3174
3175     my $vmname = $conf->{name} || "vm$vmid";
3176
3177     push @$cmd, '-name', $vmname;
3178
3179     my $use_virtio = 0;
3180
3181     my $qmpsocket = qmp_socket($vmid);
3182     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server,nowait";
3183     push @$cmd, '-mon', "chardev=qmp,mode=control";
3184
3185
3186     push @$cmd, '-pidfile' , pidfile_name($vmid);
3187
3188     push @$cmd, '-daemonize';
3189
3190     if ($conf->{smbios1}) {
3191         push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
3192     }
3193
3194     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3195         die "uefi base image not found\n" if ! -f $OVMF_CODE;
3196
3197         my $path;
3198         my $format;
3199         if (my $efidisk = $conf->{efidisk0}) {
3200             my $d = PVE::JSONSchema::parse_property_string($efidisk_fmt, $efidisk);
3201             my ($storeid, $volname) = PVE::Storage::parse_volume_id($d->{file}, 1);
3202             $format = $d->{format};
3203             if ($storeid) {
3204                 $path = PVE::Storage::path($storecfg, $d->{file});
3205                 if (!defined($format)) {
3206                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
3207                     $format = qemu_img_format($scfg, $volname);
3208                 }
3209             } else {
3210                 $path = $d->{file};
3211                 die "efidisk format must be specified\n"
3212                     if !defined($format);
3213             }
3214         } else {
3215             warn "no efidisk configured! Using temporary efivars disk.\n";
3216             $path = "/tmp/$vmid-ovmf.fd";
3217             PVE::Tools::file_copy($OVMF_VARS, $path, -s $OVMF_VARS);
3218             $format = 'raw';
3219         }
3220
3221         push @$cmd, '-drive', "if=pflash,unit=0,format=raw,readonly,file=$OVMF_CODE";
3222         push @$cmd, '-drive', "if=pflash,unit=1,format=$format,id=drive-efidisk0,file=$path";
3223     }
3224
3225
3226     # add usb controllers
3227     my @usbcontrollers = PVE::QemuServer::USB::get_usb_controllers($conf, $bridges, $q35, $usbdesc->{format}, $MAX_USB_DEVICES);
3228     push @$devices, @usbcontrollers if @usbcontrollers;
3229     my $vga = $conf->{vga};
3230
3231     my $qxlnum = vga_conf_has_spice($vga);
3232     $vga = 'qxl' if $qxlnum;
3233
3234     if (!$vga) {
3235         if (qemu_machine_feature_enabled($machine_type, $kvmver, 2, 9)) {
3236             $vga = (!$winversion || $winversion >= 6) ? 'std' : 'cirrus';
3237         } else {
3238             $vga = ($winversion >= 6) ? 'std' : 'cirrus';
3239         }
3240     }
3241
3242     # enable absolute mouse coordinates (needed by vnc)
3243     my $tablet;
3244     if (defined($conf->{tablet})) {
3245         $tablet = $conf->{tablet};
3246     } else {
3247         $tablet = $defaults->{tablet};
3248         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
3249         $tablet = 0 if $vga =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
3250     }
3251
3252     push @$devices, '-device', print_tabletdevice_full($conf) if $tablet;
3253
3254     my $kvm_off = 0;
3255     my $gpu_passthrough;
3256
3257     # host pci devices
3258     for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
3259         my $d = parse_hostpci($conf->{"hostpci$i"});
3260         next if !$d;
3261
3262         my $pcie = $d->{pcie};
3263         if($pcie){
3264             die "q35 machine model is not enabled" if !$q35;
3265             $pciaddr = print_pcie_addr("hostpci$i");
3266         }else{
3267             $pciaddr = print_pci_addr("hostpci$i", $bridges);
3268         }
3269
3270         my $rombar = defined($d->{rombar}) && !$d->{rombar} ? ',rombar=0' : '';
3271         my $romfile = $d->{romfile};
3272
3273         my $xvga = '';
3274         if ($d->{'x-vga'}) {
3275             $xvga = ',x-vga=on';
3276             $kvm_off = 1;
3277             $vga = 'none';
3278             $gpu_passthrough = 1;
3279
3280             if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3281                 $xvga = "";
3282             }
3283         }
3284         my $pcidevices = $d->{pciid};
3285         my $multifunction = 1 if @$pcidevices > 1;
3286
3287         my $j=0;
3288         foreach my $pcidevice (@$pcidevices) {
3289
3290             my $id = "hostpci$i";
3291             $id .= ".$j" if $multifunction;
3292             my $addr = $pciaddr;
3293             $addr .= ".$j" if $multifunction;
3294             my $devicestr = "vfio-pci,host=$pcidevice->{id}.$pcidevice->{function},id=$id$addr";
3295
3296             if($j == 0){
3297                 $devicestr .= "$rombar$xvga";
3298                 $devicestr .= ",multifunction=on" if $multifunction;
3299                 $devicestr .= ",romfile=/usr/share/kvm/$romfile" if $romfile;
3300             }
3301
3302             push @$devices, '-device', $devicestr;
3303             $j++;
3304         }
3305     }
3306
3307     # usb devices
3308     my @usbdevices = PVE::QemuServer::USB::get_usb_devices($conf, $usbdesc->{format}, $MAX_USB_DEVICES);
3309     push @$devices, @usbdevices if @usbdevices;
3310     # serial devices
3311     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3312         if (my $path = $conf->{"serial$i"}) {
3313             if ($path eq 'socket') {
3314                 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
3315                 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server,nowait";
3316                 push @$devices, '-device', "isa-serial,chardev=serial$i";
3317             } else {
3318                 die "no such serial device\n" if ! -c $path;
3319                 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
3320                 push @$devices, '-device', "isa-serial,chardev=serial$i";
3321             }
3322         }
3323     }
3324
3325     # parallel devices
3326     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
3327         if (my $path = $conf->{"parallel$i"}) {
3328             die "no such parallel device\n" if ! -c $path;
3329             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
3330             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
3331             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
3332         }
3333     }
3334
3335
3336     my $sockets = 1;
3337     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3338     $sockets = $conf->{sockets} if  $conf->{sockets};
3339
3340     my $cores = $conf->{cores} || 1;
3341
3342     my $maxcpus = $sockets * $cores;
3343
3344     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
3345
3346     my $allowed_vcpus = $cpuinfo->{cpus};
3347
3348     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n"
3349         if ($allowed_vcpus < $maxcpus);
3350
3351     if($hotplug_features->{cpu} && qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 7)) {
3352
3353         push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3354         for (my $i = 2; $i <= $vcpus; $i++)  {
3355             my $cpustr = print_cpu_device($conf,$i);
3356             push @$cmd, '-device', $cpustr;
3357         }
3358
3359     } else {
3360
3361         push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3362     }
3363     push @$cmd, '-nodefaults';
3364
3365     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
3366
3367     my $bootindex_hash = {};
3368     my $i = 1;
3369     foreach my $o (split(//, $bootorder)) {
3370         $bootindex_hash->{$o} = $i*100;
3371         $i++;
3372     }
3373
3374     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
3375
3376     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
3377
3378     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
3379
3380     push @$cmd, '-vga', $vga if $vga && $vga !~ m/^serial\d+$/; # for kvm 77 and later
3381
3382     if ($vga && $vga !~ m/^serial\d+$/ && $vga ne 'none'){
3383         my $socket = vnc_socket($vmid);
3384         push @$cmd,  '-vnc', "unix:$socket,x509,password";
3385     } else {
3386         push @$cmd, '-nographic';
3387     }
3388
3389     # time drift fix
3390     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
3391
3392     my $useLocaltime = $conf->{localtime};
3393
3394     if ($winversion >= 5) { # windows
3395         $useLocaltime = 1 if !defined($conf->{localtime});
3396
3397         # use time drift fix when acpi is enabled
3398         if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
3399             $tdf = 1 if !defined($conf->{tdf});
3400         }
3401     }
3402
3403     if ($winversion >= 6) {
3404         push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3405         push @$cmd, '-no-hpet';
3406     }
3407
3408     push @$rtcFlags, 'driftfix=slew' if $tdf;
3409
3410     if (!$kvm) {
3411         push @$machineFlags, 'accel=tcg';
3412     }
3413
3414     if ($machine_type) {
3415         push @$machineFlags, "type=${machine_type}";
3416     }
3417
3418     if ($conf->{startdate}) {
3419         push @$rtcFlags, "base=$conf->{startdate}";
3420     } elsif ($useLocaltime) {
3421         push @$rtcFlags, 'base=localtime';
3422     }
3423
3424     my $cpu = $kvm ? "kvm64" : "qemu64";
3425     if (my $cputype = $conf->{cpu}) {
3426         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
3427             or die "Cannot parse cpu description: $cputype\n";
3428         $cpu = $cpuconf->{cputype};
3429         $kvm_off = 1 if $cpuconf->{hidden};
3430
3431         if (defined(my $flags = $cpuconf->{flags})) {
3432             push @$cpuFlags, split(";", $flags);
3433         }
3434     }
3435
3436     push @$cpuFlags , '+lahf_lm' if $cpu eq 'kvm64';
3437
3438     push @$cpuFlags , '-x2apic'
3439         if $conf->{ostype} && $conf->{ostype} eq 'solaris';
3440
3441     push @$cpuFlags, '+sep' if $cpu eq 'kvm64' || $cpu eq 'kvm32';
3442
3443     push @$cpuFlags, '-rdtscp' if $cpu =~ m/^Opteron/;
3444
3445     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3446
3447         push @$cpuFlags , '+kvm_pv_unhalt' if $kvm;
3448         push @$cpuFlags , '+kvm_pv_eoi' if $kvm;
3449     }
3450
3451     add_hyperv_enlightenments($cpuFlags, $winversion, $machine_type, $kvmver, $conf->{bios}, $gpu_passthrough) if $kvm;
3452
3453     push @$cpuFlags, 'enforce' if $cpu ne 'host' && $kvm;
3454
3455     push @$cpuFlags, 'kvm=off' if $kvm_off;
3456
3457     my $cpu_vendor = $cpu_vendor_list->{$cpu} ||
3458         die "internal error"; # should not happen
3459
3460     push @$cpuFlags, "vendor=${cpu_vendor}"
3461         if $cpu_vendor ne 'default';
3462
3463     $cpu .= "," . join(',', @$cpuFlags) if scalar(@$cpuFlags);
3464
3465     push @$cmd, '-cpu', $cpu;
3466
3467     PVE::QemuServer::Memory::config($conf, $vmid, $sockets, $cores, $defaults, $hotplug_features, $cmd);
3468
3469     push @$cmd, '-S' if $conf->{freeze};
3470
3471     push @$cmd, '-k', $conf->{keyboard} if defined($conf->{keyboard});
3472
3473     # enable sound
3474     #my $soundhw = $conf->{soundhw} || $defaults->{soundhw};
3475     #push @$cmd, '-soundhw', 'es1370';
3476     #push @$cmd, '-soundhw', $soundhw if $soundhw;
3477
3478     if (parse_guest_agent($conf)->{enabled}) {
3479         my $qgasocket = qmp_socket($vmid, 1);
3480         my $pciaddr = print_pci_addr("qga0", $bridges);
3481         push @$devices, '-chardev', "socket,path=$qgasocket,server,nowait,id=qga0";
3482         push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3483         push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3484     }
3485
3486     my $spice_port;
3487
3488     if ($qxlnum) {
3489         if ($qxlnum > 1) {
3490             if ($winversion){
3491                 for(my $i = 1; $i < $qxlnum; $i++){
3492                     my $pciaddr = print_pci_addr("vga$i", $bridges);
3493                     push @$devices, '-device', "qxl,id=vga$i,ram_size=67108864,vram_size=33554432$pciaddr";
3494                 }
3495             } else {
3496                 # assume other OS works like Linux
3497                 push @$cmd, '-global', 'qxl-vga.ram_size=134217728';
3498                 push @$cmd, '-global', 'qxl-vga.vram_size=67108864';
3499             }
3500         }
3501
3502         my $pciaddr = print_pci_addr("spice", $bridges);
3503
3504         my $nodename = PVE::INotify::nodename();
3505         my $pfamily = PVE::Tools::get_host_address_family($nodename);
3506         my @nodeaddrs = PVE::Tools::getaddrinfo_all('localhost', family => $pfamily);
3507         die "failed to get an ip address of type $pfamily for 'localhost'\n" if !@nodeaddrs;
3508         my $localhost = PVE::Network::addr_to_ip($nodeaddrs[0]->{addr});
3509         $spice_port = PVE::Tools::next_spice_port($pfamily, $localhost);
3510
3511         push @$devices, '-spice', "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";
3512
3513         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3514         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3515         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3516     }
3517
3518     # enable balloon by default, unless explicitly disabled
3519     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3520         $pciaddr = print_pci_addr("balloon0", $bridges);
3521         push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
3522     }
3523
3524     if ($conf->{watchdog}) {
3525         my $wdopts = parse_watchdog($conf->{watchdog});
3526         $pciaddr = print_pci_addr("watchdog", $bridges);
3527         my $watchdog = $wdopts->{model} || 'i6300esb';
3528         push @$devices, '-device', "$watchdog$pciaddr";
3529         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3530     }
3531
3532     my $vollist = [];
3533     my $scsicontroller = {};
3534     my $ahcicontroller = {};
3535     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3536
3537     # Add iscsi initiator name if available
3538     if (my $initiator = get_initiator_name()) {
3539         push @$devices, '-iscsi', "initiator-name=$initiator";
3540     }
3541
3542     foreach_drive($conf, sub {
3543         my ($ds, $drive) = @_;
3544
3545         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3546             push @$vollist, $drive->{file};
3547         }
3548
3549         # ignore efidisk here, already added in bios/fw handling code above
3550         return if $drive->{interface} eq 'efidisk';
3551
3552         $use_virtio = 1 if $ds =~ m/^virtio/;
3553
3554         if (drive_is_cdrom ($drive)) {
3555             if ($bootindex_hash->{d}) {
3556                 $drive->{bootindex} = $bootindex_hash->{d};
3557                 $bootindex_hash->{d} += 1;
3558             }
3559         } else {
3560             if ($bootindex_hash->{c}) {
3561                 $drive->{bootindex} = $bootindex_hash->{c} if $conf->{bootdisk} && ($conf->{bootdisk} eq $ds);
3562                 $bootindex_hash->{c} += 1;
3563             }
3564         }
3565
3566         if($drive->{interface} eq 'virtio'){
3567            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3568         }
3569
3570         if ($drive->{interface} eq 'scsi') {
3571
3572             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
3573
3574             $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges);
3575             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
3576
3577             my $iothread = '';
3578             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
3579                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3580                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3581             } elsif ($drive->{iothread}) {
3582                 warn "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n";
3583             }
3584
3585             my $queues = '';
3586             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
3587                 $queues = ",num_queues=$drive->{queues}";
3588             }
3589
3590             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues" if !$scsicontroller->{$controller};
3591             $scsicontroller->{$controller}=1;
3592         }
3593
3594         if ($drive->{interface} eq 'sata') {
3595            my $controller = int($drive->{index} / $MAX_SATA_DISKS);
3596            $pciaddr = print_pci_addr("ahci$controller", $bridges);
3597            push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr" if !$ahcicontroller->{$controller};
3598            $ahcicontroller->{$controller}=1;
3599         }
3600
3601         my $drive_cmd = print_drive_full($storecfg, $vmid, $drive);
3602         push @$devices, '-drive',$drive_cmd;
3603         push @$devices, '-device', print_drivedevice_full($storecfg, $conf, $vmid, $drive, $bridges);
3604     });
3605
3606     for (my $i = 0; $i < $MAX_NETS; $i++) {
3607          next if !$conf->{"net$i"};
3608          my $d = parse_net($conf->{"net$i"});
3609          next if !$d;
3610
3611          $use_virtio = 1 if $d->{model} eq 'virtio';
3612
3613          if ($bootindex_hash->{n}) {
3614             $d->{bootindex} = $bootindex_hash->{n};
3615             $bootindex_hash->{n} += 1;
3616          }
3617
3618          my $netdevfull = print_netdev_full($vmid,$conf,$d,"net$i");
3619          push @$devices, '-netdev', $netdevfull;
3620
3621          my $netdevicefull = print_netdevice_full($vmid, $conf, $d, "net$i", $bridges, $use_old_bios_files);
3622          push @$devices, '-device', $netdevicefull;
3623     }
3624
3625     if (!$q35) {
3626         # add pci bridges
3627         if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3628            $bridges->{1} = 1;
3629            $bridges->{2} = 1;
3630         }
3631
3632         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
3633
3634         while (my ($k, $v) = each %$bridges) {
3635             $pciaddr = print_pci_addr("pci.$k");
3636             unshift @$devices, '-device', "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr" if $k > 0;
3637         }
3638     }
3639
3640     # add custom args
3641     if ($conf->{args}) {
3642         my $aa = PVE::Tools::split_args($conf->{args});
3643         push @$cmd, @$aa;
3644     }
3645
3646     push @$cmd, @$devices;
3647     push @$cmd, '-rtc', join(',', @$rtcFlags)
3648         if scalar(@$rtcFlags);
3649     push @$cmd, '-machine', join(',', @$machineFlags)
3650         if scalar(@$machineFlags);
3651     push @$cmd, '-global', join(',', @$globalFlags)
3652         if scalar(@$globalFlags);
3653
3654     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
3655 }
3656
3657 sub vnc_socket {
3658     my ($vmid) = @_;
3659     return "${var_run_tmpdir}/$vmid.vnc";
3660 }
3661
3662 sub spice_port {
3663     my ($vmid) = @_;
3664
3665     my $res = vm_mon_cmd($vmid, 'query-spice');
3666
3667     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
3668 }
3669
3670 sub qmp_socket {
3671     my ($vmid, $qga) = @_;
3672     my $sockettype = $qga ? 'qga' : 'qmp';
3673     return "${var_run_tmpdir}/$vmid.$sockettype";
3674 }
3675
3676 sub pidfile_name {
3677     my ($vmid) = @_;
3678     return "${var_run_tmpdir}/$vmid.pid";
3679 }
3680
3681 sub vm_devices_list {
3682     my ($vmid) = @_;
3683
3684     my $res = vm_mon_cmd($vmid, 'query-pci');
3685     my $devices_to_check = [];
3686     my $devices = {};
3687     foreach my $pcibus (@$res) {
3688         push @$devices_to_check, @{$pcibus->{devices}},
3689     }
3690
3691     while (@$devices_to_check) {
3692         my $to_check = [];
3693         for my $d (@$devices_to_check) {
3694             $devices->{$d->{'qdev_id'}} = 1 if $d->{'qdev_id'};
3695             next if !$d->{'pci_bridge'};
3696
3697             $devices->{$d->{'qdev_id'}} += scalar(@{$d->{'pci_bridge'}->{devices}});
3698             push @$to_check, @{$d->{'pci_bridge'}->{devices}};
3699         }
3700         $devices_to_check = $to_check;
3701     }
3702
3703     my $resblock = vm_mon_cmd($vmid, 'query-block');
3704     foreach my $block (@$resblock) {
3705         if($block->{device} =~ m/^drive-(\S+)/){
3706                 $devices->{$1} = 1;
3707         }
3708     }
3709
3710     my $resmice = vm_mon_cmd($vmid, 'query-mice');
3711     foreach my $mice (@$resmice) {
3712         if ($mice->{name} eq 'QEMU HID Tablet') {
3713             $devices->{tablet} = 1;
3714             last;
3715         }
3716     }
3717
3718     # for usb devices there is no query-usb
3719     # but we can iterate over the entries in
3720     # qom-list path=/machine/peripheral
3721     my $resperipheral = vm_mon_cmd($vmid, 'qom-list', path => '/machine/peripheral');
3722     foreach my $per (@$resperipheral) {
3723         if ($per->{name} =~ m/^usb\d+$/) {
3724             $devices->{$per->{name}} = 1;
3725         }
3726     }
3727
3728     return $devices;
3729 }
3730
3731 sub vm_deviceplug {
3732     my ($storecfg, $conf, $vmid, $deviceid, $device) = @_;
3733
3734     my $q35 = machine_type_is_q35($conf);
3735
3736     my $devices_list = vm_devices_list($vmid);
3737     return 1 if defined($devices_list->{$deviceid});
3738
3739     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid); # add PCI bridge if we need it for the device
3740
3741     if ($deviceid eq 'tablet') {
3742
3743         qemu_deviceadd($vmid, print_tabletdevice_full($conf));
3744
3745     } elsif ($deviceid =~ m/^usb(\d+)$/) {
3746
3747         die "usb hotplug currently not reliable\n";
3748         # since we can't reliably hot unplug all added usb devices
3749         # and usb passthrough disables live migration
3750         # we disable usb hotplugging for now
3751         qemu_deviceadd($vmid, PVE::QemuServer::USB::print_usbdevice_full($conf, $deviceid, $device));
3752
3753     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3754
3755         qemu_iothread_add($vmid, $deviceid, $device);
3756
3757         qemu_driveadd($storecfg, $vmid, $device);
3758         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
3759
3760         qemu_deviceadd($vmid, $devicefull);
3761         eval { qemu_deviceaddverify($vmid, $deviceid); };
3762         if (my $err = $@) {
3763             eval { qemu_drivedel($vmid, $deviceid); };
3764             warn $@ if $@;
3765             die $err;
3766         }
3767
3768     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3769
3770
3771         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
3772         my $pciaddr = print_pci_addr($deviceid);
3773         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
3774
3775         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
3776
3777         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
3778             qemu_iothread_add($vmid, $deviceid, $device);
3779             $devicefull .= ",iothread=iothread-$deviceid";
3780         }
3781
3782         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
3783             $devicefull .= ",num_queues=$device->{queues}";
3784         }
3785
3786         qemu_deviceadd($vmid, $devicefull);
3787         qemu_deviceaddverify($vmid, $deviceid);
3788
3789     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3790
3791         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device);
3792         qemu_driveadd($storecfg, $vmid, $device);
3793
3794         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
3795         eval { qemu_deviceadd($vmid, $devicefull); };
3796         if (my $err = $@) {
3797             eval { qemu_drivedel($vmid, $deviceid); };
3798             warn $@ if $@;
3799             die $err;
3800         }
3801
3802     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3803
3804         return undef if !qemu_netdevadd($vmid, $conf, $device, $deviceid);
3805
3806         my $machine_type = PVE::QemuServer::qemu_machine_pxe($vmid, $conf);
3807         my $use_old_bios_files = undef;
3808         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3809
3810         my $netdevicefull = print_netdevice_full($vmid, $conf, $device, $deviceid, undef, $use_old_bios_files);
3811         qemu_deviceadd($vmid, $netdevicefull);
3812         eval { qemu_deviceaddverify($vmid, $deviceid); };
3813         if (my $err = $@) {
3814             eval { qemu_netdevdel($vmid, $deviceid); };
3815             warn $@ if $@;
3816             die $err;
3817         }
3818
3819     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
3820
3821         my $bridgeid = $2;
3822         my $pciaddr = print_pci_addr($deviceid);
3823         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
3824
3825         qemu_deviceadd($vmid, $devicefull);
3826         qemu_deviceaddverify($vmid, $deviceid);
3827
3828     } else {
3829         die "can't hotplug device '$deviceid'\n";
3830     }
3831
3832     return 1;
3833 }
3834
3835 # fixme: this should raise exceptions on error!
3836 sub vm_deviceunplug {
3837     my ($vmid, $conf, $deviceid) = @_;
3838
3839     my $devices_list = vm_devices_list($vmid);
3840     return 1 if !defined($devices_list->{$deviceid});
3841
3842     die "can't unplug bootdisk" if $conf->{bootdisk} && $conf->{bootdisk} eq $deviceid;
3843
3844     if ($deviceid eq 'tablet') {
3845
3846         qemu_devicedel($vmid, $deviceid);
3847
3848     } elsif ($deviceid =~ m/^usb\d+$/) {
3849
3850         die "usb hotplug currently not reliable\n";
3851         # when unplugging usb devices this way,
3852         # there may be remaining usb controllers/hubs
3853         # so we disable it for now
3854         qemu_devicedel($vmid, $deviceid);
3855         qemu_devicedelverify($vmid, $deviceid);
3856
3857     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3858
3859         qemu_devicedel($vmid, $deviceid);
3860         qemu_devicedelverify($vmid, $deviceid);
3861         qemu_drivedel($vmid, $deviceid);
3862         qemu_iothread_del($conf, $vmid, $deviceid);
3863
3864     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3865
3866         qemu_devicedel($vmid, $deviceid);
3867         qemu_devicedelverify($vmid, $deviceid);
3868         qemu_iothread_del($conf, $vmid, $deviceid);
3869
3870     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3871
3872         qemu_devicedel($vmid, $deviceid);
3873         qemu_drivedel($vmid, $deviceid);
3874         qemu_deletescsihw($conf, $vmid, $deviceid);
3875
3876     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3877
3878         qemu_devicedel($vmid, $deviceid);
3879         qemu_devicedelverify($vmid, $deviceid);
3880         qemu_netdevdel($vmid, $deviceid);
3881
3882     } else {
3883         die "can't unplug device '$deviceid'\n";
3884     }
3885
3886     return 1;
3887 }
3888
3889 sub qemu_deviceadd {
3890     my ($vmid, $devicefull) = @_;
3891
3892     $devicefull = "driver=".$devicefull;
3893     my %options =  split(/[=,]/, $devicefull);
3894
3895     vm_mon_cmd($vmid, "device_add" , %options);
3896 }
3897
3898 sub qemu_devicedel {
3899     my ($vmid, $deviceid) = @_;
3900
3901     my $ret = vm_mon_cmd($vmid, "device_del", id => $deviceid);
3902 }
3903
3904 sub qemu_iothread_add {
3905     my($vmid, $deviceid, $device) = @_;
3906
3907     if ($device->{iothread}) {
3908         my $iothreads = vm_iothreads_list($vmid);
3909         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
3910     }
3911 }
3912
3913 sub qemu_iothread_del {
3914     my($conf, $vmid, $deviceid) = @_;
3915
3916     my $device = parse_drive($deviceid, $conf->{$deviceid});
3917     if ($device->{iothread}) {
3918         my $iothreads = vm_iothreads_list($vmid);
3919         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
3920     }
3921 }
3922
3923 sub qemu_objectadd {
3924     my($vmid, $objectid, $qomtype) = @_;
3925
3926     vm_mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
3927
3928     return 1;
3929 }
3930
3931 sub qemu_objectdel {
3932     my($vmid, $objectid) = @_;
3933
3934     vm_mon_cmd($vmid, "object-del", id => $objectid);
3935
3936     return 1;
3937 }
3938
3939 sub qemu_driveadd {
3940     my ($storecfg, $vmid, $device) = @_;
3941
3942     my $drive = print_drive_full($storecfg, $vmid, $device);
3943     $drive =~ s/\\/\\\\/g;
3944     my $ret = vm_human_monitor_command($vmid, "drive_add auto \"$drive\"");
3945
3946     # If the command succeeds qemu prints: "OK"
3947     return 1 if $ret =~ m/OK/s;
3948
3949     die "adding drive failed: $ret\n";
3950 }
3951
3952 sub qemu_drivedel {
3953     my($vmid, $deviceid) = @_;
3954
3955     my $ret = vm_human_monitor_command($vmid, "drive_del drive-$deviceid");
3956     $ret =~ s/^\s+//;
3957
3958     return 1 if $ret eq "";
3959
3960     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
3961     return 1 if $ret =~ m/Device \'.*?\' not found/s;
3962
3963     die "deleting drive $deviceid failed : $ret\n";
3964 }
3965
3966 sub qemu_deviceaddverify {
3967     my ($vmid, $deviceid) = @_;
3968
3969     for (my $i = 0; $i <= 5; $i++) {
3970          my $devices_list = vm_devices_list($vmid);
3971          return 1 if defined($devices_list->{$deviceid});
3972          sleep 1;
3973     }
3974
3975     die "error on hotplug device '$deviceid'\n";
3976 }
3977
3978
3979 sub qemu_devicedelverify {
3980     my ($vmid, $deviceid) = @_;
3981
3982     # need to verify that the device is correctly removed as device_del
3983     # is async and empty return is not reliable
3984
3985     for (my $i = 0; $i <= 5; $i++) {
3986          my $devices_list = vm_devices_list($vmid);
3987          return 1 if !defined($devices_list->{$deviceid});
3988          sleep 1;
3989     }
3990
3991     die "error on hot-unplugging device '$deviceid'\n";
3992 }
3993
3994 sub qemu_findorcreatescsihw {
3995     my ($storecfg, $conf, $vmid, $device) = @_;
3996
3997     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3998
3999     my $scsihwid="$controller_prefix$controller";
4000     my $devices_list = vm_devices_list($vmid);
4001
4002     if(!defined($devices_list->{$scsihwid})) {
4003         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device);
4004     }
4005
4006     return 1;
4007 }
4008
4009 sub qemu_deletescsihw {
4010     my ($conf, $vmid, $opt) = @_;
4011
4012     my $device = parse_drive($opt, $conf->{$opt});
4013
4014     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
4015         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
4016         return 1;
4017     }
4018
4019     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4020
4021     my $devices_list = vm_devices_list($vmid);
4022     foreach my $opt (keys %{$devices_list}) {
4023         if (PVE::QemuServer::is_valid_drivename($opt)) {
4024             my $drive = PVE::QemuServer::parse_drive($opt, $conf->{$opt});
4025             if($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
4026                 return 1;
4027             }
4028         }
4029     }
4030
4031     my $scsihwid="scsihw$controller";
4032
4033     vm_deviceunplug($vmid, $conf, $scsihwid);
4034
4035     return 1;
4036 }
4037
4038 sub qemu_add_pci_bridge {
4039     my ($storecfg, $conf, $vmid, $device) = @_;
4040
4041     my $bridges = {};
4042
4043     my $bridgeid;
4044
4045     print_pci_addr($device, $bridges);
4046
4047     while (my ($k, $v) = each %$bridges) {
4048         $bridgeid = $k;
4049     }
4050     return 1 if !defined($bridgeid) || $bridgeid < 1;
4051
4052     my $bridge = "pci.$bridgeid";
4053     my $devices_list = vm_devices_list($vmid);
4054
4055     if (!defined($devices_list->{$bridge})) {
4056         vm_deviceplug($storecfg, $conf, $vmid, $bridge);
4057     }
4058
4059     return 1;
4060 }
4061
4062 sub qemu_set_link_status {
4063     my ($vmid, $device, $up) = @_;
4064
4065     vm_mon_cmd($vmid, "set_link", name => $device,
4066                up => $up ? JSON::true : JSON::false);
4067 }
4068
4069 sub qemu_netdevadd {
4070     my ($vmid, $conf, $device, $deviceid) = @_;
4071
4072     my $netdev = print_netdev_full($vmid, $conf, $device, $deviceid, 1);
4073     my %options =  split(/[=,]/, $netdev);
4074
4075     vm_mon_cmd($vmid, "netdev_add",  %options);
4076     return 1;
4077 }
4078
4079 sub qemu_netdevdel {
4080     my ($vmid, $deviceid) = @_;
4081
4082     vm_mon_cmd($vmid, "netdev_del", id => $deviceid);
4083 }
4084
4085 sub qemu_usb_hotplug {
4086     my ($storecfg, $conf, $vmid, $deviceid, $device) = @_;
4087
4088     return if !$device;
4089
4090     # remove the old one first
4091     vm_deviceunplug($vmid, $conf, $deviceid);
4092
4093     # check if xhci controller is necessary and available
4094     if ($device->{usb3}) {
4095
4096         my $devicelist = vm_devices_list($vmid);
4097
4098         if (!$devicelist->{xhci}) {
4099             my $pciaddr = print_pci_addr("xhci");
4100             qemu_deviceadd($vmid, "nec-usb-xhci,id=xhci$pciaddr");
4101         }
4102     }
4103     my $d = parse_usb_device($device->{host});
4104     $d->{usb3} = $device->{usb3};
4105
4106     # add the new one
4107     vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $d);
4108 }
4109
4110 sub qemu_cpu_hotplug {
4111     my ($vmid, $conf, $vcpus) = @_;
4112
4113     my $machine_type = PVE::QemuServer::get_current_qemu_machine($vmid);
4114
4115     my $sockets = 1;
4116     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
4117     $sockets = $conf->{sockets} if  $conf->{sockets};
4118     my $cores = $conf->{cores} || 1;
4119     my $maxcpus = $sockets * $cores;
4120
4121     $vcpus = $maxcpus if !$vcpus;
4122
4123     die "you can't add more vcpus than maxcpus\n"
4124         if $vcpus > $maxcpus;
4125
4126     my $currentvcpus = $conf->{vcpus} || $maxcpus;
4127
4128     if ($vcpus < $currentvcpus) {
4129
4130         if (qemu_machine_feature_enabled ($machine_type, undef, 2, 7)) {
4131
4132             for (my $i = $currentvcpus; $i > $vcpus; $i--) {
4133                 qemu_devicedel($vmid, "cpu$i");
4134                 my $retry = 0;
4135                 my $currentrunningvcpus = undef;
4136                 while (1) {
4137                     $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
4138                     last if scalar(@{$currentrunningvcpus}) == $i-1;
4139                     raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
4140                     $retry++;
4141                     sleep 1;
4142                 }
4143                 #update conf after each succesfull cpu unplug
4144                 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4145                 PVE::QemuConfig->write_config($vmid, $conf);
4146             }
4147         } else {
4148             die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
4149         }
4150
4151         return;
4152     }
4153
4154     my $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
4155     die "vcpus in running vm does not match its configuration\n"
4156         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
4157
4158     if (qemu_machine_feature_enabled ($machine_type, undef, 2, 7)) {
4159
4160         for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
4161             my $cpustr = print_cpu_device($conf, $i);
4162             qemu_deviceadd($vmid, $cpustr);
4163
4164             my $retry = 0;
4165             my $currentrunningvcpus = undef;
4166             while (1) {
4167                 $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
4168                 last if scalar(@{$currentrunningvcpus}) == $i;
4169                 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
4170                 sleep 1;
4171                 $retry++;
4172             }
4173             #update conf after each succesfull cpu hotplug
4174             $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4175             PVE::QemuConfig->write_config($vmid, $conf);
4176         }
4177     } else {
4178
4179         for (my $i = $currentvcpus; $i < $vcpus; $i++) {
4180             vm_mon_cmd($vmid, "cpu-add", id => int($i));
4181         }
4182     }
4183 }
4184
4185 sub qemu_block_set_io_throttle {
4186     my ($vmid, $deviceid,
4187         $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
4188         $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max,
4189         $bps_max_length, $bps_rd_max_length, $bps_wr_max_length,
4190         $iops_max_length, $iops_rd_max_length, $iops_wr_max_length) = @_;
4191
4192     return if !check_running($vmid) ;
4193
4194     vm_mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
4195         bps => int($bps),
4196         bps_rd => int($bps_rd),
4197         bps_wr => int($bps_wr),
4198         iops => int($iops),
4199         iops_rd => int($iops_rd),
4200         iops_wr => int($iops_wr),
4201         bps_max => int($bps_max),
4202         bps_rd_max => int($bps_rd_max),
4203         bps_wr_max => int($bps_wr_max),
4204         iops_max => int($iops_max),
4205         iops_rd_max => int($iops_rd_max),
4206         iops_wr_max => int($iops_wr_max),
4207         bps_max_length => int($bps_max_length),
4208         bps_rd_max_length => int($bps_rd_max_length),
4209         bps_wr_max_length => int($bps_wr_max_length),
4210         iops_max_length => int($iops_max_length),
4211         iops_rd_max_length => int($iops_rd_max_length),
4212         iops_wr_max_length => int($iops_wr_max_length),
4213     );
4214
4215 }
4216
4217 # old code, only used to shutdown old VM after update
4218 sub __read_avail {
4219     my ($fh, $timeout) = @_;
4220
4221     my $sel = new IO::Select;
4222     $sel->add($fh);
4223
4224     my $res = '';
4225     my $buf;
4226
4227     my @ready;
4228     while (scalar (@ready = $sel->can_read($timeout))) {
4229         my $count;
4230         if ($count = $fh->sysread($buf, 8192)) {
4231             if ($buf =~ /^(.*)\(qemu\) $/s) {
4232                 $res .= $1;
4233                 last;
4234             } else {
4235                 $res .= $buf;
4236             }
4237         } else {
4238             if (!defined($count)) {
4239                 die "$!\n";
4240             }
4241             last;
4242         }
4243     }
4244
4245     die "monitor read timeout\n" if !scalar(@ready);
4246
4247     return $res;
4248 }
4249
4250 sub qemu_block_resize {
4251     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4252
4253     my $running = check_running($vmid);
4254
4255     $size = 0 if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4256
4257     return if !$running;
4258
4259     vm_mon_cmd($vmid, "block_resize", device => $deviceid, size => int($size));
4260
4261 }
4262
4263 sub qemu_volume_snapshot {
4264     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4265
4266     my $running = check_running($vmid);
4267
4268     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4269         vm_mon_cmd($vmid, "snapshot-drive", device => $deviceid, name => $snap);
4270     } else {
4271         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4272     }
4273 }
4274
4275 sub qemu_volume_snapshot_delete {
4276     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4277
4278     my $running = check_running($vmid);
4279
4280     if($running) {
4281
4282         $running = undef;
4283         my $conf = PVE::QemuConfig->load_config($vmid);
4284         foreach_drive($conf, sub {
4285             my ($ds, $drive) = @_;
4286             $running = 1 if $drive->{file} eq $volid;
4287         });
4288     }
4289
4290     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4291         vm_mon_cmd($vmid, "delete-drive-snapshot", device => $deviceid, name => $snap);
4292     } else {
4293         PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4294     }
4295 }
4296
4297 sub set_migration_caps {
4298     my ($vmid) = @_;
4299
4300     my $cap_ref = [];
4301
4302     my $enabled_cap = {
4303         "auto-converge" => 1,
4304         "xbzrle" => 1,
4305         "x-rdma-pin-all" => 0,
4306         "zero-blocks" => 0,
4307         "compress" => 0
4308     };
4309
4310     my $supported_capabilities = vm_mon_cmd_nocheck($vmid, "query-migrate-capabilities");
4311
4312     for my $supported_capability (@$supported_capabilities) {
4313         push @$cap_ref, {
4314             capability => $supported_capability->{capability},
4315             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4316         };
4317     }
4318
4319     vm_mon_cmd_nocheck($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4320 }
4321
4322 my $fast_plug_option = {
4323     'lock' => 1,
4324     'name' => 1,
4325     'onboot' => 1,
4326     'shares' => 1,
4327     'startup' => 1,
4328     'description' => 1,
4329     'protection' => 1,
4330     'vmstatestorage' => 1,
4331 };
4332
4333 # hotplug changes in [PENDING]
4334 # $selection hash can be used to only apply specified options, for
4335 # example: { cores => 1 } (only apply changed 'cores')
4336 # $errors ref is used to return error messages
4337 sub vmconfig_hotplug_pending {
4338     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4339
4340     my $defaults = load_defaults();
4341
4342     # commit values which do not have any impact on running VM first
4343     # Note: those option cannot raise errors, we we do not care about
4344     # $selection and always apply them.
4345
4346     my $add_error = sub {
4347         my ($opt, $msg) = @_;
4348         $errors->{$opt} = "hotplug problem - $msg";
4349     };
4350
4351     my $changes = 0;
4352     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4353         if ($fast_plug_option->{$opt}) {
4354             $conf->{$opt} = $conf->{pending}->{$opt};
4355             delete $conf->{pending}->{$opt};
4356             $changes = 1;
4357         }
4358     }
4359
4360     if ($changes) {
4361         PVE::QemuConfig->write_config($vmid, $conf);
4362         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4363     }
4364
4365     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4366
4367     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4368     while (my ($opt, $force) = each %$pending_delete_hash) {
4369         next if $selection && !$selection->{$opt};
4370         eval {
4371             if ($opt eq 'hotplug') {
4372                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
4373             } elsif ($opt eq 'tablet') {
4374                 die "skip\n" if !$hotplug_features->{usb};
4375                 if ($defaults->{tablet}) {
4376                     vm_deviceplug($storecfg, $conf, $vmid, $opt);
4377                 } else {
4378                     vm_deviceunplug($vmid, $conf, $opt);
4379                 }
4380             } elsif ($opt =~ m/^usb\d+/) {
4381                 die "skip\n";
4382                 # since we cannot reliably hot unplug usb devices
4383                 # we are disabling it
4384                 die "skip\n" if !$hotplug_features->{usb} || $conf->{$opt} =~ m/spice/i;
4385                 vm_deviceunplug($vmid, $conf, $opt);
4386             } elsif ($opt eq 'vcpus') {
4387                 die "skip\n" if !$hotplug_features->{cpu};
4388                 qemu_cpu_hotplug($vmid, $conf, undef);
4389             } elsif ($opt eq 'balloon') {
4390                 # enable balloon device is not hotpluggable
4391                 die "skip\n" if defined($conf->{balloon}) && $conf->{balloon} == 0;
4392                 # here we reset the ballooning value to memory
4393                 my $balloon = $conf->{memory} || $defaults->{memory};
4394                 vm_mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4395             } elsif ($fast_plug_option->{$opt}) {
4396                 # do nothing
4397             } elsif ($opt =~ m/^net(\d+)$/) {
4398                 die "skip\n" if !$hotplug_features->{network};
4399                 vm_deviceunplug($vmid, $conf, $opt);
4400             } elsif (is_valid_drivename($opt)) {
4401                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4402                 vm_deviceunplug($vmid, $conf, $opt);
4403                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4404             } elsif ($opt =~ m/^memory$/) {
4405                 die "skip\n" if !$hotplug_features->{memory};
4406                 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
4407             } elsif ($opt eq 'cpuunits') {
4408                 cgroups_write("cpu", $vmid, "cpu.shares", $defaults->{cpuunits});
4409             } elsif ($opt eq 'cpulimit') {
4410                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", -1);
4411             } else {
4412                 die "skip\n";
4413             }
4414         };
4415         if (my $err = $@) {
4416             &$add_error($opt, $err) if $err ne "skip\n";
4417         } else {
4418             # save new config if hotplug was successful
4419             delete $conf->{$opt};
4420             vmconfig_undelete_pending_option($conf, $opt);
4421             PVE::QemuConfig->write_config($vmid, $conf);
4422             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4423         }
4424     }
4425
4426     my $apply_pending_cloudinit;
4427     $apply_pending_cloudinit = sub {
4428         my ($key, $value) = @_;
4429         $apply_pending_cloudinit = sub {}; # once is enough
4430
4431         my @cloudinit_opts = keys %$confdesc_cloudinit;
4432         foreach my $opt (keys %{$conf->{pending}}) {
4433             next if !grep { $_ eq $opt } @cloudinit_opts;
4434             $conf->{$opt} = delete $conf->{pending}->{$opt};
4435         }
4436
4437         my $new_conf = { %$conf };
4438         $new_conf->{$key} = $value;
4439         PVE::QemuServer::Cloudinit::generate_cloudinitconfig($new_conf, $vmid);
4440     };
4441
4442     foreach my $opt (keys %{$conf->{pending}}) {
4443         next if $selection && !$selection->{$opt};
4444         my $value = $conf->{pending}->{$opt};
4445         eval {
4446             if ($opt eq 'hotplug') {
4447                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
4448             } elsif ($opt eq 'tablet') {
4449                 die "skip\n" if !$hotplug_features->{usb};
4450                 if ($value == 1) {
4451                     vm_deviceplug($storecfg, $conf, $vmid, $opt);
4452                 } elsif ($value == 0) {
4453                     vm_deviceunplug($vmid, $conf, $opt);
4454                 }
4455             } elsif ($opt =~ m/^usb\d+$/) {
4456                 die "skip\n";
4457                 # since we cannot reliably hot unplug usb devices
4458                 # we are disabling it
4459                 die "skip\n" if !$hotplug_features->{usb} || $value =~ m/spice/i;
4460                 my $d = eval { PVE::JSONSchema::parse_property_string($usbdesc->{format}, $value) };
4461                 die "skip\n" if !$d;
4462                 qemu_usb_hotplug($storecfg, $conf, $vmid, $opt, $d);
4463             } elsif ($opt eq 'vcpus') {
4464                 die "skip\n" if !$hotplug_features->{cpu};
4465                 qemu_cpu_hotplug($vmid, $conf, $value);
4466             } elsif ($opt eq 'balloon') {
4467                 # enable/disable balloning device is not hotpluggable
4468                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
4469                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
4470                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
4471
4472                 # allow manual ballooning if shares is set to zero
4473                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
4474                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
4475                     vm_mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4476                 }
4477             } elsif ($opt =~ m/^net(\d+)$/) {
4478                 # some changes can be done without hotplug
4479                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
4480                                     $vmid, $opt, $value);
4481             } elsif (is_valid_drivename($opt)) {
4482                 # some changes can be done without hotplug
4483                 my $drive = parse_drive($opt, $value);
4484                 if (drive_is_cloudinit($drive)) {
4485                     &$apply_pending_cloudinit($opt, $value);
4486                 }
4487                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
4488                                      $vmid, $opt, $value, 1);
4489             } elsif ($opt =~ m/^memory$/) { #dimms
4490                 die "skip\n" if !$hotplug_features->{memory};
4491                 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
4492             } elsif ($opt eq 'cpuunits') {
4493                 cgroups_write("cpu", $vmid, "cpu.shares", $conf->{pending}->{$opt});
4494             } elsif ($opt eq 'cpulimit') {
4495                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
4496                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", $cpulimit);
4497             } else {
4498                 die "skip\n";  # skip non-hot-pluggable options
4499             }
4500         };
4501         if (my $err = $@) {
4502             &$add_error($opt, $err) if $err ne "skip\n";
4503         } else {
4504             # save new config if hotplug was successful
4505             $conf->{$opt} = $value;
4506             delete $conf->{pending}->{$opt};
4507             PVE::QemuConfig->write_config($vmid, $conf);
4508             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4509         }
4510     }
4511 }
4512
4513 sub try_deallocate_drive {
4514     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
4515
4516     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
4517         my $volid = $drive->{file};
4518         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
4519             my $sid = PVE::Storage::parse_volume_id($volid);
4520             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
4521
4522             # check if the disk is really unused
4523             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
4524                 if is_volume_in_use($storecfg, $conf, $key, $volid);
4525             PVE::Storage::vdisk_free($storecfg, $volid);
4526             return 1;
4527         } else {
4528             # If vm is not owner of this disk remove from config
4529             return 1;
4530         }
4531     }
4532
4533     return undef;
4534 }
4535
4536 sub vmconfig_delete_or_detach_drive {
4537     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
4538
4539     my $drive = parse_drive($opt, $conf->{$opt});
4540
4541     my $rpcenv = PVE::RPCEnvironment::get();
4542     my $authuser = $rpcenv->get_user();
4543
4544     if ($force) {
4545         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
4546         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
4547     } else {
4548         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
4549     }
4550 }
4551
4552 sub vmconfig_apply_pending {
4553     my ($vmid, $conf, $storecfg) = @_;
4554
4555     # cold plug
4556
4557     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4558     while (my ($opt, $force) = each %$pending_delete_hash) {
4559         die "internal error" if $opt =~ m/^unused/;
4560         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4561         if (!defined($conf->{$opt})) {
4562             vmconfig_undelete_pending_option($conf, $opt);
4563             PVE::QemuConfig->write_config($vmid, $conf);
4564         } elsif (is_valid_drivename($opt)) {
4565             vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4566             vmconfig_undelete_pending_option($conf, $opt);
4567             delete $conf->{$opt};
4568             PVE::QemuConfig->write_config($vmid, $conf);
4569         } else {
4570             vmconfig_undelete_pending_option($conf, $opt);
4571             delete $conf->{$opt};
4572             PVE::QemuConfig->write_config($vmid, $conf);
4573         }
4574     }
4575
4576     $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4577
4578     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4579         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4580
4581         if (defined($conf->{$opt}) && ($conf->{$opt} eq $conf->{pending}->{$opt})) {
4582             # skip if nothing changed
4583         } elsif (is_valid_drivename($opt)) {
4584             vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
4585                 if defined($conf->{$opt});
4586             $conf->{$opt} = $conf->{pending}->{$opt};
4587         } else {
4588             $conf->{$opt} = $conf->{pending}->{$opt};
4589         }
4590
4591         delete $conf->{pending}->{$opt};
4592         PVE::QemuConfig->write_config($vmid, $conf);
4593     }
4594 }
4595
4596 my $safe_num_ne = sub {
4597     my ($a, $b) = @_;
4598
4599     return 0 if !defined($a) && !defined($b);
4600     return 1 if !defined($a);
4601     return 1 if !defined($b);
4602
4603     return $a != $b;
4604 };
4605
4606 my $safe_string_ne = sub {
4607     my ($a, $b) = @_;
4608
4609     return 0 if !defined($a) && !defined($b);
4610     return 1 if !defined($a);
4611     return 1 if !defined($b);
4612
4613     return $a ne $b;
4614 };
4615
4616 sub vmconfig_update_net {
4617     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value) = @_;
4618
4619     my $newnet = parse_net($value);
4620
4621     if ($conf->{$opt}) {
4622         my $oldnet = parse_net($conf->{$opt});
4623
4624         if (&$safe_string_ne($oldnet->{model}, $newnet->{model}) ||
4625             &$safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
4626             &$safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
4627             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
4628
4629             # for non online change, we try to hot-unplug
4630             die "skip\n" if !$hotplug;
4631             vm_deviceunplug($vmid, $conf, $opt);
4632         } else {
4633
4634             die "internal error" if $opt !~ m/net(\d+)/;
4635             my $iface = "tap${vmid}i$1";
4636
4637             if (&$safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
4638                 &$safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
4639                 &$safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
4640                 &$safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
4641                 PVE::Network::tap_unplug($iface);
4642                 PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
4643             } elsif (&$safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
4644                 # Rate can be applied on its own but any change above needs to
4645                 # include the rate in tap_plug since OVS resets everything.
4646                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
4647             }
4648
4649             if (&$safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
4650                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
4651             }
4652
4653             return 1;
4654         }
4655     }
4656
4657     if ($hotplug) {
4658         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet);
4659     } else {
4660         die "skip\n";
4661     }
4662 }
4663
4664 sub vmconfig_update_disk {
4665     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $force) = @_;
4666
4667     # fixme: do we need force?
4668
4669     my $drive = parse_drive($opt, $value);
4670
4671     if ($conf->{$opt}) {
4672
4673         if (my $old_drive = parse_drive($opt, $conf->{$opt}))  {
4674
4675             my $media = $drive->{media} || 'disk';
4676             my $oldmedia = $old_drive->{media} || 'disk';
4677             die "unable to change media type\n" if $media ne $oldmedia;
4678
4679             if (!drive_is_cdrom($old_drive)) {
4680
4681                 if ($drive->{file} ne $old_drive->{file}) {
4682
4683                     die "skip\n" if !$hotplug;
4684
4685                     # unplug and register as unused
4686                     vm_deviceunplug($vmid, $conf, $opt);
4687                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
4688
4689                 } else {
4690                     # update existing disk
4691
4692                     # skip non hotpluggable value
4693                     if (&$safe_string_ne($drive->{discard}, $old_drive->{discard}) ||
4694                         &$safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
4695                         &$safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
4696                         &$safe_string_ne($drive->{cache}, $old_drive->{cache})) {
4697                         die "skip\n";
4698                     }
4699
4700                     # apply throttle
4701                     if (&$safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
4702                         &$safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
4703                         &$safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
4704                         &$safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
4705                         &$safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
4706                         &$safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
4707                         &$safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
4708                         &$safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
4709                         &$safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
4710                         &$safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
4711                         &$safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
4712                         &$safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max}) ||
4713                         &$safe_num_ne($drive->{bps_max_length}, $old_drive->{bps_max_length}) ||
4714                         &$safe_num_ne($drive->{bps_rd_max_length}, $old_drive->{bps_rd_max_length}) ||
4715                         &$safe_num_ne($drive->{bps_wr_max_length}, $old_drive->{bps_wr_max_length}) ||
4716                         &$safe_num_ne($drive->{iops_max_length}, $old_drive->{iops_max_length}) ||
4717                         &$safe_num_ne($drive->{iops_rd_max_length}, $old_drive->{iops_rd_max_length}) ||
4718                         &$safe_num_ne($drive->{iops_wr_max_length}, $old_drive->{iops_wr_max_length})) {
4719
4720                         qemu_block_set_io_throttle($vmid,"drive-$opt",
4721                                                    ($drive->{mbps} || 0)*1024*1024,
4722                                                    ($drive->{mbps_rd} || 0)*1024*1024,
4723                                                    ($drive->{mbps_wr} || 0)*1024*1024,
4724                                                    $drive->{iops} || 0,
4725                                                    $drive->{iops_rd} || 0,
4726                                                    $drive->{iops_wr} || 0,
4727                                                    ($drive->{mbps_max} || 0)*1024*1024,
4728                                                    ($drive->{mbps_rd_max} || 0)*1024*1024,
4729                                                    ($drive->{mbps_wr_max} || 0)*1024*1024,
4730                                                    $drive->{iops_max} || 0,
4731                                                    $drive->{iops_rd_max} || 0,
4732                                                    $drive->{iops_wr_max} || 0,
4733                                                    $drive->{bps_max_length} || 1,
4734                                                    $drive->{bps_rd_max_length} || 1,
4735                                                    $drive->{bps_wr_max_length} || 1,
4736                                                    $drive->{iops_max_length} || 1,
4737                                                    $drive->{iops_rd_max_length} || 1,
4738                                                    $drive->{iops_wr_max_length} || 1);
4739
4740                     }
4741
4742                     return 1;
4743                 }
4744
4745             } else { # cdrom
4746
4747                 if ($drive->{file} eq 'none') {
4748                     vm_mon_cmd($vmid, "eject",force => JSON::true,device => "drive-$opt");
4749                     if (drive_is_cloudinit($old_drive)) {
4750                         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive);
4751                     }
4752                 } else {
4753                     my $path = get_iso_path($storecfg, $vmid, $drive->{file});
4754                     vm_mon_cmd($vmid, "eject", force => JSON::true,device => "drive-$opt"); # force eject if locked
4755                     vm_mon_cmd($vmid, "change", device => "drive-$opt",target => "$path") if $path;
4756                 }
4757
4758                 return 1;
4759             }
4760         }
4761     }
4762
4763     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
4764     # hotplug new disks
4765     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
4766     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive);
4767 }
4768
4769 sub vm_start {
4770     my ($storecfg, $vmid, $statefile, $skiplock, $migratedfrom, $paused,
4771         $forcemachine, $spice_ticket, $migration_network, $migration_type, $targetstorage) = @_;
4772
4773     PVE::QemuConfig->lock_config($vmid, sub {
4774         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
4775
4776         die "you can't start a vm if it's a template\n" if PVE::QemuConfig->is_template($conf);
4777
4778         PVE::QemuConfig->check_lock($conf) if !$skiplock;
4779
4780         die "VM $vmid already running\n" if check_running($vmid, undef, $migratedfrom);
4781
4782         if (!$statefile && scalar(keys %{$conf->{pending}})) {
4783             vmconfig_apply_pending($vmid, $conf, $storecfg);
4784             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4785         }
4786
4787         PVE::QemuServer::Cloudinit::generate_cloudinitconfig($conf, $vmid);
4788
4789         my $defaults = load_defaults();
4790
4791         # set environment variable useful inside network script
4792         $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
4793
4794         my $local_volumes = {};
4795
4796         if ($targetstorage) {
4797             foreach_drive($conf, sub {
4798                 my ($ds, $drive) = @_;
4799
4800                 return if drive_is_cdrom($drive);
4801
4802                 my $volid = $drive->{file};
4803
4804                 return if !$volid;
4805
4806                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
4807
4808                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
4809                 return if $scfg->{shared};
4810                 $local_volumes->{$ds} = [$volid, $storeid, $volname];
4811             });
4812
4813             my $format = undef;
4814
4815             foreach my $opt (sort keys %$local_volumes) {
4816
4817                 my ($volid, $storeid, $volname) = @{$local_volumes->{$opt}};
4818                 my $drive = parse_drive($opt, $conf->{$opt});
4819
4820                 #if remote storage is specified, use default format
4821                 if ($targetstorage && $targetstorage ne "1") {
4822                     $storeid = $targetstorage;
4823                     my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
4824                     $format = $defFormat;
4825                 } else {
4826                     #else we use same format than original
4827                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
4828                     $format = qemu_img_format($scfg, $volid);
4829                 }
4830
4831                 my $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $format, undef, ($drive->{size}/1024));
4832                 my $newdrive = $drive;
4833                 $newdrive->{format} = $format;
4834                 $newdrive->{file} = $newvolid;
4835                 my $drivestr = PVE::QemuServer::print_drive($vmid, $newdrive);
4836                 $local_volumes->{$opt} = $drivestr;
4837                 #pass drive to conf for command line
4838                 $conf->{$opt} = $drivestr;
4839             }
4840         }
4841
4842         my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine);
4843
4844         my $migrate_port = 0;
4845         my $migrate_uri;
4846         if ($statefile) {
4847             if ($statefile eq 'tcp') {
4848                 my $localip = "localhost";
4849                 my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
4850                 my $nodename = PVE::INotify::nodename();
4851
4852                 if (!defined($migration_type)) {
4853                     if (defined($datacenterconf->{migration}->{type})) {
4854                         $migration_type = $datacenterconf->{migration}->{type};
4855                     } else {
4856                         $migration_type = 'secure';
4857                     }
4858                 }
4859
4860                 if ($migration_type eq 'insecure') {
4861                     my $migrate_network_addr = PVE::Cluster::get_local_migration_ip($migration_network);
4862                     if ($migrate_network_addr) {
4863                         $localip = $migrate_network_addr;
4864                     } else {
4865                         $localip = PVE::Cluster::remote_node_ip($nodename, 1);
4866                     }
4867
4868                     $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
4869                 }
4870
4871                 my $pfamily = PVE::Tools::get_host_address_family($nodename);
4872                 $migrate_port = PVE::Tools::next_migrate_port($pfamily);
4873                 $migrate_uri = "tcp:${localip}:${migrate_port}";
4874                 push @$cmd, '-incoming', $migrate_uri;
4875                 push @$cmd, '-S';
4876
4877             } elsif ($statefile eq 'unix') {
4878                 # should be default for secure migrations as a ssh TCP forward
4879                 # tunnel is not deterministic reliable ready and fails regurarly
4880                 # to set up in time, so use UNIX socket forwards
4881                 my $socket_addr = "/run/qemu-server/$vmid.migrate";
4882                 unlink $socket_addr;
4883
4884                 $migrate_uri = "unix:$socket_addr";
4885
4886                 push @$cmd, '-incoming', $migrate_uri;
4887                 push @$cmd, '-S';
4888
4889             } else {
4890                 push @$cmd, '-loadstate', $statefile;
4891             }
4892         } elsif ($paused) {
4893             push @$cmd, '-S';
4894         }
4895
4896         # host pci devices
4897         for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
4898           my $d = parse_hostpci($conf->{"hostpci$i"});
4899           next if !$d;
4900           my $pcidevices = $d->{pciid};
4901           foreach my $pcidevice (@$pcidevices) {
4902                 my $pciid = $pcidevice->{id}.".".$pcidevice->{function};
4903
4904                 my $info = pci_device_info("0000:$pciid");
4905                 die "IOMMU not present\n" if !check_iommu_support();
4906                 die "no pci device info for device '$pciid'\n" if !$info;
4907                 die "can't unbind/bind pci group to vfio '$pciid'\n" if !pci_dev_group_bind_to_vfio($pciid);
4908                 die "can't reset pci device '$pciid'\n" if $info->{has_fl_reset} and !pci_dev_reset($info);
4909           }
4910         }
4911
4912         PVE::Storage::activate_volumes($storecfg, $vollist);
4913
4914         if (!check_running($vmid, 1)) {
4915             eval {
4916                 run_command(['/bin/systemctl', 'stop', "$vmid.scope"],
4917                     outfunc => sub {}, errfunc => sub {});
4918             };
4919         }
4920
4921         my $cpuunits = defined($conf->{cpuunits}) ? $conf->{cpuunits}
4922                                                   : $defaults->{cpuunits};
4923
4924         my $start_timeout = $conf->{hugepages} ? 300 : 30;
4925         my %run_params = (timeout => $statefile ? undef : $start_timeout, umask => 0077);
4926
4927         my %properties = (
4928             Slice => 'qemu.slice',
4929             KillMode => 'none',
4930             CPUShares => $cpuunits
4931         );
4932
4933         if (my $cpulimit = $conf->{cpulimit}) {
4934             $properties{CPUQuota} = int($cpulimit * 100);
4935         }
4936         $properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
4937
4938         my $run_qemu = sub {
4939             PVE::Tools::run_fork sub {
4940                 PVE::Systemd::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
4941                 run_command($cmd, %run_params);
4942             };
4943         };
4944
4945         if ($conf->{hugepages}) {
4946
4947             my $code = sub {
4948                 my $hugepages_topology = PVE::QemuServer::Memory::hugepages_topology($conf);
4949                 my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
4950
4951                 PVE::QemuServer::Memory::hugepages_mount();
4952                 PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
4953
4954                 eval { $run_qemu->() };
4955                 if (my $err = $@) {
4956                     PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology);
4957                     die $err;
4958                 }
4959
4960                 PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology);
4961             };
4962             eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
4963
4964         } else {
4965             eval { $run_qemu->() };
4966         }
4967
4968         if (my $err = $@) {
4969             # deactivate volumes if start fails
4970             eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
4971             die "start failed: $err";
4972         }
4973
4974         print "migration listens on $migrate_uri\n" if $migrate_uri;
4975
4976         if ($statefile && $statefile ne 'tcp' && $statefile ne 'unix')  {
4977             eval { vm_mon_cmd_nocheck($vmid, "cont"); };
4978             warn $@ if $@;
4979         }
4980
4981         #start nbd server for storage migration
4982         if ($targetstorage) {
4983             my $nodename = PVE::INotify::nodename();
4984             my $migrate_network_addr = PVE::Cluster::get_local_migration_ip($migration_network);
4985             my $localip = $migrate_network_addr ? $migrate_network_addr : PVE::Cluster::remote_node_ip($nodename, 1);
4986             my $pfamily = PVE::Tools::get_host_address_family($nodename);
4987             $migrate_port = PVE::Tools::next_migrate_port($pfamily);
4988
4989             vm_mon_cmd_nocheck($vmid, "nbd-server-start", addr => { type => 'inet', data => { host => "${localip}", port => "${migrate_port}" } } );
4990
4991             $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
4992
4993             foreach my $opt (sort keys %$local_volumes) {
4994                 my $volid = $local_volumes->{$opt};
4995                 vm_mon_cmd_nocheck($vmid, "nbd-server-add", device => "drive-$opt", writable => JSON::true );
4996                 my $migrate_storage_uri = "nbd:${localip}:${migrate_port}:exportname=drive-$opt";
4997                 print "storage migration listens on $migrate_storage_uri volume:$volid\n";
4998             }
4999         }
5000
5001         if ($migratedfrom) {
5002             eval {
5003                 set_migration_caps($vmid);
5004             };
5005             warn $@ if $@;
5006
5007             if ($spice_port) {
5008                 print "spice listens on port $spice_port\n";
5009                 if ($spice_ticket) {
5010                     vm_mon_cmd_nocheck($vmid, "set_password", protocol => 'spice', password => $spice_ticket);
5011                     vm_mon_cmd_nocheck($vmid, "expire_password", protocol => 'spice', time => "+30");
5012                 }
5013             }
5014
5015         } else {
5016             vm_mon_cmd_nocheck($vmid, "balloon", value => $conf->{balloon}*1024*1024)
5017                 if !$statefile && $conf->{balloon};
5018
5019             foreach my $opt (keys %$conf) {
5020                 next if $opt !~  m/^net\d+$/;
5021                 my $nicconf = parse_net($conf->{$opt});
5022                 qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
5023             }
5024         }
5025
5026         vm_mon_cmd_nocheck($vmid, 'qom-set',
5027                     path => "machine/peripheral/balloon0",
5028                     property => "guest-stats-polling-interval",
5029                     value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
5030
5031     });
5032 }
5033
5034 sub vm_mon_cmd {
5035     my ($vmid, $execute, %params) = @_;
5036
5037     my $cmd = { execute => $execute, arguments => \%params };
5038     vm_qmp_command($vmid, $cmd);
5039 }
5040
5041 sub vm_mon_cmd_nocheck {
5042     my ($vmid, $execute, %params) = @_;
5043
5044     my $cmd = { execute => $execute, arguments => \%params };
5045     vm_qmp_command($vmid, $cmd, 1);
5046 }
5047
5048 sub vm_qmp_command {
5049     my ($vmid, $cmd, $nocheck) = @_;
5050
5051     my $res;
5052
5053     my $timeout;
5054     if ($cmd->{arguments} && $cmd->{arguments}->{timeout}) {
5055         $timeout = $cmd->{arguments}->{timeout};
5056         delete $cmd->{arguments}->{timeout};
5057     }
5058
5059     eval {
5060         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
5061         my $sname = qmp_socket($vmid);
5062         if (-e $sname) { # test if VM is reasonambe new and supports qmp/qga
5063             my $qmpclient = PVE::QMPClient->new();
5064
5065             $res = $qmpclient->cmd($vmid, $cmd, $timeout);
5066         } else {
5067             die "unable to open monitor socket\n";
5068         }
5069     };
5070     if (my $err = $@) {
5071         syslog("err", "VM $vmid qmp command failed - $err");
5072         die $err;
5073     }
5074
5075     return $res;
5076 }
5077
5078 sub vm_human_monitor_command {
5079     my ($vmid, $cmdline) = @_;
5080
5081     my $res;
5082
5083     my $cmd = {
5084         execute => 'human-monitor-command',
5085         arguments => { 'command-line' => $cmdline},
5086     };
5087
5088     return vm_qmp_command($vmid, $cmd);
5089 }
5090
5091 sub vm_commandline {
5092     my ($storecfg, $vmid) = @_;
5093
5094     my $conf = PVE::QemuConfig->load_config($vmid);
5095
5096     my $defaults = load_defaults();
5097
5098     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults);
5099
5100     return PVE::Tools::cmd2string($cmd);
5101 }
5102
5103 sub vm_reset {
5104     my ($vmid, $skiplock) = @_;
5105
5106     PVE::QemuConfig->lock_config($vmid, sub {
5107
5108         my $conf = PVE::QemuConfig->load_config($vmid);
5109
5110         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5111
5112         vm_mon_cmd($vmid, "system_reset");
5113     });
5114 }
5115
5116 sub get_vm_volumes {
5117     my ($conf) = @_;
5118
5119     my $vollist = [];
5120     foreach_volid($conf, sub {
5121         my ($volid, $attr) = @_;
5122
5123         return if $volid =~ m|^/|;
5124
5125         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5126         return if !$sid;
5127
5128         push @$vollist, $volid;
5129     });
5130
5131     return $vollist;
5132 }
5133
5134 sub vm_stop_cleanup {
5135     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
5136
5137     eval {
5138
5139         if (!$keepActive) {
5140             my $vollist = get_vm_volumes($conf);
5141             PVE::Storage::deactivate_volumes($storecfg, $vollist);
5142         }
5143
5144         foreach my $ext (qw(mon qmp pid vnc qga)) {
5145             unlink "/var/run/qemu-server/${vmid}.$ext";
5146         }
5147
5148         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
5149     };
5150     warn $@ if $@; # avoid errors - just warn
5151 }
5152
5153 # Note: use $nockeck to skip tests if VM configuration file exists.
5154 # We need that when migration VMs to other nodes (files already moved)
5155 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
5156 sub vm_stop {
5157     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
5158
5159     $force = 1 if !defined($force) && !$shutdown;
5160
5161     if ($migratedfrom){
5162         my $pid = check_running($vmid, $nocheck, $migratedfrom);
5163         kill 15, $pid if $pid;
5164         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
5165         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
5166         return;
5167     }
5168
5169     PVE::QemuConfig->lock_config($vmid, sub {
5170
5171         my $pid = check_running($vmid, $nocheck);
5172         return if !$pid;
5173
5174         my $conf;
5175         if (!$nocheck) {
5176             $conf = PVE::QemuConfig->load_config($vmid);
5177             PVE::QemuConfig->check_lock($conf) if !$skiplock;
5178             if (!defined($timeout) && $shutdown && $conf->{startup}) {
5179                 my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
5180                 $timeout = $opts->{down} if $opts->{down};
5181             }
5182         }
5183
5184         $timeout = 60 if !defined($timeout);
5185
5186         eval {
5187             if ($shutdown) {
5188                 if (defined($conf) && parse_guest_agent($conf)->{enabled}) {
5189                     vm_qmp_command($vmid, { execute => "guest-shutdown" }, $nocheck);
5190                 } else {
5191                     vm_qmp_command($vmid, { execute => "system_powerdown" }, $nocheck);
5192                 }
5193             } else {
5194                 vm_qmp_command($vmid, { execute => "quit" }, $nocheck);
5195             }
5196         };
5197         my $err = $@;
5198
5199         if (!$err) {
5200             my $count = 0;
5201             while (($count < $timeout) && check_running($vmid, $nocheck)) {
5202                 $count++;
5203                 sleep 1;
5204             }
5205
5206             if ($count >= $timeout) {
5207                 if ($force) {
5208                     warn "VM still running - terminating now with SIGTERM\n";
5209                     kill 15, $pid;
5210                 } else {
5211                     die "VM quit/powerdown failed - got timeout\n";
5212                 }
5213             } else {
5214                 vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5215                 return;
5216             }
5217         } else {
5218             if ($force) {
5219                 warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
5220                 kill 15, $pid;
5221             } else {
5222                 die "VM quit/powerdown failed\n";
5223             }
5224         }
5225
5226         # wait again
5227         $timeout = 10;
5228
5229         my $count = 0;
5230         while (($count < $timeout) && check_running($vmid, $nocheck)) {
5231             $count++;
5232             sleep 1;
5233         }
5234
5235         if ($count >= $timeout) {
5236             warn "VM still running - terminating now with SIGKILL\n";
5237             kill 9, $pid;
5238             sleep 1;
5239         }
5240
5241         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5242    });
5243 }
5244
5245 sub vm_suspend {
5246     my ($vmid, $skiplock) = @_;
5247
5248     PVE::QemuConfig->lock_config($vmid, sub {
5249
5250         my $conf = PVE::QemuConfig->load_config($vmid);
5251
5252         PVE::QemuConfig->check_lock($conf)
5253             if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
5254
5255         vm_mon_cmd($vmid, "stop");
5256     });
5257 }
5258
5259 sub vm_resume {
5260     my ($vmid, $skiplock, $nocheck) = @_;
5261
5262     PVE::QemuConfig->lock_config($vmid, sub {
5263
5264         my $res = vm_mon_cmd($vmid, 'query-status');
5265         my $resume_cmd = 'cont';
5266
5267         if ($res->{status} && $res->{status} eq 'suspended') {
5268             $resume_cmd = 'system_wakeup';
5269         }
5270
5271         if (!$nocheck) {
5272
5273             my $conf = PVE::QemuConfig->load_config($vmid);
5274
5275             PVE::QemuConfig->check_lock($conf)
5276                 if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
5277
5278             vm_mon_cmd($vmid, $resume_cmd);
5279
5280         } else {
5281             vm_mon_cmd_nocheck($vmid, $resume_cmd);
5282         }
5283     });
5284 }
5285
5286 sub vm_sendkey {
5287     my ($vmid, $skiplock, $key) = @_;
5288
5289     PVE::QemuConfig->lock_config($vmid, sub {
5290
5291         my $conf = PVE::QemuConfig->load_config($vmid);
5292
5293         # there is no qmp command, so we use the human monitor command
5294         vm_human_monitor_command($vmid, "sendkey $key");
5295     });
5296 }
5297
5298 sub vm_destroy {
5299     my ($storecfg, $vmid, $skiplock) = @_;
5300
5301     PVE::QemuConfig->lock_config($vmid, sub {
5302
5303         my $conf = PVE::QemuConfig->load_config($vmid);
5304
5305         if (!check_running($vmid)) {
5306             destroy_vm($storecfg, $vmid, undef, $skiplock);
5307         } else {
5308             die "VM $vmid is running - destroy failed\n";
5309         }
5310     });
5311 }
5312
5313 # pci helpers
5314
5315 sub file_write {
5316     my ($filename, $buf) = @_;
5317
5318     my $fh = IO::File->new($filename, "w");
5319     return undef if !$fh;
5320
5321     my $res = print $fh $buf;
5322
5323     $fh->close();
5324
5325     return $res;
5326 }
5327
5328 sub pci_device_info {
5329     my ($name) = @_;
5330
5331     my $res;
5332
5333     return undef if $name !~ m/^([a-f0-9]{4}):([a-f0-9]{2}):([a-f0-9]{2})\.([a-f0-9])$/;
5334     my ($domain, $bus, $slot, $func) = ($1, $2, $3, $4);
5335
5336     my $irq = file_read_firstline("$pcisysfs/devices/$name/irq");
5337     return undef if !defined($irq) || $irq !~ m/^\d+$/;
5338
5339     my $vendor = file_read_firstline("$pcisysfs/devices/$name/vendor");
5340     return undef if !defined($vendor) || $vendor !~ s/^0x//;
5341
5342     my $product = file_read_firstline("$pcisysfs/devices/$name/device");
5343     return undef if !defined($product) || $product !~ s/^0x//;
5344
5345     $res = {
5346         name => $name,
5347         vendor => $vendor,
5348         product => $product,
5349         domain => $domain,
5350         bus => $bus,
5351         slot => $slot,
5352         func => $func,
5353         irq => $irq,
5354         has_fl_reset => -f "$pcisysfs/devices/$name/reset" || 0,
5355     };
5356
5357     return $res;
5358 }
5359
5360 sub pci_dev_reset {
5361     my ($dev) = @_;
5362
5363     my $name = $dev->{name};
5364
5365     my $fn = "$pcisysfs/devices/$name/reset";
5366
5367     return file_write($fn, "1");
5368 }
5369
5370 sub pci_dev_bind_to_vfio {
5371     my ($dev) = @_;
5372
5373     my $name = $dev->{name};
5374
5375     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
5376
5377     if (!-d $vfio_basedir) {
5378         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
5379     }
5380     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
5381
5382     my $testdir = "$vfio_basedir/$name";
5383     return 1 if -d $testdir;
5384
5385     my $data = "$dev->{vendor} $dev->{product}";
5386     return undef if !file_write("$vfio_basedir/new_id", $data);
5387
5388     my $fn = "$pcisysfs/devices/$name/driver/unbind";
5389     if (!file_write($fn, $name)) {
5390         return undef if -f $fn;
5391     }
5392
5393     $fn = "$vfio_basedir/bind";
5394     if (! -d $testdir) {
5395         return undef if !file_write($fn, $name);
5396     }
5397
5398     return -d $testdir;
5399 }
5400
5401 sub pci_dev_group_bind_to_vfio {
5402     my ($pciid) = @_;
5403
5404     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
5405
5406     if (!-d $vfio_basedir) {
5407         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
5408     }
5409     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
5410
5411     # get IOMMU group devices
5412     opendir(my $D, "$pcisysfs/devices/0000:$pciid/iommu_group/devices/") || die "Cannot open iommu_group: $!\n";
5413       my @devs = grep /^0000:/, readdir($D);
5414     closedir($D);
5415
5416     foreach my $pciid (@devs) {
5417         $pciid =~ m/^([:\.\da-f]+)$/ or die "PCI ID $pciid not valid!\n";
5418
5419         # pci bridges, switches or root ports are not supported
5420         # they have a pci_bus subdirectory so skip them
5421         next if (-e "$pcisysfs/devices/$pciid/pci_bus");
5422
5423         my $info = pci_device_info($1);
5424         pci_dev_bind_to_vfio($info) || die "Cannot bind $pciid to vfio\n";
5425     }
5426
5427     return 1;
5428 }
5429
5430 # vzdump restore implementaion
5431
5432 sub tar_archive_read_firstfile {
5433     my $archive = shift;
5434
5435     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
5436
5437     # try to detect archive type first
5438     my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
5439         die "unable to open file '$archive'\n";
5440     my $firstfile = <$fh>;
5441     kill 15, $pid;
5442     close $fh;
5443
5444     die "ERROR: archive contaions no data\n" if !$firstfile;
5445     chomp $firstfile;
5446
5447     return $firstfile;
5448 }
5449
5450 sub tar_restore_cleanup {
5451     my ($storecfg, $statfile) = @_;
5452
5453     print STDERR "starting cleanup\n";
5454
5455     if (my $fd = IO::File->new($statfile, "r")) {
5456         while (defined(my $line = <$fd>)) {
5457             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5458                 my $volid = $2;
5459                 eval {
5460                     if ($volid =~ m|^/|) {
5461                         unlink $volid || die 'unlink failed\n';
5462                     } else {
5463                         PVE::Storage::vdisk_free($storecfg, $volid);
5464                     }
5465                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
5466                 };
5467                 print STDERR "unable to cleanup '$volid' - $@" if $@;
5468             } else {
5469                 print STDERR "unable to parse line in statfile - $line";
5470             }
5471         }
5472         $fd->close();
5473     }
5474 }
5475
5476 sub restore_archive {
5477     my ($archive, $vmid, $user, $opts) = @_;
5478
5479     my $format = $opts->{format};
5480     my $comp;
5481
5482     if ($archive =~ m/\.tgz$/ || $archive =~ m/\.tar\.gz$/) {
5483         $format = 'tar' if !$format;
5484         $comp = 'gzip';
5485     } elsif ($archive =~ m/\.tar$/) {
5486         $format = 'tar' if !$format;
5487     } elsif ($archive =~ m/.tar.lzo$/) {
5488         $format = 'tar' if !$format;
5489         $comp = 'lzop';
5490     } elsif ($archive =~ m/\.vma$/) {
5491         $format = 'vma' if !$format;
5492     } elsif ($archive =~ m/\.vma\.gz$/) {
5493         $format = 'vma' if !$format;
5494         $comp = 'gzip';
5495     } elsif ($archive =~ m/\.vma\.lzo$/) {
5496         $format = 'vma' if !$format;
5497         $comp = 'lzop';
5498     } else {
5499         $format = 'vma' if !$format; # default
5500     }
5501
5502     # try to detect archive format
5503     if ($format eq 'tar') {
5504         return restore_tar_archive($archive, $vmid, $user, $opts);
5505     } else {
5506         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
5507     }
5508 }
5509
5510 sub restore_update_config_line {
5511     my ($outfd, $cookie, $vmid, $map, $line, $unique) = @_;
5512
5513     return if $line =~ m/^\#qmdump\#/;
5514     return if $line =~ m/^\#vzdump\#/;
5515     return if $line =~ m/^lock:/;
5516     return if $line =~ m/^unused\d+:/;
5517     return if $line =~ m/^parent:/;
5518     return if $line =~ m/^template:/; # restored VM is never a template
5519
5520     my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
5521     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
5522         # try to convert old 1.X settings
5523         my ($id, $ind, $ethcfg) = ($1, $2, $3);
5524         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
5525             my ($model, $macaddr) = split(/\=/, $devconfig);
5526             $macaddr = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if !$macaddr || $unique;
5527             my $net = {
5528                 model => $model,
5529                 bridge => "vmbr$ind",
5530                 macaddr => $macaddr,
5531             };
5532             my $netstr = print_net($net);
5533
5534             print $outfd "net$cookie->{netcount}: $netstr\n";
5535             $cookie->{netcount}++;
5536         }
5537     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
5538         my ($id, $netstr) = ($1, $2);
5539         my $net = parse_net($netstr);
5540         $net->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if $net->{macaddr};
5541         $netstr = print_net($net);
5542         print $outfd "$id: $netstr\n";
5543     } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk)\d+):\s*(\S+)\s*$/) {
5544         my $virtdev = $1;
5545         my $value = $3;
5546         my $di = parse_drive($virtdev, $value);
5547         if (defined($di->{backup}) && !$di->{backup}) {
5548             print $outfd "#$line";
5549         } elsif ($map->{$virtdev}) {
5550             delete $di->{format}; # format can change on restore
5551             $di->{file} = $map->{$virtdev};
5552             $value = print_drive($vmid, $di);
5553             print $outfd "$virtdev: $value\n";
5554         } else {
5555             print $outfd $line;
5556         }
5557     } elsif (($line =~ m/^(smbios1: )(.*)/) && $unique) {
5558         my ($uuid, $uuid_str);
5559         UUID::generate($uuid);
5560         UUID::unparse($uuid, $uuid_str);
5561         my $smbios1 = parse_smbios1($2);
5562         $smbios1->{uuid} = $uuid_str;
5563         print $outfd $1.print_smbios1($smbios1)."\n";
5564     } else {
5565         print $outfd $line;
5566     }
5567 }
5568
5569 sub scan_volids {
5570     my ($cfg, $vmid) = @_;
5571
5572     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid);
5573
5574     my $volid_hash = {};
5575     foreach my $storeid (keys %$info) {
5576         foreach my $item (@{$info->{$storeid}}) {
5577             next if !($item->{volid} && $item->{size});
5578             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
5579             $volid_hash->{$item->{volid}} = $item;
5580         }
5581     }
5582
5583     return $volid_hash;
5584 }
5585
5586 sub is_volume_in_use {
5587     my ($storecfg, $conf, $skip_drive, $volid) = @_;
5588
5589     my $path = PVE::Storage::path($storecfg, $volid);
5590
5591     my $scan_config = sub {
5592         my ($cref, $snapname) = @_;
5593
5594         foreach my $key (keys %$cref) {
5595             my $value = $cref->{$key};
5596             if (is_valid_drivename($key)) {
5597                 next if $skip_drive && $key eq $skip_drive;
5598                 my $drive = parse_drive($key, $value);
5599                 next if !$drive || !$drive->{file} || drive_is_cdrom($drive);
5600                 return 1 if $volid eq $drive->{file};
5601                 if ($drive->{file} =~ m!^/!) {
5602                     return 1 if $drive->{file} eq $path;
5603                 } else {
5604                     my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file}, 1);
5605                     next if !$storeid;
5606                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid, 1);
5607                     next if !$scfg;
5608                     return 1 if $path eq PVE::Storage::path($storecfg, $drive->{file}, $snapname);
5609                 }
5610             }
5611         }
5612
5613         return 0;
5614     };
5615
5616     return 1 if &$scan_config($conf);
5617
5618     undef $skip_drive;
5619
5620     foreach my $snapname (keys %{$conf->{snapshots}}) {
5621         return 1 if &$scan_config($conf->{snapshots}->{$snapname}, $snapname);
5622     }
5623
5624     return 0;
5625 }
5626
5627 sub update_disksize {
5628     my ($vmid, $conf, $volid_hash) = @_;
5629
5630     my $changes;
5631     my $prefix = "VM $vmid:";
5632
5633     # used and unused disks
5634     my $referenced = {};
5635
5636     # Note: it is allowed to define multiple storages with same path (alias), so
5637     # we need to check both 'volid' and real 'path' (two different volid can point
5638     # to the same path).
5639
5640     my $referencedpath = {};
5641
5642     # update size info
5643     foreach my $opt (keys %$conf) {
5644         if (is_valid_drivename($opt)) {
5645             my $drive = parse_drive($opt, $conf->{$opt});
5646             my $volid = $drive->{file};
5647             next if !$volid;
5648
5649             $referenced->{$volid} = 1;
5650             if ($volid_hash->{$volid} &&
5651                 (my $path = $volid_hash->{$volid}->{path})) {
5652                 $referencedpath->{$path} = 1;
5653             }
5654
5655             next if drive_is_cdrom($drive);
5656             next if !$volid_hash->{$volid};
5657
5658             $drive->{size} = $volid_hash->{$volid}->{size};
5659             my $new = print_drive($vmid, $drive);
5660             if ($new ne $conf->{$opt}) {
5661                 $changes = 1;
5662                 $conf->{$opt} = $new;
5663                 print "$prefix update disk '$opt' information.\n";
5664             }
5665         }
5666     }
5667
5668     # remove 'unusedX' entry if volume is used
5669     foreach my $opt (keys %$conf) {
5670         next if $opt !~ m/^unused\d+$/;
5671         my $volid = $conf->{$opt};
5672         my $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
5673         if ($referenced->{$volid} || ($path && $referencedpath->{$path})) {
5674             print "$prefix remove entry '$opt', its volume '$volid' is in use.\n";
5675             $changes = 1;
5676             delete $conf->{$opt};
5677         }
5678
5679         $referenced->{$volid} = 1;
5680         $referencedpath->{$path} = 1 if $path;
5681     }
5682
5683     foreach my $volid (sort keys %$volid_hash) {
5684         next if $volid =~ m/vm-$vmid-state-/;
5685         next if $referenced->{$volid};
5686         my $path = $volid_hash->{$volid}->{path};
5687         next if !$path; # just to be sure
5688         next if $referencedpath->{$path};
5689         $changes = 1;
5690         my $key = PVE::QemuConfig->add_unused_volume($conf, $volid);
5691         print "$prefix add unreferenced volume '$volid' as '$key' to config.\n";
5692         $referencedpath->{$path} = 1; # avoid to add more than once (aliases)
5693     }
5694
5695     return $changes;
5696 }
5697
5698 sub rescan {
5699     my ($vmid, $nolock, $dryrun) = @_;
5700
5701     my $cfg = PVE::Storage::config();
5702
5703     # FIXME: Remove once our RBD plugin can handle CT and VM on a single storage
5704     # see: https://pve.proxmox.com/pipermail/pve-devel/2018-July/032900.html
5705     foreach my $stor (keys %{$cfg->{ids}}) {
5706         delete($cfg->{ids}->{$stor}) if ! $cfg->{ids}->{$stor}->{content}->{images};
5707     }
5708
5709     print "rescan volumes...\n";
5710     my $volid_hash = scan_volids($cfg, $vmid);
5711
5712     my $updatefn =  sub {
5713         my ($vmid) = @_;
5714
5715         my $conf = PVE::QemuConfig->load_config($vmid);
5716
5717         PVE::QemuConfig->check_lock($conf);
5718
5719         my $vm_volids = {};
5720         foreach my $volid (keys %$volid_hash) {
5721             my $info = $volid_hash->{$volid};
5722             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
5723         }
5724
5725         my $changes = update_disksize($vmid, $conf, $vm_volids);
5726
5727         PVE::QemuConfig->write_config($vmid, $conf) if $changes && !$dryrun;
5728     };
5729
5730     if (defined($vmid)) {
5731         if ($nolock) {
5732             &$updatefn($vmid);
5733         } else {
5734             PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
5735         }
5736     } else {
5737         my $vmlist = config_list();
5738         foreach my $vmid (keys %$vmlist) {
5739             if ($nolock) {
5740                 &$updatefn($vmid);
5741             } else {
5742                 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
5743             }
5744         }
5745     }
5746 }
5747
5748 sub restore_vma_archive {
5749     my ($archive, $vmid, $user, $opts, $comp) = @_;
5750
5751     my $readfrom = $archive;
5752
5753     my $cfg = PVE::Storage::config();
5754     my $commands = [];
5755     my $bwlimit = $opts->{bwlimit};
5756
5757     my $dbg_cmdstring = '';
5758     my $add_pipe = sub {
5759         my ($cmd) = @_;
5760         push @$commands, $cmd;
5761         $dbg_cmdstring .= ' | ' if length($dbg_cmdstring);
5762         $dbg_cmdstring .= PVE::Tools::cmd2string($cmd);
5763         $readfrom = '-';
5764     };
5765
5766     my $input = undef;
5767     if ($archive eq '-') {
5768         $input = '<&STDIN';
5769     } else {
5770         # If we use a backup from a PVE defined storage we also consider that
5771         # storage's rate limit:
5772         my (undef, $volid) = PVE::Storage::path_to_volume_id($cfg, $archive);
5773         if (defined($volid)) {
5774             my ($sid, undef) = PVE::Storage::parse_volume_id($volid);
5775             my $readlimit = PVE::Storage::get_bandwidth_limit('restore', [$sid], $bwlimit);
5776             if ($readlimit) {
5777                 print STDERR "applying read rate limit: $readlimit\n";
5778                 my $cstream = ['cstream', '-t', $readlimit*1024, '--', $readfrom];
5779                 $add_pipe->($cstream);
5780             }
5781         }
5782     }
5783
5784     if ($comp) {
5785         my $cmd;
5786         if ($comp eq 'gzip') {
5787             $cmd = ['zcat', $readfrom];
5788         } elsif ($comp eq 'lzop') {
5789             $cmd = ['lzop', '-d', '-c', $readfrom];
5790         } else {
5791             die "unknown compression method '$comp'\n";
5792         }
5793         $add_pipe->($cmd);
5794     }
5795
5796     my $tmpdir = "/var/tmp/vzdumptmp$$";
5797     rmtree $tmpdir;
5798
5799     # disable interrupts (always do cleanups)
5800     local $SIG{INT} =
5801         local $SIG{TERM} =
5802         local $SIG{QUIT} =
5803         local $SIG{HUP} = sub { warn "got interrupt - ignored\n"; };
5804
5805     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
5806     POSIX::mkfifo($mapfifo, 0600);
5807     my $fifofh;
5808
5809     my $openfifo = sub {
5810         open($fifofh, '>', $mapfifo) || die $!;
5811     };
5812
5813     $add_pipe->(['vma', 'extract', '-v', '-r', $mapfifo, $readfrom, $tmpdir]);
5814
5815     my $oldtimeout;
5816     my $timeout = 5;
5817
5818     my $devinfo = {};
5819
5820     my $rpcenv = PVE::RPCEnvironment::get();
5821
5822     my $conffile = PVE::QemuConfig->config_file($vmid);
5823     my $tmpfn = "$conffile.$$.tmp";
5824
5825     # Note: $oldconf is undef if VM does not exists
5826     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
5827     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
5828
5829     my %storage_limits;
5830
5831     my $print_devmap = sub {
5832         my $virtdev_hash = {};
5833
5834         my $cfgfn = "$tmpdir/qemu-server.conf";
5835
5836         # we can read the config - that is already extracted
5837         my $fh = IO::File->new($cfgfn, "r") ||
5838             "unable to read qemu-server.conf - $!\n";
5839
5840         my $fwcfgfn = "$tmpdir/qemu-server.fw";
5841         if (-f $fwcfgfn) {
5842             my $pve_firewall_dir = '/etc/pve/firewall';
5843             mkdir $pve_firewall_dir; # make sure the dir exists
5844             PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
5845         }
5846
5847         while (defined(my $line = <$fh>)) {
5848             if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
5849                 my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
5850                 die "archive does not contain data for drive '$virtdev'\n"
5851                     if !$devinfo->{$devname};
5852                 if (defined($opts->{storage})) {
5853                     $storeid = $opts->{storage} || 'local';
5854                 } elsif (!$storeid) {
5855                     $storeid = 'local';
5856                 }
5857                 $format = 'raw' if !$format;
5858                 $devinfo->{$devname}->{devname} = $devname;
5859                 $devinfo->{$devname}->{virtdev} = $virtdev;
5860                 $devinfo->{$devname}->{format} = $format;
5861                 $devinfo->{$devname}->{storeid} = $storeid;
5862
5863                 # check permission on storage
5864                 my $pool = $opts->{pool}; # todo: do we need that?
5865                 if ($user ne 'root@pam') {
5866                     $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
5867                 }
5868
5869                 $storage_limits{$storeid} = $bwlimit;
5870
5871                 $virtdev_hash->{$virtdev} = $devinfo->{$devname};
5872             }
5873         }
5874
5875         foreach my $key (keys %storage_limits) {
5876             my $limit = PVE::Storage::get_bandwidth_limit('restore', [$key], $bwlimit);
5877             next if !$limit;
5878             print STDERR "rate limit for storage $key: $limit KiB/s\n";
5879             $storage_limits{$key} = $limit * 1024;
5880         }
5881
5882         foreach my $devname (keys %$devinfo) {
5883             die "found no device mapping information for device '$devname'\n"
5884                 if !$devinfo->{$devname}->{virtdev};
5885         }
5886
5887         # create empty/temp config
5888         if ($oldconf) {
5889             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
5890             foreach_drive($oldconf, sub {
5891                 my ($ds, $drive) = @_;
5892
5893                 return if drive_is_cdrom($drive);
5894
5895                 my $volid = $drive->{file};
5896
5897                 return if !$volid || $volid =~ m|^/|;
5898
5899                 my ($path, $owner) = PVE::Storage::path($cfg, $volid);
5900                 return if !$path || !$owner || ($owner != $vmid);
5901
5902                 # Note: only delete disk we want to restore
5903                 # other volumes will become unused
5904                 if ($virtdev_hash->{$ds}) {
5905                     eval { PVE::Storage::vdisk_free($cfg, $volid); };
5906                     if (my $err = $@) {
5907                         warn $err;
5908                     }
5909                 }
5910             });
5911
5912             # delete vmstate files
5913             # since after the restore we have no snapshots anymore
5914             foreach my $snapname (keys %{$oldconf->{snapshots}}) {
5915                 my $snap = $oldconf->{snapshots}->{$snapname};
5916                 if ($snap->{vmstate}) {
5917                     eval { PVE::Storage::vdisk_free($cfg, $snap->{vmstate}); };
5918                     if (my $err = $@) {
5919                         warn $err;
5920                     }
5921                 }
5922             }
5923         }
5924
5925         my $map = {};
5926         foreach my $virtdev (sort keys %$virtdev_hash) {
5927             my $d = $virtdev_hash->{$virtdev};
5928             my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
5929             my $storeid = $d->{storeid};
5930             my $scfg = PVE::Storage::storage_config($cfg, $storeid);
5931
5932             my $map_opts = '';
5933             if (my $limit = $storage_limits{$storeid}) {
5934                 $map_opts .= "throttling.bps=$limit:throttling.group=$storeid:";
5935             }
5936
5937             # test if requested format is supported
5938             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($cfg, $storeid);
5939             my $supported = grep { $_ eq $d->{format} } @$validFormats;
5940             $d->{format} = $defFormat if !$supported;
5941
5942             my $volid = PVE::Storage::vdisk_alloc($cfg, $storeid, $vmid,
5943                                                   $d->{format}, undef, $alloc_size);
5944             print STDERR "new volume ID is '$volid'\n";
5945             $d->{volid} = $volid;
5946             my $path = PVE::Storage::path($cfg, $volid);
5947
5948             PVE::Storage::activate_volumes($cfg,[$volid]);
5949
5950             my $write_zeros = 1;
5951             if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
5952                 $write_zeros = 0;
5953             }
5954
5955             print $fifofh "${map_opts}format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
5956
5957             print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
5958             $map->{$virtdev} = $volid;
5959         }
5960
5961         $fh->seek(0, 0) || die "seek failed - $!\n";
5962
5963         my $outfd = new IO::File ($tmpfn, "w") ||
5964             die "unable to write config for VM $vmid\n";
5965
5966         my $cookie = { netcount => 0 };
5967         while (defined(my $line = <$fh>)) {
5968             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
5969         }
5970
5971         $fh->close();
5972         $outfd->close();
5973     };
5974
5975     eval {
5976         # enable interrupts
5977         local $SIG{INT} =
5978             local $SIG{TERM} =
5979             local $SIG{QUIT} =
5980             local $SIG{HUP} =
5981             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
5982         local $SIG{ALRM} = sub { die "got timeout\n"; };
5983
5984         $oldtimeout = alarm($timeout);
5985
5986         my $parser = sub {
5987             my $line = shift;
5988
5989             print "$line\n";
5990
5991             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
5992                 my ($dev_id, $size, $devname) = ($1, $2, $3);
5993                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
5994             } elsif ($line =~ m/^CTIME: /) {
5995                 # we correctly received the vma config, so we can disable
5996                 # the timeout now for disk allocation (set to 10 minutes, so
5997                 # that we always timeout if something goes wrong)
5998                 alarm(600);
5999                 &$print_devmap();
6000                 print $fifofh "done\n";
6001                 my $tmp = $oldtimeout || 0;
6002                 $oldtimeout = undef;
6003                 alarm($tmp);
6004                 close($fifofh);
6005             }
6006         };
6007
6008         print "restore vma archive: $dbg_cmdstring\n";
6009         run_command($commands, input => $input, outfunc => $parser, afterfork => $openfifo);
6010     };
6011     my $err = $@;
6012
6013     alarm($oldtimeout) if $oldtimeout;
6014
6015     my $vollist = [];
6016     foreach my $devname (keys %$devinfo) {
6017         my $volid = $devinfo->{$devname}->{volid};
6018         push @$vollist, $volid if $volid;
6019     }
6020
6021     PVE::Storage::deactivate_volumes($cfg, $vollist);
6022
6023     unlink $mapfifo;
6024
6025     if ($err) {
6026         rmtree $tmpdir;
6027         unlink $tmpfn;
6028
6029         foreach my $devname (keys %$devinfo) {
6030             my $volid = $devinfo->{$devname}->{volid};
6031             next if !$volid;
6032             eval {
6033                 if ($volid =~ m|^/|) {
6034                     unlink $volid || die 'unlink failed\n';
6035                 } else {
6036                     PVE::Storage::vdisk_free($cfg, $volid);
6037                 }
6038                 print STDERR "temporary volume '$volid' sucessfuly removed\n";
6039             };
6040             print STDERR "unable to cleanup '$volid' - $@" if $@;
6041         }
6042         die $err;
6043     }
6044
6045     rmtree $tmpdir;
6046
6047     rename($tmpfn, $conffile) ||
6048         die "unable to commit configuration file '$conffile'\n";
6049
6050     PVE::Cluster::cfs_update(); # make sure we read new file
6051
6052     eval { rescan($vmid, 1); };
6053     warn $@ if $@;
6054 }
6055
6056 sub restore_tar_archive {
6057     my ($archive, $vmid, $user, $opts) = @_;
6058
6059     if ($archive ne '-') {
6060         my $firstfile = tar_archive_read_firstfile($archive);
6061         die "ERROR: file '$archive' dos not lock like a QemuServer vzdump backup\n"
6062             if $firstfile ne 'qemu-server.conf';
6063     }
6064
6065     my $storecfg = PVE::Storage::config();
6066
6067     # destroy existing data - keep empty config
6068     my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
6069     destroy_vm($storecfg, $vmid, 1) if -f $vmcfgfn;
6070
6071     my $tocmd = "/usr/lib/qemu-server/qmextract";
6072
6073     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
6074     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
6075     $tocmd .= ' --prealloc' if $opts->{prealloc};
6076     $tocmd .= ' --info' if $opts->{info};
6077
6078     # tar option "xf" does not autodetect compression when read from STDIN,
6079     # so we pipe to zcat
6080     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
6081         PVE::Tools::shellquote("--to-command=$tocmd");
6082
6083     my $tmpdir = "/var/tmp/vzdumptmp$$";
6084     mkpath $tmpdir;
6085
6086     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
6087     local $ENV{VZDUMP_VMID} = $vmid;
6088     local $ENV{VZDUMP_USER} = $user;
6089
6090     my $conffile = PVE::QemuConfig->config_file($vmid);
6091     my $tmpfn = "$conffile.$$.tmp";
6092
6093     # disable interrupts (always do cleanups)
6094     local $SIG{INT} =
6095         local $SIG{TERM} =
6096         local $SIG{QUIT} =
6097         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
6098
6099     eval {
6100         # enable interrupts
6101         local $SIG{INT} =
6102             local $SIG{TERM} =
6103             local $SIG{QUIT} =
6104             local $SIG{HUP} =
6105             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6106
6107         if ($archive eq '-') {
6108             print "extracting archive from STDIN\n";
6109             run_command($cmd, input => "<&STDIN");
6110         } else {
6111             print "extracting archive '$archive'\n";
6112             run_command($cmd);
6113         }
6114
6115         return if $opts->{info};
6116
6117         # read new mapping
6118         my $map = {};
6119         my $statfile = "$tmpdir/qmrestore.stat";
6120         if (my $fd = IO::File->new($statfile, "r")) {
6121             while (defined (my $line = <$fd>)) {
6122                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
6123                     $map->{$1} = $2 if $1;
6124                 } else {
6125                     print STDERR "unable to parse line in statfile - $line\n";
6126                 }
6127             }
6128             $fd->close();
6129         }
6130
6131         my $confsrc = "$tmpdir/qemu-server.conf";
6132
6133         my $srcfd = new IO::File($confsrc, "r") ||
6134             die "unable to open file '$confsrc'\n";
6135
6136         my $outfd = new IO::File ($tmpfn, "w") ||
6137             die "unable to write config for VM $vmid\n";
6138
6139         my $cookie = { netcount => 0 };
6140         while (defined (my $line = <$srcfd>)) {
6141             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
6142         }
6143
6144         $srcfd->close();
6145         $outfd->close();
6146     };
6147     my $err = $@;
6148
6149     if ($err) {
6150
6151         unlink $tmpfn;
6152
6153         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
6154
6155         die $err;
6156     }
6157
6158     rmtree $tmpdir;
6159
6160     rename $tmpfn, $conffile ||
6161         die "unable to commit configuration file '$conffile'\n";
6162
6163     PVE::Cluster::cfs_update(); # make sure we read new file
6164
6165     eval { rescan($vmid, 1); };
6166     warn $@ if $@;
6167 };
6168
6169 sub foreach_storage_used_by_vm {
6170     my ($conf, $func) = @_;
6171
6172     my $sidhash = {};
6173
6174     foreach_drive($conf, sub {
6175         my ($ds, $drive) = @_;
6176         return if drive_is_cdrom($drive);
6177
6178         my $volid = $drive->{file};
6179
6180         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
6181         $sidhash->{$sid} = $sid if $sid;
6182     });
6183
6184     foreach my $sid (sort keys %$sidhash) {
6185         &$func($sid);
6186     }
6187 }
6188
6189 sub do_snapshots_with_qemu {
6190     my ($storecfg, $volid) = @_;
6191
6192     my $storage_name = PVE::Storage::parse_volume_id($volid);
6193
6194     if ($qemu_snap_storage->{$storecfg->{ids}->{$storage_name}->{type}}
6195         && !$storecfg->{ids}->{$storage_name}->{krbd}){
6196         return 1;
6197     }
6198
6199     if ($volid =~ m/\.(qcow2|qed)$/){
6200         return 1;
6201     }
6202
6203     return undef;
6204 }
6205
6206 sub qga_check_running {
6207     my ($vmid, $nowarn) = @_;
6208
6209     eval { vm_mon_cmd($vmid, "guest-ping", timeout => 3); };
6210     if ($@) {
6211         warn "Qemu Guest Agent is not running - $@" if !$nowarn;
6212         return 0;
6213     }
6214     return 1;
6215 }
6216
6217 sub template_create {
6218     my ($vmid, $conf, $disk) = @_;
6219
6220     my $storecfg = PVE::Storage::config();
6221
6222     foreach_drive($conf, sub {
6223         my ($ds, $drive) = @_;
6224
6225         return if drive_is_cdrom($drive);
6226         return if $disk && $ds ne $disk;
6227
6228         my $volid = $drive->{file};
6229         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
6230
6231         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
6232         $drive->{file} = $voliddst;
6233         $conf->{$ds} = print_drive($vmid, $drive);
6234         PVE::QemuConfig->write_config($vmid, $conf);
6235     });
6236 }
6237
6238 sub qemu_img_convert {
6239     my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized) = @_;
6240
6241     my $storecfg = PVE::Storage::config();
6242     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
6243     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
6244
6245     if ($src_storeid && $dst_storeid) {
6246
6247         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
6248
6249         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
6250         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6251
6252         my $src_format = qemu_img_format($src_scfg, $src_volname);
6253         my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
6254
6255         my $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
6256         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6257
6258         my $cmd = [];
6259         push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
6260         push @$cmd, '-s', $snapname if($snapname && $src_format eq "qcow2");
6261         push @$cmd, '-t', 'none' if $dst_scfg->{type} eq 'zfspool';
6262         push @$cmd, '-T', 'none' if $src_scfg->{type} eq 'zfspool';
6263         push @$cmd, '-f', $src_format, '-O', $dst_format, $src_path;
6264         if ($is_zero_initialized) {
6265             push @$cmd, "zeroinit:$dst_path";
6266         } else {
6267             push @$cmd, $dst_path;
6268         }
6269
6270         my $parser = sub {
6271             my $line = shift;
6272             if($line =~ m/\((\S+)\/100\%\)/){
6273                 my $percent = $1;
6274                 my $transferred = int($size * $percent / 100);
6275                 my $remaining = $size - $transferred;
6276
6277                 print "transferred: $transferred bytes remaining: $remaining bytes total: $size bytes progression: $percent %\n";
6278             }
6279
6280         };
6281
6282         eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
6283         my $err = $@;
6284         die "copy failed: $err" if $err;
6285     }
6286 }
6287
6288 sub qemu_img_format {
6289     my ($scfg, $volname) = @_;
6290
6291     if ($scfg->{path} && $volname =~ m/\.($QEMU_FORMAT_RE)$/) {
6292         return $1;
6293     } else {
6294         return "raw";
6295     }
6296 }
6297
6298 sub qemu_drive_mirror {
6299     my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $skipcomplete, $qga) = @_;
6300
6301     $jobs = {} if !$jobs;
6302
6303     my $qemu_target;
6304     my $format;
6305     $jobs->{"drive-$drive"} = {};
6306
6307     if ($dst_volid =~ /^nbd:/) {
6308         $qemu_target = $dst_volid;
6309         $format = "nbd";
6310     } else {
6311         my $storecfg = PVE::Storage::config();
6312         my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
6313
6314         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6315
6316         $format = qemu_img_format($dst_scfg, $dst_volname);
6317
6318         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6319
6320         $qemu_target = $is_zero_initialized ? "zeroinit:$dst_path" : $dst_path;
6321     }
6322
6323     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $qemu_target };
6324     $opts->{format} = $format if $format;
6325
6326     print "drive mirror is starting for drive-$drive\n";
6327
6328     eval { vm_mon_cmd($vmid, "drive-mirror", %$opts); }; #if a job already run for this device,it's throw an error
6329
6330     if (my $err = $@) {
6331         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6332         die "mirroring error: $err";
6333     }
6334
6335     qemu_drive_mirror_monitor ($vmid, $vmiddst, $jobs, $skipcomplete, $qga);
6336 }
6337
6338 sub qemu_drive_mirror_monitor {
6339     my ($vmid, $vmiddst, $jobs, $skipcomplete, $qga) = @_;
6340
6341     eval {
6342         my $err_complete = 0;
6343
6344         while (1) {
6345             die "storage migration timed out\n" if $err_complete > 300;
6346
6347             my $stats = vm_mon_cmd($vmid, "query-block-jobs");
6348
6349             my $running_mirror_jobs = {};
6350             foreach my $stat (@$stats) {
6351                 next if $stat->{type} ne 'mirror';
6352                 $running_mirror_jobs->{$stat->{device}} = $stat;
6353             }
6354
6355             my $readycounter = 0;
6356
6357             foreach my $job (keys %$jobs) {
6358
6359                 if(defined($jobs->{$job}->{complete}) && !defined($running_mirror_jobs->{$job})) {
6360                     print "$job : finished\n";
6361                     delete $jobs->{$job};
6362                     next;
6363                 }
6364
6365                 die "$job: mirroring has been cancelled\n" if !defined($running_mirror_jobs->{$job});
6366
6367                 my $busy = $running_mirror_jobs->{$job}->{busy};
6368                 my $ready = $running_mirror_jobs->{$job}->{ready};
6369                 if (my $total = $running_mirror_jobs->{$job}->{len}) {
6370                     my $transferred = $running_mirror_jobs->{$job}->{offset} || 0;
6371                     my $remaining = $total - $transferred;
6372                     my $percent = sprintf "%.2f", ($transferred * 100 / $total);
6373
6374                     print "$job: transferred: $transferred bytes remaining: $remaining bytes total: $total bytes progression: $percent % busy: $busy ready: $ready \n";
6375                 }
6376
6377                 $readycounter++ if $running_mirror_jobs->{$job}->{ready};
6378             }
6379
6380             last if scalar(keys %$jobs) == 0;
6381
6382             if ($readycounter == scalar(keys %$jobs)) {
6383                 print "all mirroring jobs are ready \n";
6384                 last if $skipcomplete; #do the complete later
6385
6386                 if ($vmiddst && $vmiddst != $vmid) {
6387                     my $agent_running = $qga && qga_check_running($vmid);
6388                     if ($agent_running) {
6389                         print "freeze filesystem\n";
6390                         eval { PVE::QemuServer::vm_mon_cmd($vmid, "guest-fsfreeze-freeze"); };
6391                     } else {
6392                         print "suspend vm\n";
6393                         eval { PVE::QemuServer::vm_suspend($vmid, 1); };
6394                     }
6395
6396                     # if we clone a disk for a new target vm, we don't switch the disk
6397                     PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs);
6398
6399                     if ($agent_running) {
6400                         print "unfreeze filesystem\n";
6401                         eval { PVE::QemuServer::vm_mon_cmd($vmid, "guest-fsfreeze-thaw"); };
6402                     } else {
6403                         print "resume vm\n";
6404                         eval {  PVE::QemuServer::vm_resume($vmid, 1, 1); };
6405                     }
6406
6407                     last;
6408                 } else {
6409
6410                     foreach my $job (keys %$jobs) {
6411                         # try to switch the disk if source and destination are on the same guest
6412                         print "$job: Completing block job...\n";
6413
6414                         eval { vm_mon_cmd($vmid, "block-job-complete", device => $job) };
6415                         if ($@ =~ m/cannot be completed/) {
6416                             print "$job: Block job cannot be completed, try again.\n";
6417                             $err_complete++;
6418                         }else {
6419                             print "$job: Completed successfully.\n";
6420                             $jobs->{$job}->{complete} = 1;
6421                         }
6422                     }
6423                 }
6424             }
6425             sleep 1;
6426         }
6427     };
6428     my $err = $@;
6429
6430     if ($err) {
6431         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6432         die "mirroring error: $err";
6433     }
6434
6435 }
6436
6437 sub qemu_blockjobs_cancel {
6438     my ($vmid, $jobs) = @_;
6439
6440     foreach my $job (keys %$jobs) {
6441         print "$job: Cancelling block job\n";
6442         eval { vm_mon_cmd($vmid, "block-job-cancel", device => $job); };
6443         $jobs->{$job}->{cancel} = 1;
6444     }
6445
6446     while (1) {
6447         my $stats = vm_mon_cmd($vmid, "query-block-jobs");
6448
6449         my $running_jobs = {};
6450         foreach my $stat (@$stats) {
6451             $running_jobs->{$stat->{device}} = $stat;
6452         }
6453
6454         foreach my $job (keys %$jobs) {
6455
6456             if (defined($jobs->{$job}->{cancel}) && !defined($running_jobs->{$job})) {
6457                 print "$job: Done.\n";
6458                 delete $jobs->{$job};
6459             }
6460         }
6461
6462         last if scalar(keys %$jobs) == 0;
6463
6464         sleep 1;
6465     }
6466 }
6467
6468 sub clone_disk {
6469     my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
6470         $newvmid, $storage, $format, $full, $newvollist, $jobs, $skipcomplete, $qga) = @_;
6471
6472     my $newvolid;
6473
6474     if (!$full) {
6475         print "create linked clone of drive $drivename ($drive->{file})\n";
6476         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
6477         push @$newvollist, $newvolid;
6478     } else {
6479
6480         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6481         $storeid = $storage if $storage;
6482
6483         my $dst_format = resolve_dst_disk_format($storecfg, $storeid, $volname, $format);
6484         my ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 3);
6485
6486         print "create full clone of drive $drivename ($drive->{file})\n";
6487         my $name = undef;
6488         if (drive_is_cloudinit($drive)) {
6489             $name = "vm-$newvmid-cloudinit";
6490             # cloudinit only supports raw and qcow2 atm:
6491             if ($dst_format eq 'qcow2') {
6492                 $name .= '.qcow2';
6493             } elsif ($dst_format ne 'raw') {
6494                 die "clone: unhandled format for cloudinit image\n";
6495             }
6496         }
6497         $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $newvmid, $dst_format, $name, ($size/1024));
6498         push @$newvollist, $newvolid;
6499
6500         PVE::Storage::activate_volumes($storecfg, [$newvolid]);
6501
6502         my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
6503         if (!$running || $snapname) {
6504             qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit);
6505         } else {
6506
6507             my $kvmver = get_running_qemu_version ($vmid);
6508             if (!qemu_machine_feature_enabled (undef, $kvmver, 2, 7)) {
6509                 die "drive-mirror with iothread requires qemu version 2.7 or higher\n"
6510                     if $drive->{iothread};
6511             }
6512
6513             qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid, $sparseinit, $jobs, $skipcomplete, $qga);
6514         }
6515     }
6516
6517     my ($size) = PVE::Storage::volume_size_info($storecfg, $newvolid, 3);
6518
6519     my $disk = $drive;
6520     $disk->{format} = undef;
6521     $disk->{file} = $newvolid;
6522     $disk->{size} = $size;
6523
6524     return $disk;
6525 }
6526
6527 # this only works if VM is running
6528 sub get_current_qemu_machine {
6529     my ($vmid) = @_;
6530
6531     my $cmd = { execute => 'query-machines', arguments => {} };
6532     my $res = vm_qmp_command($vmid, $cmd);
6533
6534     my ($current, $default);
6535     foreach my $e (@$res) {
6536         $default = $e->{name} if $e->{'is-default'};
6537         $current = $e->{name} if $e->{'is-current'};
6538     }
6539
6540     # fallback to the default machine if current is not supported by qemu
6541     return $current || $default || 'pc';
6542 }
6543
6544 sub get_running_qemu_version {
6545     my ($vmid) = @_;
6546     my $cmd = { execute => 'query-version', arguments => {} };
6547     my $res = vm_qmp_command($vmid, $cmd);
6548     return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
6549 }
6550
6551 sub qemu_machine_feature_enabled {
6552     my ($machine, $kvmver, $version_major, $version_minor) = @_;
6553
6554     my $current_major;
6555     my $current_minor;
6556
6557     if ($machine && $machine =~ m/^(pc(-i440fx|-q35)?-(\d+)\.(\d+))/) {
6558
6559         $current_major = $3;
6560         $current_minor = $4;
6561
6562     } elsif ($kvmver =~ m/^(\d+)\.(\d+)/) {
6563
6564         $current_major = $1;
6565         $current_minor = $2;
6566     }
6567
6568     return 1 if $current_major >= $version_major && $current_minor >= $version_minor;
6569
6570
6571 }
6572
6573 sub qemu_machine_pxe {
6574     my ($vmid, $conf, $machine) = @_;
6575
6576     $machine =  PVE::QemuServer::get_current_qemu_machine($vmid) if !$machine;
6577
6578     if ($conf->{machine} && $conf->{machine} =~ m/\.pxe$/) {
6579         $machine .= '.pxe';
6580     }
6581
6582     return $machine;
6583 }
6584
6585 sub qemu_use_old_bios_files {
6586     my ($machine_type) = @_;
6587
6588     return if !$machine_type;
6589
6590     my $use_old_bios_files = undef;
6591
6592     if ($machine_type =~ m/^(\S+)\.pxe$/) {
6593         $machine_type = $1;
6594         $use_old_bios_files = 1;
6595     } else {
6596         my $kvmver = kvm_user_version();
6597         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
6598         # load new efi bios files on migration. So this hack is required to allow
6599         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
6600         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
6601         $use_old_bios_files = !qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 4);
6602     }
6603
6604     return ($use_old_bios_files, $machine_type);
6605 }
6606
6607 sub create_efidisk {
6608     my ($storecfg, $storeid, $vmid, $fmt) = @_;
6609
6610     die "EFI vars default image not found\n" if ! -f $OVMF_VARS;
6611
6612     my $vars_size = PVE::Tools::convert_size(-s $OVMF_VARS, 'b' => 'kb');
6613     my $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $fmt, undef, $vars_size);
6614     PVE::Storage::activate_volumes($storecfg, [$volid]);
6615
6616     my $path = PVE::Storage::path($storecfg, $volid);
6617     eval {
6618         run_command(['/usr/bin/qemu-img', 'convert', '-n', '-f', 'raw', '-O', $fmt, $OVMF_VARS, $path]);
6619     };
6620     die "Copying EFI vars image failed: $@" if $@;
6621
6622     return ($volid, $vars_size);
6623 }
6624
6625 sub lspci {
6626
6627     my $devices = {};
6628
6629     dir_glob_foreach("$pcisysfs/devices", '[a-f0-9]{4}:([a-f0-9]{2}:[a-f0-9]{2})\.([0-9])', sub {
6630             my (undef, $id, $function) = @_;
6631             my $res = { id => $id, function => $function};
6632             push @{$devices->{$id}}, $res;
6633     });
6634
6635     # Entries should be sorted by functions.
6636     foreach my $id (keys %$devices) {
6637         my $dev = $devices->{$id};
6638         $devices->{$id} = [ sort { $a->{function} <=> $b->{function} } @$dev ];
6639     }
6640
6641     return $devices;
6642 }
6643
6644 sub vm_iothreads_list {
6645     my ($vmid) = @_;
6646
6647     my $res = vm_mon_cmd($vmid, 'query-iothreads');
6648
6649     my $iothreads = {};
6650     foreach my $iothread (@$res) {
6651         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
6652     }
6653
6654     return $iothreads;
6655 }
6656
6657 sub scsihw_infos {
6658     my ($conf, $drive) = @_;
6659
6660     my $maxdev = 0;
6661
6662     if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)) {
6663         $maxdev = 7;
6664     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
6665         $maxdev = 1;
6666     } else {
6667         $maxdev = 256;
6668     }
6669
6670     my $controller = int($drive->{index} / $maxdev);
6671     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single') ? "virtioscsi" : "scsihw";
6672
6673     return ($maxdev, $controller, $controller_prefix);
6674 }
6675
6676 sub add_hyperv_enlightenments {
6677     my ($cpuFlags, $winversion, $machine_type, $kvmver, $bios, $gpu_passthrough) = @_;
6678
6679     return if $winversion < 6;
6680     return if $bios && $bios eq 'ovmf' && $winversion < 8;
6681
6682     push @$cpuFlags , 'hv_vendor_id=proxmox' if $gpu_passthrough;
6683
6684     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
6685         push @$cpuFlags , 'hv_spinlocks=0x1fff';
6686         push @$cpuFlags , 'hv_vapic';
6687         push @$cpuFlags , 'hv_time';
6688     } else {
6689         push @$cpuFlags , 'hv_spinlocks=0xffff';
6690     }
6691
6692     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 6)) {
6693         push @$cpuFlags , 'hv_reset';
6694         push @$cpuFlags , 'hv_vpindex';
6695         push @$cpuFlags , 'hv_runtime';
6696     }
6697
6698     if ($winversion >= 7) {
6699         push @$cpuFlags , 'hv_relaxed';
6700
6701         if (qemu_machine_feature_enabled ($machine_type, $kvmver, 3, 0)) {
6702             push @$cpuFlags , 'hv_synic';
6703             push @$cpuFlags , 'hv_stimer';
6704         }
6705     }
6706 }
6707
6708 sub windows_version {
6709     my ($ostype) = @_;
6710
6711     return 0 if !$ostype;
6712
6713     my $winversion = 0;
6714
6715     if($ostype eq 'wxp' || $ostype eq 'w2k3' || $ostype eq 'w2k') {
6716         $winversion = 5;
6717     } elsif($ostype eq 'w2k8' || $ostype eq 'wvista') {
6718         $winversion = 6;
6719     } elsif ($ostype =~ m/^win(\d+)$/) {
6720         $winversion = $1;
6721     }
6722
6723     return $winversion;
6724 }
6725
6726 sub resolve_dst_disk_format {
6727         my ($storecfg, $storeid, $src_volname, $format) = @_;
6728         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
6729
6730         if (!$format) {
6731             # if no target format is specified, use the source disk format as hint
6732             if ($src_volname) {
6733                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6734                 $format = qemu_img_format($scfg, $src_volname);
6735             } else {
6736                 return $defFormat;
6737             }
6738         }
6739
6740         # test if requested format is supported - else use default
6741         my $supported = grep { $_ eq $format } @$validFormats;
6742         $format = $defFormat if !$supported;
6743         return $format;
6744 }
6745
6746 sub resolve_first_disk {
6747     my $conf = shift;
6748     my @disks = PVE::QemuServer::valid_drive_names();
6749     my $firstdisk;
6750     foreach my $ds (reverse @disks) {
6751         next if !$conf->{$ds};
6752         my $disk = PVE::QemuServer::parse_drive($ds, $conf->{$ds});
6753         next if PVE::QemuServer::drive_is_cdrom($disk);
6754         $firstdisk = $ds;
6755     }
6756     return $firstdisk;
6757 }
6758
6759 sub generate_smbios1_uuid {
6760     my ($uuid, $uuid_str);
6761     UUID::generate($uuid);
6762     UUID::unparse($uuid, $uuid_str);
6763     return "uuid=$uuid_str";
6764 }
6765
6766 # bash completion helper
6767
6768 sub complete_backup_archives {
6769     my ($cmdname, $pname, $cvalue) = @_;
6770
6771     my $cfg = PVE::Storage::config();
6772
6773     my $storeid;
6774
6775     if ($cvalue =~ m/^([^:]+):/) {
6776         $storeid = $1;
6777     }
6778
6779     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
6780
6781     my $res = [];
6782     foreach my $id (keys %$data) {
6783         foreach my $item (@{$data->{$id}}) {
6784             next if $item->{format} !~ m/^vma\.(gz|lzo)$/;
6785             push @$res, $item->{volid} if defined($item->{volid});
6786         }
6787     }
6788
6789     return $res;
6790 }
6791
6792 my $complete_vmid_full = sub {
6793     my ($running) = @_;
6794
6795     my $idlist = vmstatus();
6796
6797     my $res = [];
6798
6799     foreach my $id (keys %$idlist) {
6800         my $d = $idlist->{$id};
6801         if (defined($running)) {
6802             next if $d->{template};
6803             next if $running && $d->{status} ne 'running';
6804             next if !$running && $d->{status} eq 'running';
6805         }
6806         push @$res, $id;
6807
6808     }
6809     return $res;
6810 };
6811
6812 sub complete_vmid {
6813     return &$complete_vmid_full();
6814 }
6815
6816 sub complete_vmid_stopped {
6817     return &$complete_vmid_full(0);
6818 }
6819
6820 sub complete_vmid_running {
6821     return &$complete_vmid_full(1);
6822 }
6823
6824 sub complete_storage {
6825
6826     my $cfg = PVE::Storage::config();
6827     my $ids = $cfg->{ids};
6828
6829     my $res = [];
6830     foreach my $sid (keys %$ids) {
6831         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
6832         next if !$ids->{$sid}->{content}->{images};
6833         push @$res, $sid;
6834     }
6835
6836     return $res;
6837 }
6838
6839 sub nbd_stop {
6840     my ($vmid) = @_;
6841
6842     vm_mon_cmd($vmid, 'nbd-server-stop');
6843 }
6844
6845 1;