PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5 use POSIX;
   6 use IO::Handle;
   7 use IO::Select;
   8 use IO::File;
   9 use IO::Dir;
  10 use IO::Socket::UNIX;
  11 use File::Basename;
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use Digest::SHA;
  16 use Fcntl ':flock';
  17 use Cwd 'abs_path';
  18 use IPC::Open3;
  19 use JSON;
  20 use Fcntl;
  21 use PVE::SafeSyslog;
  22 use Storable qw(dclone);
  23 use PVE::Exception qw(raise raise_param_exc);
  24 use PVE::Storage;
  25 use PVE::Tools qw(run_command lock_file lock_file_full file_read_firstline dir_glob_foreach $IPV6RE);
  26 use PVE::JSONSchema qw(get_standard_option);
  27 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file cfs_lock_file);
  28 use PVE::INotify;
  29 use PVE::ProcFSTools;
  30 use PVE::QemuConfig;
  31 use PVE::QMPClient;
  32 use PVE::RPCEnvironment;
  33 use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr);
  34 use PVE::QemuServer::Memory;
  35 use PVE::QemuServer::USB qw(parse_usb_device);
  36 use PVE::QemuServer::Cloudinit;
  37 use Time::HiRes qw(gettimeofday);
  38 use File::Copy qw(copy);
  39 use URI::Escape;
  40
  41 my $EDK2_FW_BASE = '/usr/share/pve-edk2-firmware/';
  42 my $OVMF_CODE = "$EDK2_FW_BASE/OVMF_CODE.fd";
  43 my $OVMF_VARS = "$EDK2_FW_BASE/OVMF_VARS.fd";
  44
  45 my $qemu_snap_storage = {rbd => 1, sheepdog => 1};
  46
  47 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  48
  49 my $QEMU_FORMAT_RE = qr/raw|cow|qcow|qcow2|qed|vmdk|cloop/;
  50
  51 # Note about locking: we use flock on the config file protect
  52 # against concurent actions.
  53 # Aditionaly, we have a 'lock' setting in the config file. This
  54 # can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
  55 # allowed when such lock is set. But you can ignore this kind of
  56 # lock with the --skiplock flag.
  57
  58 cfs_register_file('/qemu-server/',
  59                   \&parse_vm_config,
  60                   \&write_vm_config);
  61
  62 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
  63     description => "Some command save/restore state from this location.",
  64     type => 'string',
  65     maxLength => 128,
  66     optional => 1,
  67 });
  68
  69 PVE::JSONSchema::register_standard_option('pve-snapshot-name', {
  70     description => "The name of the snapshot.",
  71     type => 'string', format => 'pve-configid',
  72     maxLength => 40,
  73 });
  74
  75 PVE::JSONSchema::register_standard_option('pve-qm-image-format', {
  76     type => 'string',
  77     enum => [qw(raw cow qcow qed qcow2 vmdk cloop)],
  78     description => "The drive's backing file's data format.",
  79     optional => 1,
  80 });
  81
  82 #no warnings 'redefine';
  83
  84 sub cgroups_write {
  85    my ($controller, $vmid, $option, $value) = @_;
  86
  87    my $path = "/sys/fs/cgroup/$controller/qemu.slice/$vmid.scope/$option";
  88    PVE::ProcFSTools::write_proc_entry($path, $value);
  89
  90 }
  91
  92 my $nodename = PVE::INotify::nodename();
  93
  94 mkdir "/etc/pve/nodes/$nodename";
  95 my $confdir = "/etc/pve/nodes/$nodename/qemu-server";
  96 mkdir $confdir;
  97
  98 my $var_run_tmpdir = "/var/run/qemu-server";
  99 mkdir $var_run_tmpdir;
 100
 101 my $lock_dir = "/var/lock/qemu-server";
 102 mkdir $lock_dir;
 103
 104 my $pcisysfs = "/sys/bus/pci";
 105
 106 my $cpu_vendor_list = {
 107     # Intel CPUs
 108     486 => 'GenuineIntel',
 109     pentium => 'GenuineIntel',
 110     pentium2  => 'GenuineIntel',
 111     pentium3  => 'GenuineIntel',
 112     coreduo => 'GenuineIntel',
 113     core2duo => 'GenuineIntel',
 114     Conroe  => 'GenuineIntel',
 115     Penryn  => 'GenuineIntel',
 116     Nehalem  => 'GenuineIntel',
 117     'Nehalem-IBRS'  => 'GenuineIntel',
 118     Westmere => 'GenuineIntel',
 119     'Westmere-IBRS' => 'GenuineIntel',
 120     SandyBridge => 'GenuineIntel',
 121     'SandyBridge-IBRS' => 'GenuineIntel',
 122     IvyBridge => 'GenuineIntel',
 123     'IvyBridge-IBRS' => 'GenuineIntel',
 124     Haswell => 'GenuineIntel',
 125     'Haswell-IBRS' => 'GenuineIntel',
 126     'Haswell-noTSX' => 'GenuineIntel',
 127     'Haswell-noTSX-IBRS' => 'GenuineIntel',
 128     Broadwell => 'GenuineIntel',
 129     'Broadwell-IBRS' => 'GenuineIntel',
 130     'Broadwell-noTSX' => 'GenuineIntel',
 131     'Broadwell-noTSX-IBRS' => 'GenuineIntel',
 132     'Skylake-Client' => 'GenuineIntel',
 133     'Skylake-Client-IBRS' => 'GenuineIntel',
 134     'Skylake-Server' => 'GenuineIntel',
 135     'Skylake-Server-IBRS' => 'GenuineIntel',
 136
 137     # AMD CPUs
 138     athlon => 'AuthenticAMD',
 139     phenom  => 'AuthenticAMD',
 140     Opteron_G1  => 'AuthenticAMD',
 141     Opteron_G2  => 'AuthenticAMD',
 142     Opteron_G3  => 'AuthenticAMD',
 143     Opteron_G4  => 'AuthenticAMD',
 144     Opteron_G5  => 'AuthenticAMD',
 145     EPYC => 'AuthenticAMD',
 146     'EPYC-IBPB' => 'AuthenticAMD',
 147
 148     # generic types, use vendor from host node
 149     host => 'default',
 150     kvm32 => 'default',
 151     kvm64 => 'default',
 152     qemu32 => 'default',
 153     qemu64 => 'default',
 154     max => 'default',
 155 };
 156
 157 my $cpu_flag = qr/[+-](pcid|spec-ctrl)/;
 158
 159 my $cpu_fmt = {
 160     cputype => {
 161         description => "Emulated CPU type.",
 162         type => 'string',
 163         enum => [ sort { "\L$a" cmp "\L$b" } keys %$cpu_vendor_list ],
 164         default => 'kvm64',
 165         default_key => 1,
 166     },
 167     hidden => {
 168         description => "Do not identify as a KVM virtual machine.",
 169         type => 'boolean',
 170         optional => 1,
 171         default => 0
 172     },
 173     flags => {
 174         description => "List of additional CPU flags separated by ';'."
 175                      . " Use '+FLAG' to enable, '-FLAG' to disable a flag."
 176                      . " Currently supported flags: 'pcid', 'spec-ctrl'.",
 177         format_description => '+FLAG[;-FLAG...]',
 178         type => 'string',
 179         pattern => qr/$cpu_flag(;$cpu_flag)*/,
 180         optional => 1,
 181     },
 182 };
 183
 184 my $watchdog_fmt = {
 185     model => {
 186         default_key => 1,
 187         type => 'string',
 188         enum => [qw(i6300esb ib700)],
 189         description => "Watchdog type to emulate.",
 190         default => 'i6300esb',
 191         optional => 1,
 192     },
 193     action => {
 194         type => 'string',
 195         enum => [qw(reset shutdown poweroff pause debug none)],
 196         description => "The action to perform if after activation the guest fails to poll the watchdog in time.",
 197         optional => 1,
 198     },
 199 };
 200 PVE::JSONSchema::register_format('pve-qm-watchdog', $watchdog_fmt);
 201
 202 my $confdesc = {
 203     onboot => {
 204         optional => 1,
 205         type => 'boolean',
 206         description => "Specifies whether a VM will be started during system bootup.",
 207         default => 0,
 208     },
 209     autostart => {
 210         optional => 1,
 211         type => 'boolean',
 212         description => "Automatic restart after crash (currently ignored).",
 213         default => 0,
 214     },
 215     hotplug => {
 216         optional => 1,
 217         type => 'string', format => 'pve-hotplug-features',
 218         description => "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
 219         default => 'network,disk,usb',
 220     },
 221     reboot => {
 222         optional => 1,
 223         type => 'boolean',
 224         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 225         default => 1,
 226     },
 227     lock => {
 228         optional => 1,
 229         type => 'string',
 230         description => "Lock/unlock the VM.",
 231         enum => [qw(migrate backup snapshot rollback)],
 232     },
 233     cpulimit => {
 234         optional => 1,
 235         type => 'number',
 236         description => "Limit of CPU usage.",
 237         verbose_description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
 238         minimum => 0,
 239         maximum => 128,
 240         default => 0,
 241     },
 242     cpuunits => {
 243         optional => 1,
 244         type => 'integer',
 245         description => "CPU weight for a VM.",
 246         verbose_description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.",
 247         minimum => 2,
 248         maximum => 262144,
 249         default => 1024,
 250     },
 251     memory => {
 252         optional => 1,
 253         type => 'integer',
 254         description => "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
 255         minimum => 16,
 256         default => 512,
 257     },
 258     balloon => {
 259         optional => 1,
 260         type => 'integer',
 261         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 262         minimum => 0,
 263     },
 264     shares => {
 265         optional => 1,
 266         type => 'integer',
 267         description => "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning",
 268         minimum => 0,
 269         maximum => 50000,
 270         default => 1000,
 271     },
 272     keyboard => {
 273         optional => 1,
 274         type => 'string',
 275         description => "Keybord layout for vnc server. Default is read from the '/etc/pve/datacenter.conf' configuration file.".
 276                        "It should not be necessary to set it.",
 277         enum => PVE::Tools::kvmkeymaplist(),
 278         default => undef,
 279     },
 280     name => {
 281         optional => 1,
 282         type => 'string', format => 'dns-name',
 283         description => "Set a name for the VM. Only used on the configuration web interface.",
 284     },
 285     scsihw => {
 286         optional => 1,
 287         type => 'string',
 288         description => "SCSI controller model",
 289         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 290         default => 'lsi',
 291     },
 292     description => {
 293         optional => 1,
 294         type => 'string',
 295         description => "Description for the VM. Only used on the configuration web interface. This is saved as comment inside the configuration file.",
 296     },
 297     ostype => {
 298         optional => 1,
 299         type => 'string',
 300         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 l24 l26 solaris)],
 301         description => "Specify guest operating system.",
 302         verbose_description => <<EODESC,
 303 Specify guest operating system. This is used to enable special
 304 optimization/features for specific operating systems:
 305
 306 [horizontal]
 307 other;; unspecified OS
 308 wxp;; Microsoft Windows XP
 309 w2k;; Microsoft Windows 2000
 310 w2k3;; Microsoft Windows 2003
 311 w2k8;; Microsoft Windows 2008
 312 wvista;; Microsoft Windows Vista
 313 win7;; Microsoft Windows 7
 314 win8;; Microsoft Windows 8/2012/2012r2
 315 win10;; Microsoft Windows 10/2016
 316 l24;; Linux 2.4 Kernel
 317 l26;; Linux 2.6/3.X Kernel
 318 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
 319 EODESC
 320     },
 321     boot => {
 322         optional => 1,
 323         type => 'string',
 324         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n).",
 325         pattern => '[acdn]{1,4}',
 326         default => 'cdn',
 327     },
 328     bootdisk => {
 329         optional => 1,
 330         type => 'string', format => 'pve-qm-bootdisk',
 331         description => "Enable booting from specified disk.",
 332         pattern => '(ide|sata|scsi|virtio)\d+',
 333     },
 334     smp => {
 335         optional => 1,
 336         type => 'integer',
 337         description => "The number of CPUs. Please use option -sockets instead.",
 338         minimum => 1,
 339         default => 1,
 340     },
 341     sockets => {
 342         optional => 1,
 343         type => 'integer',
 344         description => "The number of CPU sockets.",
 345         minimum => 1,
 346         default => 1,
 347     },
 348     cores => {
 349         optional => 1,
 350         type => 'integer',
 351         description => "The number of cores per socket.",
 352         minimum => 1,
 353         default => 1,
 354     },
 355     numa => {
 356         optional => 1,
 357         type => 'boolean',
 358         description => "Enable/disable NUMA.",
 359         default => 0,
 360     },
 361     hugepages => {
 362         optional => 1,
 363         type => 'string',
 364         description => "Enable/disable hugepages memory.",
 365         enum => [qw(any 2 1024)],
 366     },
 367     vcpus => {
 368         optional => 1,
 369         type => 'integer',
 370         description => "Number of hotplugged vcpus.",
 371         minimum => 1,
 372         default => 0,
 373     },
 374     acpi => {
 375         optional => 1,
 376         type => 'boolean',
 377         description => "Enable/disable ACPI.",
 378         default => 1,
 379     },
 380     agent => {
 381         optional => 1,
 382         type => 'boolean',
 383         description => "Enable/disable Qemu GuestAgent.",
 384         default => 0,
 385     },
 386     kvm => {
 387         optional => 1,
 388         type => 'boolean',
 389         description => "Enable/disable KVM hardware virtualization.",
 390         default => 1,
 391     },
 392     tdf => {
 393         optional => 1,
 394         type => 'boolean',
 395         description => "Enable/disable time drift fix.",
 396         default => 0,
 397     },
 398     localtime => {
 399         optional => 1,
 400         type => 'boolean',
 401         description => "Set the real time clock to local time. This is enabled by default if ostype indicates a Microsoft OS.",
 402     },
 403     freeze => {
 404         optional => 1,
 405         type => 'boolean',
 406         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 407     },
 408     vga => {
 409         optional => 1,
 410         type => 'string',
 411         description => "Select the VGA type.",
 412         verbose_description => "Select the VGA type. If you want to use high resolution" .
 413             " modes (>= 1280x1024x16) then you should use the options " .
 414             "'std' or 'vmware'. Default is 'std' for win8/win7/w2k8, and " .
 415             "'cirrus' for other OS types. The 'qxl' option enables the SPICE " .
 416             "display sever. For win* OS you can select how many independent " .
 417             "displays you want, Linux guests can add displays them self. " .
 418             "You can also run without any graphic card, using a serial device" .
 419             " as terminal.",
 420         enum => [qw(std cirrus vmware qxl serial0 serial1 serial2 serial3 qxl2 qxl3 qxl4)],
 421     },
 422     watchdog => {
 423         optional => 1,
 424         type => 'string', format => 'pve-qm-watchdog',
 425         description => "Create a virtual hardware watchdog device.",
 426         verbose_description => "Create a virtual hardware watchdog device. Once enabled" .
 427             " (by a guest action), the watchdog must be periodically polled " .
 428             "by an agent inside the guest or else the watchdog will reset " .
 429             "the guest (or execute the respective action specified)",
 430     },
 431     startdate => {
 432         optional => 1,
 433         type => 'string',
 434         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 435         description => "Set the initial date of the real time clock. Valid format for date are: 'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 436         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 437         default => 'now',
 438     },
 439     startup =>  get_standard_option('pve-startup-order'),
 440     template => {
 441         optional => 1,
 442         type => 'boolean',
 443         description => "Enable/disable Template.",
 444         default => 0,
 445     },
 446     args => {
 447         optional => 1,
 448         type => 'string',
 449         description => "Arbitrary arguments passed to kvm.",
 450         verbose_description => <<EODESCR,
 451 Arbitrary arguments passed to kvm, for example:
 452
 453 args: -no-reboot -no-hpet
 454
 455 NOTE: this option is for experts only.
 456 EODESCR
 457     },
 458     tablet => {
 459         optional => 1,
 460         type => 'boolean',
 461         default => 1,
 462         description => "Enable/disable the USB tablet device.",
 463         verbose_description => "Enable/disable the USB tablet device. This device is " .
 464             "usually needed to allow absolute mouse positioning with VNC. " .
 465             "Else the mouse runs out of sync with normal VNC clients. " .
 466             "If you're running lots of console-only guests on one host, " .
 467             "you may consider disabling this to save some context switches. " .
 468             "This is turned off by default if you use spice (-vga=qxl).",
 469     },
 470     migrate_speed => {
 471         optional => 1,
 472         type => 'integer',
 473         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 474         minimum => 0,
 475         default => 0,
 476     },
 477     migrate_downtime => {
 478         optional => 1,
 479         type => 'number',
 480         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 481         minimum => 0,
 482         default => 0.1,
 483     },
 484     cdrom => {
 485         optional => 1,
 486         type => 'string', format => 'pve-qm-ide',
 487         typetext => '<volume>',
 488         description => "This is an alias for option -ide2",
 489     },
 490     cpu => {
 491         optional => 1,
 492         description => "Emulated CPU type.",
 493         type => 'string',
 494         format => $cpu_fmt,
 495     },
 496     parent => get_standard_option('pve-snapshot-name', {
 497         optional => 1,
 498         description => "Parent snapshot name. This is used internally, and should not be modified.",
 499     }),
 500     snaptime => {
 501         optional => 1,
 502         description => "Timestamp for snapshots.",
 503         type => 'integer',
 504         minimum => 0,
 505     },
 506     vmstate => {
 507         optional => 1,
 508         type => 'string', format => 'pve-volume-id',
 509         description => "Reference to a volume which stores the VM state. This is used internally for snapshots.",
 510     },
 511     vmstatestorage => get_standard_option('pve-storage-id', {
 512         description => "Default storage for VM state volumes/files.",
 513         optional => 1,
 514     }),
 515     machine => {
 516         description => "Specific the Qemu machine type.",
 517         type => 'string',
 518         pattern => '(pc|pc(-i440fx)?-\d+\.\d+(\.pxe)?|q35|pc-q35-\d+\.\d+(\.pxe)?)',
 519         maxLength => 40,
 520         optional => 1,
 521     },
 522     smbios1 => {
 523         description => "Specify SMBIOS type 1 fields.",
 524         type => 'string', format => 'pve-qm-smbios1',
 525         maxLength => 256,
 526         optional => 1,
 527     },
 528     protection => {
 529         optional => 1,
 530         type => 'boolean',
 531         description => "Sets the protection flag of the VM. This will disable the remove VM and remove disk operations.",
 532         default => 0,
 533     },
 534     bios => {
 535         optional => 1,
 536         type => 'string',
 537         enum => [ qw(seabios ovmf) ],
 538         description => "Select BIOS implementation.",
 539         default => 'seabios',
 540     },
 541 };
 542
 543 my $confdesc_cloudinit = {
 544     citype => {
 545         optional => 1,
 546         type => 'string',
 547         description => 'Specifies the cloud-init configuration format. The default depends on the configured operating system type (`ostype`. We use the `nocloud` format for Linux, and `configdrive2` for windows.',
 548         enum => ['configdrive2', 'nocloud'],
 549     },
 550     ciuser => {
 551         optional => 1,
 552         type => 'string',
 553         description => "cloud-init: User name to change ssh keys and password for instead of the image's configured default user.",
 554     },
 555     cipassword => {
 556         optional => 1,
 557         type => 'string',
 558         description => 'cloud-init: Password to assign the user. Using this is generally not recommended. Use ssh keys instead. Also note that older cloud-init versions do not support hashed passwords.',
 559     },
 560     searchdomain => {
 561         optional => 1,
 562         type => 'string',
 563         description => "cloud-init: Sets DNS search domains for a container. Create will automatically use the setting from the host if neither searchdomain nor nameserver are set.",
 564     },
 565     nameserver => {
 566         optional => 1,
 567         type => 'string', format => 'address-list',
 568         description => "cloud-init: Sets DNS server IP address for a container. Create will automatically use the setting from the host if neither searchdomain nor nameserver are set.",
 569     },
 570     sshkeys => {
 571         optional => 1,
 572         type => 'string',
 573         format => 'urlencoded',
 574         description => "cloud-init: Setup public SSH keys (one key per line, OpenSSH format).",
 575     },
 576 };
 577
 578 # what about other qemu settings ?
 579 #cpu => 'string',
 580 #machine => 'string',
 581 #fda => 'file',
 582 #fdb => 'file',
 583 #mtdblock => 'file',
 584 #sd => 'file',
 585 #pflash => 'file',
 586 #snapshot => 'bool',
 587 #bootp => 'file',
 588 ##tftp => 'dir',
 589 ##smb => 'dir',
 590 #kernel => 'file',
 591 #append => 'string',
 592 #initrd => 'file',
 593 ##soundhw => 'string',
 594
 595 while (my ($k, $v) = each %$confdesc) {
 596     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 597 }
 598
 599 my $MAX_IDE_DISKS = 4;
 600 my $MAX_SCSI_DISKS = 14;
 601 my $MAX_VIRTIO_DISKS = 16;
 602 my $MAX_SATA_DISKS = 6;
 603 my $MAX_USB_DEVICES = 5;
 604 my $MAX_NETS = 32;
 605 my $MAX_UNUSED_DISKS = 8;
 606 my $MAX_HOSTPCI_DEVICES = 4;
 607 my $MAX_SERIAL_PORTS = 4;
 608 my $MAX_PARALLEL_PORTS = 3;
 609 my $MAX_NUMA = 8;
 610
 611 my $numa_fmt = {
 612     cpus => {
 613         type => "string",
 614         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 615         description => "CPUs accessing this NUMA node.",
 616         format_description => "id[-id];...",
 617     },
 618     memory => {
 619         type => "number",
 620         description => "Amount of memory this NUMA node provides.",
 621         optional => 1,
 622     },
 623     hostnodes => {
 624         type => "string",
 625         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 626         description => "Host NUMA nodes to use.",
 627         format_description => "id[-id];...",
 628         optional => 1,
 629     },
 630     policy => {
 631         type => 'string',
 632         enum => [qw(preferred bind interleave)],
 633         description => "NUMA allocation policy.",
 634         optional => 1,
 635     },
 636 };
 637 PVE::JSONSchema::register_format('pve-qm-numanode', $numa_fmt);
 638 my $numadesc = {
 639     optional => 1,
 640     type => 'string', format => $numa_fmt,
 641     description => "NUMA topology.",
 642 };
 643 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 644
 645 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 646     $confdesc->{"numa$i"} = $numadesc;
 647 }
 648
 649 my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000',  'pcnet',  'virtio',
 650                       'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3',
 651                       'e1000-82540em', 'e1000-82544gc', 'e1000-82545em'];
 652 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 653
 654 my $net_fmt_bridge_descr = <<__EOD__;
 655 Bridge to attach the network device to. The Proxmox VE standard bridge
 656 is called 'vmbr0'.
 657
 658 If you do not specify a bridge, we create a kvm user (NATed) network
 659 device, which provides DHCP and DNS services. The following addresses
 660 are used:
 661
 662  10.0.2.2   Gateway
 663  10.0.2.3   DNS Server
 664  10.0.2.4   SMB Server
 665
 666 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 667 __EOD__
 668
 669 my $net_fmt = {
 670     macaddr => {
 671         type => 'string',
 672         pattern => qr/[0-9a-f]{2}(?::[0-9a-f]{2}){5}/i,
 673         description => "MAC address. That address must be unique withing your network. This is automatically generated if not specified.",
 674         format_description => "XX:XX:XX:XX:XX:XX",
 675         optional => 1,
 676     },
 677     model => {
 678         type => 'string',
 679         description => "Network Card Model. The 'virtio' model provides the best performance with very low CPU overhead. If your guest does not support this driver, it is usually best to use 'e1000'.",
 680         enum => $nic_model_list,
 681         default_key => 1,
 682     },
 683     (map { $_ => { keyAlias => 'model', alias => 'macaddr' }} @$nic_model_list),
 684     bridge => {
 685         type => 'string',
 686         description => $net_fmt_bridge_descr,
 687         format_description => 'bridge',
 688         optional => 1,
 689     },
 690     queues => {
 691         type => 'integer',
 692         minimum => 0, maximum => 16,
 693         description => 'Number of packet queues to be used on the device.',
 694         optional => 1,
 695     },
 696     rate => {
 697         type => 'number',
 698         minimum => 0,
 699         description => "Rate limit in mbps (megabytes per second) as floating point number.",
 700         optional => 1,
 701     },
 702     tag => {
 703         type => 'integer',
 704         minimum => 1, maximum => 4094,
 705         description => 'VLAN tag to apply to packets on this interface.',
 706         optional => 1,
 707     },
 708     trunks => {
 709         type => 'string',
 710         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 711         description => 'VLAN trunks to pass through this interface.',
 712         format_description => 'vlanid[;vlanid...]',
 713         optional => 1,
 714     },
 715     firewall => {
 716         type => 'boolean',
 717         description => 'Whether this interface should be protected by the firewall.',
 718         optional => 1,
 719     },
 720     link_down => {
 721         type => 'boolean',
 722         description => 'Whether this interface should be disconnected (like pulling the plug).',
 723         optional => 1,
 724     },
 725 };
 726
 727 my $netdesc = {
 728     optional => 1,
 729     type => 'string', format => $net_fmt,
 730     description => "Specify network devices.",
 731 };
 732
 733 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 734
 735 my $ipconfig_fmt = {
 736     ip => {
 737         type => 'string',
 738         format => 'pve-ipv4-config',
 739         format_description => 'IPv4Format/CIDR',
 740         description => 'IPv4 address in CIDR format.',
 741         optional => 1,
 742         default => 'dhcp',
 743     },
 744     gw => {
 745         type => 'string',
 746         format => 'ipv4',
 747         format_description => 'GatewayIPv4',
 748         description => 'Default gateway for IPv4 traffic.',
 749         optional => 1,
 750         requires => 'ip',
 751     },
 752     ip6 => {
 753         type => 'string',
 754         format => 'pve-ipv6-config',
 755         format_description => 'IPv6Format/CIDR',
 756         description => 'IPv6 address in CIDR format.',
 757         optional => 1,
 758         default => 'dhcp',
 759     },
 760     gw6 => {
 761         type => 'string',
 762         format => 'ipv6',
 763         format_description => 'GatewayIPv6',
 764         description => 'Default gateway for IPv6 traffic.',
 765         optional => 1,
 766         requires => 'ip6',
 767     },
 768 };
 769 PVE::JSONSchema::register_format('pve-qm-ipconfig', $ipconfig_fmt);
 770 my $ipconfigdesc = {
 771     optional => 1,
 772     type => 'string', format => 'pve-qm-ipconfig',
 773     description => <<'EODESCR',
 774 cloud-init: Specify IP addresses and gateways for the corresponding interface.
 775
 776 IP addresses use CIDR notation, gateways are optional but need an IP of the same type specified.
 777
 778 The special string 'dhcp' can be used for IP addresses to use DHCP, in which case no explicit gateway should be provided.
 779 For IPv6 the special string 'auto' can be used to use stateless autoconfiguration.
 780
 781 If cloud-init is enabled and neither an IPv4 nor an IPv6 address is specified, it defaults to using dhcp on IPv4.
 782 EODESCR
 783 };
 784 PVE::JSONSchema::register_standard_option("pve-qm-ipconfig", $netdesc);
 785
 786 for (my $i = 0; $i < $MAX_NETS; $i++)  {
 787     $confdesc->{"net$i"} = $netdesc;
 788     $confdesc_cloudinit->{"ipconfig$i"} = $ipconfigdesc;
 789 }
 790
 791 foreach my $key (keys %$confdesc_cloudinit) {
 792     $confdesc->{$key} = $confdesc_cloudinit->{$key};
 793 }
 794
 795 PVE::JSONSchema::register_format('pve-volume-id-or-qm-path', \&verify_volume_id_or_qm_path);
 796 sub verify_volume_id_or_qm_path {
 797     my ($volid, $noerr) = @_;
 798
 799     if ($volid eq 'none' || $volid eq 'cdrom' || $volid =~ m|^/|) {
 800         return $volid;
 801     }
 802
 803     # if its neither 'none' nor 'cdrom' nor a path, check if its a volume-id
 804     $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') };
 805     if ($@) {
 806         return undef if $noerr;
 807         die $@;
 808     }
 809     return $volid;
 810 }
 811
 812 my $drivename_hash;
 813
 814 my %drivedesc_base = (
 815     volume => { alias => 'file' },
 816     file => {
 817         type => 'string',
 818         format => 'pve-volume-id-or-qm-path',
 819         default_key => 1,
 820         format_description => 'volume',
 821         description => "The drive's backing volume.",
 822     },
 823     media => {
 824         type => 'string',
 825         enum => [qw(cdrom disk)],
 826         description => "The drive's media type.",
 827         default => 'disk',
 828         optional => 1
 829     },
 830     cyls => {
 831         type => 'integer',
 832         description => "Force the drive's physical geometry to have a specific cylinder count.",
 833         optional => 1
 834     },
 835     heads => {
 836         type => 'integer',
 837         description => "Force the drive's physical geometry to have a specific head count.",
 838         optional => 1
 839     },
 840     secs => {
 841         type => 'integer',
 842         description => "Force the drive's physical geometry to have a specific sector count.",
 843         optional => 1
 844     },
 845     trans => {
 846         type => 'string',
 847         enum => [qw(none lba auto)],
 848         description => "Force disk geometry bios translation mode.",
 849         optional => 1,
 850     },
 851     snapshot => {
 852         type => 'boolean',
 853         description => "Controls qemu's snapshot mode feature."
 854             . " If activated, changes made to the disk are temporary and will"
 855             . " be discarded when the VM is shutdown.",
 856         optional => 1,
 857     },
 858     cache => {
 859         type => 'string',
 860         enum => [qw(none writethrough writeback unsafe directsync)],
 861         description => "The drive's cache mode",
 862         optional => 1,
 863     },
 864     format => get_standard_option('pve-qm-image-format'),
 865     size => {
 866         type => 'string',
 867         format => 'disk-size',
 868         format_description => 'DiskSize',
 869         description => "Disk size. This is purely informational and has no effect.",
 870         optional => 1,
 871     },
 872     backup => {
 873         type => 'boolean',
 874         description => "Whether the drive should be included when making backups.",
 875         optional => 1,
 876     },
 877     replicate => {
 878         type => 'boolean',
 879         description => 'Whether the drive should considered for replication jobs.',
 880         optional => 1,
 881         default => 1,
 882     },
 883     rerror => {
 884         type => 'string',
 885         enum => [qw(ignore report stop)],
 886         description => 'Read error action.',
 887         optional => 1,
 888     },
 889     werror => {
 890         type => 'string',
 891         enum => [qw(enospc ignore report stop)],
 892         description => 'Write error action.',
 893         optional => 1,
 894     },
 895     aio => {
 896         type => 'string',
 897         enum => [qw(native threads)],
 898         description => 'AIO type to use.',
 899         optional => 1,
 900     },
 901     discard => {
 902         type => 'string',
 903         enum => [qw(ignore on)],
 904         description => 'Controls whether to pass discard/trim requests to the underlying storage.',
 905         optional => 1,
 906     },
 907     detect_zeroes => {
 908         type => 'boolean',
 909         description => 'Controls whether to detect and try to optimize writes of zeroes.',
 910         optional => 1,
 911     },
 912     serial => {
 913         type => 'string',
 914         format => 'urlencoded',
 915         format_description => 'serial',
 916         maxLength => 20*3, # *3 since it's %xx url enoded
 917         description => "The drive's reported serial number, url-encoded, up to 20 bytes long.",
 918         optional => 1,
 919     },
 920     shared => {
 921         type => 'boolean',
 922         description => 'Mark this locally-managed volume as available on all nodes',
 923         verbose_description => "Mark this locally-managed volume as available on all nodes.\n\nWARNING: This option does not share the volume automatically, it assumes it is shared already!",
 924         optional => 1,
 925         default => 0,
 926     }
 927 );
 928
 929 my %iothread_fmt = ( iothread => {
 930         type => 'boolean',
 931         description => "Whether to use iothreads for this drive",
 932         optional => 1,
 933 });
 934
 935 my %model_fmt = (
 936     model => {
 937         type => 'string',
 938         format => 'urlencoded',
 939         format_description => 'model',
 940         maxLength => 40*3, # *3 since it's %xx url enoded
 941         description => "The drive's reported model name, url-encoded, up to 40 bytes long.",
 942         optional => 1,
 943     },
 944 );
 945
 946 my %queues_fmt = (
 947     queues => {
 948         type => 'integer',
 949         description => "Number of queues.",
 950         minimum => 2,
 951         optional => 1
 952     }
 953 );
 954
 955 my %scsiblock_fmt = (
 956     scsiblock => {
 957         type => 'boolean',
 958         description => "whether to use scsi-block for full passthrough of host block device\n\nWARNING: can lead to I/O errors in combination with low memory or high memory fragmentation on host",
 959         optional => 1,
 960         default => 0,
 961     },
 962 );
 963
 964 my $add_throttle_desc = sub {
 965     my ($key, $type, $what, $unit, $longunit, $minimum) = @_;
 966     my $d = {
 967         type => $type,
 968         format_description => $unit,
 969         description => "Maximum $what in $longunit.",
 970         optional => 1,
 971     };
 972     $d->{minimum} = $minimum if defined($minimum);
 973     $drivedesc_base{$key} = $d;
 974 };
 975 # throughput: (leaky bucket)
 976 $add_throttle_desc->('bps',     'integer', 'r/w speed',   'bps',  'bytes per second');
 977 $add_throttle_desc->('bps_rd',  'integer', 'read speed',  'bps',  'bytes per second');
 978 $add_throttle_desc->('bps_wr',  'integer', 'write speed', 'bps',  'bytes per second');
 979 $add_throttle_desc->('mbps',    'number',  'r/w speed',   'mbps', 'megabytes per second');
 980 $add_throttle_desc->('mbps_rd', 'number',  'read speed',  'mbps', 'megabytes per second');
 981 $add_throttle_desc->('mbps_wr', 'number',  'write speed', 'mbps', 'megabytes per second');
 982 $add_throttle_desc->('iops',    'integer', 'r/w I/O',     'iops', 'operations per second');
 983 $add_throttle_desc->('iops_rd', 'integer', 'read I/O',    'iops', 'operations per second');
 984 $add_throttle_desc->('iops_wr', 'integer', 'write I/O',   'iops', 'operations per second');
 985
 986 # pools: (pool of IO before throttling starts taking effect)
 987 $add_throttle_desc->('mbps_max',    'number',  'unthrottled r/w pool',       'mbps', 'megabytes per second');
 988 $add_throttle_desc->('mbps_rd_max', 'number',  'unthrottled read pool',      'mbps', 'megabytes per second');
 989 $add_throttle_desc->('mbps_wr_max', 'number',  'unthrottled write pool',     'mbps', 'megabytes per second');
 990 $add_throttle_desc->('iops_max',    'integer', 'unthrottled r/w I/O pool',   'iops', 'operations per second');
 991 $add_throttle_desc->('iops_rd_max', 'integer', 'unthrottled read I/O pool',  'iops', 'operations per second');
 992 $add_throttle_desc->('iops_wr_max', 'integer', 'unthrottled write I/O pool', 'iops', 'operations per second');
 993
 994 # burst lengths
 995 $add_throttle_desc->('bps_max_length',     'integer', 'length of I/O bursts',       'seconds', 'seconds', 1);
 996 $add_throttle_desc->('bps_rd_max_length',  'integer', 'length of read I/O bursts',  'seconds', 'seconds', 1);
 997 $add_throttle_desc->('bps_wr_max_length',  'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
 998 $add_throttle_desc->('iops_max_length',    'integer', 'length of I/O bursts',       'seconds', 'seconds', 1);
 999 $add_throttle_desc->('iops_rd_max_length', 'integer', 'length of read I/O bursts',  'seconds', 'seconds', 1);
1000 $add_throttle_desc->('iops_wr_max_length', 'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
1001
1002 # legacy support
1003 $drivedesc_base{'bps_rd_length'} = { alias => 'bps_rd_max_length' };
1004 $drivedesc_base{'bps_wr_length'} = { alias => 'bps_wr_max_length' };
1005 $drivedesc_base{'iops_rd_length'} = { alias => 'iops_rd_max_length' };
1006 $drivedesc_base{'iops_wr_length'} = { alias => 'iops_wr_max_length' };
1007
1008 my $ide_fmt = {
1009     %drivedesc_base,
1010     %model_fmt,
1011 };
1012 PVE::JSONSchema::register_format("pve-qm-ide", $ide_fmt);
1013
1014 my $idedesc = {
1015     optional => 1,
1016     type => 'string', format => $ide_fmt,
1017     description => "Use volume as IDE hard disk or CD-ROM (n is 0 to " .($MAX_IDE_DISKS -1) . ").",
1018 };
1019 PVE::JSONSchema::register_standard_option("pve-qm-ide", $idedesc);
1020
1021 my $scsi_fmt = {
1022     %drivedesc_base,
1023     %iothread_fmt,
1024     %queues_fmt,
1025     %scsiblock_fmt,
1026 };
1027 my $scsidesc = {
1028     optional => 1,
1029     type => 'string', format => $scsi_fmt,
1030     description => "Use volume as SCSI hard disk or CD-ROM (n is 0 to " . ($MAX_SCSI_DISKS - 1) . ").",
1031 };
1032 PVE::JSONSchema::register_standard_option("pve-qm-scsi", $scsidesc);
1033
1034 my $sata_fmt = {
1035     %drivedesc_base,
1036 };
1037 my $satadesc = {
1038     optional => 1,
1039     type => 'string', format => $sata_fmt,
1040     description => "Use volume as SATA hard disk or CD-ROM (n is 0 to " . ($MAX_SATA_DISKS - 1). ").",
1041 };
1042 PVE::JSONSchema::register_standard_option("pve-qm-sata", $satadesc);
1043
1044 my $virtio_fmt = {
1045     %drivedesc_base,
1046     %iothread_fmt,
1047 };
1048 my $virtiodesc = {
1049     optional => 1,
1050     type => 'string', format => $virtio_fmt,
1051     description => "Use volume as VIRTIO hard disk (n is 0 to " . ($MAX_VIRTIO_DISKS - 1) . ").",
1052 };
1053 PVE::JSONSchema::register_standard_option("pve-qm-virtio", $virtiodesc);
1054
1055 my $alldrive_fmt = {
1056     %drivedesc_base,
1057     %iothread_fmt,
1058     %model_fmt,
1059     %queues_fmt,
1060     %scsiblock_fmt,
1061 };
1062
1063 my $efidisk_fmt = {
1064     volume => { alias => 'file' },
1065     file => {
1066         type => 'string',
1067         format => 'pve-volume-id-or-qm-path',
1068         default_key => 1,
1069         format_description => 'volume',
1070         description => "The drive's backing volume.",
1071     },
1072     format => get_standard_option('pve-qm-image-format'),
1073     size => {
1074         type => 'string',
1075         format => 'disk-size',
1076         format_description => 'DiskSize',
1077         description => "Disk size. This is purely informational and has no effect.",
1078         optional => 1,
1079     },
1080 };
1081
1082 my $efidisk_desc = {
1083     optional => 1,
1084     type => 'string', format => $efidisk_fmt,
1085     description => "Configure a Disk for storing EFI vars",
1086 };
1087
1088 PVE::JSONSchema::register_standard_option("pve-qm-efidisk", $efidisk_desc);
1089
1090 my $usb_fmt = {
1091     host => {
1092         default_key => 1,
1093         type => 'string', format => 'pve-qm-usb-device',
1094         format_description => 'HOSTUSBDEVICE|spice',
1095         description => <<EODESCR,
1096 The Host USB device or port or the value 'spice'. HOSTUSBDEVICE syntax is:
1097
1098  'bus-port(.port)*' (decimal numbers) or
1099  'vendor_id:product_id' (hexadeciaml numbers) or
1100  'spice'
1101
1102 You can use the 'lsusb -t' command to list existing usb devices.
1103
1104 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1105
1106 The value 'spice' can be used to add a usb redirection devices for spice.
1107 EODESCR
1108     },
1109     usb3 => {
1110         optional => 1,
1111         type => 'boolean',
1112         description => "Specifies whether if given host option is a USB3 device or port (this does currently not work reliably with spice redirection and is then ignored).",
1113         default => 0,
1114     },
1115 };
1116
1117 my $usbdesc = {
1118     optional => 1,
1119     type => 'string', format => $usb_fmt,
1120     description => "Configure an USB device (n is 0 to 4).",
1121 };
1122 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
1123
1124 # NOTE: the match-groups of this regex are used in parse_hostpci
1125 my $PCIRE = qr/([a-f0-9]{2}:[a-f0-9]{2})(?:\.([a-f0-9]))?/;
1126 my $hostpci_fmt = {
1127     host => {
1128         default_key => 1,
1129         type => 'string',
1130         pattern => qr/$PCIRE(;$PCIRE)*/,
1131         format_description => 'HOSTPCIID[;HOSTPCIID2...]',
1132         description => <<EODESCR,
1133 Host PCI device pass through. The PCI ID of a host's PCI device or a list
1134 of PCI virtual functions of the host. HOSTPCIID syntax is:
1135
1136 'bus:dev.func' (hexadecimal numbers)
1137
1138 You can us the 'lspci' command to list existing PCI devices.
1139 EODESCR
1140     },
1141     rombar => {
1142         type => 'boolean',
1143         description =>  "Specify whether or not the device's ROM will be visible in the guest's memory map.",
1144         optional => 1,
1145         default => 1,
1146     },
1147     romfile => {
1148         type => 'string',
1149         pattern => '[^,;]+',
1150         format_description => 'string',
1151         description => "Custom pci device rom filename (must be located in /usr/share/kvm/).",
1152         optional => 1,
1153     },
1154     pcie => {
1155         type => 'boolean',
1156         description =>  "Choose the PCI-express bus (needs the 'q35' machine model).",
1157         optional => 1,
1158         default => 0,
1159     },
1160     'x-vga' => {
1161         type => 'boolean',
1162         description =>  "Enable vfio-vga device support.",
1163         optional => 1,
1164         default => 0,
1165     },
1166 };
1167 PVE::JSONSchema::register_format('pve-qm-hostpci', $hostpci_fmt);
1168
1169 my $hostpcidesc = {
1170         optional => 1,
1171         type => 'string', format => 'pve-qm-hostpci',
1172         description => "Map host PCI devices into guest.",
1173         verbose_description =>  <<EODESCR,
1174 Map host PCI devices into guest.
1175
1176 NOTE: This option allows direct access to host hardware. So it is no longer
1177 possible to migrate such machines - use with special care.
1178
1179 CAUTION: Experimental! User reported problems with this option.
1180 EODESCR
1181 };
1182 PVE::JSONSchema::register_standard_option("pve-qm-hostpci", $hostpcidesc);
1183
1184 my $serialdesc = {
1185         optional => 1,
1186         type => 'string',
1187         pattern => '(/dev/.+|socket)',
1188         description =>  "Create a serial device inside the VM (n is 0 to 3)",
1189         verbose_description =>  <<EODESCR,
1190 Create a serial device inside the VM (n is 0 to 3), and pass through a
1191 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1192 host side (use 'qm terminal' to open a terminal connection).
1193
1194 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines - use with special care.
1195
1196 CAUTION: Experimental! User reported problems with this option.
1197 EODESCR
1198 };
1199
1200 my $paralleldesc= {
1201         optional => 1,
1202         type => 'string',
1203         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
1204         description =>  "Map host parallel devices (n is 0 to 2).",
1205         verbose_description =>  <<EODESCR,
1206 Map host parallel devices (n is 0 to 2).
1207
1208 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1209
1210 CAUTION: Experimental! User reported problems with this option.
1211 EODESCR
1212 };
1213
1214 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
1215     $confdesc->{"parallel$i"} = $paralleldesc;
1216 }
1217
1218 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
1219     $confdesc->{"serial$i"} = $serialdesc;
1220 }
1221
1222 for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
1223     $confdesc->{"hostpci$i"} = $hostpcidesc;
1224 }
1225
1226 for (my $i = 0; $i < $MAX_IDE_DISKS; $i++)  {
1227     $drivename_hash->{"ide$i"} = 1;
1228     $confdesc->{"ide$i"} = $idedesc;
1229 }
1230
1231 for (my $i = 0; $i < $MAX_SATA_DISKS; $i++)  {
1232     $drivename_hash->{"sata$i"} = 1;
1233     $confdesc->{"sata$i"} = $satadesc;
1234 }
1235
1236 for (my $i = 0; $i < $MAX_SCSI_DISKS; $i++)  {
1237     $drivename_hash->{"scsi$i"} = 1;
1238     $confdesc->{"scsi$i"} = $scsidesc ;
1239 }
1240
1241 for (my $i = 0; $i < $MAX_VIRTIO_DISKS; $i++)  {
1242     $drivename_hash->{"virtio$i"} = 1;
1243     $confdesc->{"virtio$i"} = $virtiodesc;
1244 }
1245
1246 $drivename_hash->{efidisk0} = 1;
1247 $confdesc->{efidisk0} = $efidisk_desc;
1248
1249 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
1250     $confdesc->{"usb$i"} = $usbdesc;
1251 }
1252
1253 my $unuseddesc = {
1254     optional => 1,
1255     type => 'string', format => 'pve-volume-id',
1256     description => "Reference to unused volumes. This is used internally, and should not be modified manually.",
1257 };
1258
1259 for (my $i = 0; $i < $MAX_UNUSED_DISKS; $i++)  {
1260     $confdesc->{"unused$i"} = $unuseddesc;
1261 }
1262
1263 my $kvm_api_version = 0;
1264
1265 sub kvm_version {
1266
1267     return $kvm_api_version if $kvm_api_version;
1268
1269     my $fh = IO::File->new("</dev/kvm") ||
1270         return 0;
1271
1272     if (my $v = $fh->ioctl(KVM_GET_API_VERSION(), 0)) {
1273         $kvm_api_version = $v;
1274     }
1275
1276     $fh->close();
1277
1278     return  $kvm_api_version;
1279 }
1280
1281 my $kvm_user_version;
1282
1283 sub kvm_user_version {
1284
1285     return $kvm_user_version if $kvm_user_version;
1286
1287     $kvm_user_version = 'unknown';
1288
1289     my $code = sub {
1290         my $line = shift;
1291         if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1292             $kvm_user_version = $2;
1293         }
1294     };
1295
1296     eval { run_command("kvm -version", outfunc => $code); };
1297     warn $@ if $@;
1298
1299     return $kvm_user_version;
1300
1301 }
1302
1303 my $kernel_has_vhost_net = -c '/dev/vhost-net';
1304
1305 sub valid_drive_names {
1306     # order is important - used to autoselect boot disk
1307     return ((map { "ide$_" } (0 .. ($MAX_IDE_DISKS - 1))),
1308             (map { "scsi$_" } (0 .. ($MAX_SCSI_DISKS - 1))),
1309             (map { "virtio$_" } (0 .. ($MAX_VIRTIO_DISKS - 1))),
1310             (map { "sata$_" } (0 .. ($MAX_SATA_DISKS - 1))),
1311             'efidisk0');
1312 }
1313
1314 sub is_valid_drivename {
1315     my $dev = shift;
1316
1317     return defined($drivename_hash->{$dev});
1318 }
1319
1320 sub option_exists {
1321     my $key = shift;
1322     return defined($confdesc->{$key});
1323 }
1324
1325 sub nic_models {
1326     return $nic_model_list;
1327 }
1328
1329 sub os_list_description {
1330
1331     return {
1332         other => 'Other',
1333         wxp => 'Windows XP',
1334         w2k => 'Windows 2000',
1335         w2k3 =>, 'Windows 2003',
1336         w2k8 => 'Windows 2008',
1337         wvista => 'Windows Vista',
1338         win7 => 'Windows 7',
1339         win8 => 'Windows 8/2012',
1340         win10 => 'Windows 10/2016',
1341         l24 => 'Linux 2.4',
1342         l26 => 'Linux 2.6',
1343     };
1344 }
1345
1346 my $cdrom_path;
1347
1348 sub get_cdrom_path {
1349
1350     return  $cdrom_path if $cdrom_path;
1351
1352     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
1353     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
1354     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
1355 }
1356
1357 sub get_iso_path {
1358     my ($storecfg, $vmid, $cdrom) = @_;
1359
1360     if ($cdrom eq 'cdrom') {
1361         return get_cdrom_path();
1362     } elsif ($cdrom eq 'none') {
1363         return '';
1364     } elsif ($cdrom =~ m|^/|) {
1365         return $cdrom;
1366     } else {
1367         return PVE::Storage::path($storecfg, $cdrom);
1368     }
1369 }
1370
1371 # try to convert old style file names to volume IDs
1372 sub filename_to_volume_id {
1373     my ($vmid, $file, $media) = @_;
1374
1375      if (!($file eq 'none' || $file eq 'cdrom' ||
1376           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1377
1378         return undef if $file =~ m|/|;
1379
1380         if ($media && $media eq 'cdrom') {
1381             $file = "local:iso/$file";
1382         } else {
1383             $file = "local:$vmid/$file";
1384         }
1385     }
1386
1387     return $file;
1388 }
1389
1390 sub verify_media_type {
1391     my ($opt, $vtype, $media) = @_;
1392
1393     return if !$media;
1394
1395     my $etype;
1396     if ($media eq 'disk') {
1397         $etype = 'images';
1398     } elsif ($media eq 'cdrom') {
1399         $etype = 'iso';
1400     } else {
1401         die "internal error";
1402     }
1403
1404     return if ($vtype eq $etype);
1405
1406     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1407 }
1408
1409 sub cleanup_drive_path {
1410     my ($opt, $storecfg, $drive) = @_;
1411
1412     # try to convert filesystem paths to volume IDs
1413
1414     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1415         ($drive->{file} !~ m|^/dev/.+|) &&
1416         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1417         ($drive->{file} !~ m/^\d+$/)) {
1418         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1419         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"}) if !$vtype;
1420         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1421         verify_media_type($opt, $vtype, $drive->{media});
1422         $drive->{file} = $volid;
1423     }
1424
1425     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1426 }
1427
1428 sub parse_hotplug_features {
1429     my ($data) = @_;
1430
1431     my $res = {};
1432
1433     return $res if $data eq '0';
1434
1435     $data = $confdesc->{hotplug}->{default} if $data eq '1';
1436
1437     foreach my $feature (PVE::Tools::split_list($data)) {
1438         if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
1439             $res->{$1} = 1;
1440         } else {
1441             die "invalid hotplug feature '$feature'\n";
1442         }
1443     }
1444     return $res;
1445 }
1446
1447 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1448 sub pve_verify_hotplug_features {
1449     my ($value, $noerr) = @_;
1450
1451     return $value if parse_hotplug_features($value);
1452
1453     return undef if $noerr;
1454
1455     die "unable to parse hotplug option\n";
1456 }
1457
1458 # ideX = [volume=]volume-id[,media=d][,cyls=c,heads=h,secs=s[,trans=t]]
1459 #        [,snapshot=on|off][,cache=on|off][,format=f][,backup=yes|no]
1460 #        [,rerror=ignore|report|stop][,werror=enospc|ignore|report|stop]
1461 #        [,aio=native|threads][,discard=ignore|on][,detect_zeroes=on|off]
1462 #        [,iothread=on][,serial=serial][,model=model]
1463
1464 sub parse_drive {
1465     my ($key, $data) = @_;
1466
1467     my ($interface, $index);
1468
1469     if ($key =~ m/^([^\d]+)(\d+)$/) {
1470         $interface = $1;
1471         $index = $2;
1472     } else {
1473         return undef;
1474     }
1475
1476     my $desc = $key =~ /^unused\d+$/ ? $alldrive_fmt
1477                                      : $confdesc->{$key}->{format};
1478     if (!$desc) {
1479         warn "invalid drive key: $key\n";
1480         return undef;
1481     }
1482     my $res = eval { PVE::JSONSchema::parse_property_string($desc, $data) };
1483     return undef if !$res;
1484     $res->{interface} = $interface;
1485     $res->{index} = $index;
1486
1487     my $error = 0;
1488     foreach my $opt (qw(bps bps_rd bps_wr)) {
1489         if (my $bps = defined(delete $res->{$opt})) {
1490             if (defined($res->{"m$opt"})) {
1491                 warn "both $opt and m$opt specified\n";
1492                 ++$error;
1493                 next;
1494             }
1495             $res->{"m$opt"} = sprintf("%.3f", $bps / (1024*1024.0));
1496         }
1497     }
1498
1499     # can't use the schema's 'requires' because of the mbps* => bps* "transforming aliases"
1500     for my $requirement (
1501         [mbps_max => 'mbps'],
1502         [mbps_rd_max => 'mbps_rd'],
1503         [mbps_wr_max => 'mbps_wr'],
1504         [miops_max => 'miops'],
1505         [miops_rd_max => 'miops_rd'],
1506         [miops_wr_max => 'miops_wr'],
1507         [bps_max_length => 'mbps_max'],
1508         [bps_rd_max_length => 'mbps_rd_max'],
1509         [bps_wr_max_length => 'mbps_wr_max'],
1510         [iops_max_length => 'iops_max'],
1511         [iops_rd_max_length => 'iops_rd_max'],
1512         [iops_wr_max_length => 'iops_wr_max']) {
1513         my ($option, $requires) = @$requirement;
1514         if ($res->{$option} && !$res->{$requires}) {
1515             warn "$option requires $requires\n";
1516             ++$error;
1517         }
1518     }
1519
1520     return undef if $error;
1521
1522     return undef if $res->{mbps_rd} && $res->{mbps};
1523     return undef if $res->{mbps_wr} && $res->{mbps};
1524     return undef if $res->{iops_rd} && $res->{iops};
1525     return undef if $res->{iops_wr} && $res->{iops};
1526
1527     if ($res->{media} && ($res->{media} eq 'cdrom')) {
1528         return undef if $res->{snapshot} || $res->{trans} || $res->{format};
1529         return undef if $res->{heads} || $res->{secs} || $res->{cyls};
1530         return undef if $res->{interface} eq 'virtio';
1531     }
1532
1533     if (my $size = $res->{size}) {
1534         return undef if !defined($res->{size} = PVE::JSONSchema::parse_size($size));
1535     }
1536
1537     return $res;
1538 }
1539
1540 sub print_drive {
1541     my ($vmid, $drive) = @_;
1542     my $data = { %$drive };
1543     delete $data->{$_} for qw(index interface);
1544     return PVE::JSONSchema::print_property_string($data, $alldrive_fmt);
1545 }
1546
1547 sub scsi_inquiry {
1548     my($fh, $noerr) = @_;
1549
1550     my $SG_IO = 0x2285;
1551     my $SG_GET_VERSION_NUM = 0x2282;
1552
1553     my $versionbuf = "\x00" x 8;
1554     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1555     if (!$ret) {
1556         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1557         return undef;
1558     }
1559     my $version = unpack("I", $versionbuf);
1560     if ($version < 30000) {
1561         die "scsi generic interface too old\n"  if !$noerr;
1562         return undef;
1563     }
1564
1565     my $buf = "\x00" x 36;
1566     my $sensebuf = "\x00" x 8;
1567     my $cmd = pack("C x3 C x1", 0x12, 36);
1568
1569     # see /usr/include/scsi/sg.h
1570     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1571
1572     my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1573                       length($sensebuf), 0, length($buf), $buf,
1574                       $cmd, $sensebuf, 6000);
1575
1576     $ret = ioctl($fh, $SG_IO, $packet);
1577     if (!$ret) {
1578         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1579         return undef;
1580     }
1581
1582     my @res = unpack($sg_io_hdr_t, $packet);
1583     if ($res[17] || $res[18]) {
1584         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1585         return undef;
1586     }
1587
1588     my $res = {};
1589     (my $byte0, my $byte1, $res->{vendor},
1590      $res->{product}, $res->{revision}) = unpack("C C x6 A8 A16 A4", $buf);
1591
1592     $res->{removable} = $byte1 & 128 ? 1 : 0;
1593     $res->{type} = $byte0 & 31;
1594
1595     return $res;
1596 }
1597
1598 sub path_is_scsi {
1599     my ($path) = @_;
1600
1601     my $fh = IO::File->new("+<$path") || return undef;
1602     my $res = scsi_inquiry($fh, 1);
1603     close($fh);
1604
1605     return $res;
1606 }
1607
1608 sub machine_type_is_q35 {
1609     my ($conf) = @_;
1610
1611     return $conf->{machine} && ($conf->{machine} =~ m/q35/) ? 1 : 0;
1612 }
1613
1614 sub print_tabletdevice_full {
1615     my ($conf) = @_;
1616
1617     my $q35 = machine_type_is_q35($conf);
1618
1619     # we use uhci for old VMs because tablet driver was buggy in older qemu
1620     my $usbbus = $q35 ? "ehci" : "uhci";
1621
1622     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1623 }
1624
1625 sub print_drivedevice_full {
1626     my ($storecfg, $conf, $vmid, $drive, $bridges) = @_;
1627
1628     my $device = '';
1629     my $maxdev = 0;
1630
1631     if ($drive->{interface} eq 'virtio') {
1632         my $pciaddr = print_pci_addr("$drive->{interface}$drive->{index}", $bridges);
1633         $device = "virtio-blk-pci,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}$pciaddr";
1634         $device .= ",iothread=iothread-$drive->{interface}$drive->{index}" if $drive->{iothread};
1635     } elsif ($drive->{interface} eq 'scsi') {
1636
1637         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1638         my $unit = $drive->{index} % $maxdev;
1639         my $devicetype = 'hd';
1640         my $path = '';
1641         if (drive_is_cdrom($drive)) {
1642             $devicetype = 'cd';
1643         } else {
1644             if ($drive->{file} =~ m|^/|) {
1645                 $path = $drive->{file};
1646                 if (my $info = path_is_scsi($path)) {
1647                     if ($info->{type} == 0 && $drive->{scsiblock}) {
1648                         $devicetype = 'block';
1649                     } elsif ($info->{type} == 1) { # tape
1650                         $devicetype = 'generic';
1651                     }
1652                 }
1653             } else {
1654                  $path = PVE::Storage::path($storecfg, $drive->{file});
1655             }
1656
1657             if($path =~ m/^iscsi\:\/\//){
1658                 $devicetype = 'generic';
1659             }
1660         }
1661
1662         if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)){
1663             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1664         } else {
1665             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0,lun=$drive->{index},drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1666         }
1667
1668     } elsif ($drive->{interface} eq 'ide'){
1669         $maxdev = 2;
1670         my $controller = int($drive->{index} / $maxdev);
1671         my $unit = $drive->{index} % $maxdev;
1672         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1673
1674         $device = "ide-$devicetype,bus=ide.$controller,unit=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1675         if ($devicetype eq 'hd' && (my $model = $drive->{model})) {
1676             $model = URI::Escape::uri_unescape($model);
1677             $device .= ",model=$model";
1678         }
1679     } elsif ($drive->{interface} eq 'sata'){
1680         my $controller = int($drive->{index} / $MAX_SATA_DISKS);
1681         my $unit = $drive->{index} % $MAX_SATA_DISKS;
1682         $device = "ide-drive,bus=ahci$controller.$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1683     } elsif ($drive->{interface} eq 'usb') {
1684         die "implement me";
1685         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1686     } else {
1687         die "unsupported interface type";
1688     }
1689
1690     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1691
1692     if (my $serial = $drive->{serial}) {
1693         $serial = URI::Escape::uri_unescape($serial);
1694         $device .= ",serial=$serial";
1695     }
1696
1697
1698     return $device;
1699 }
1700
1701 sub get_initiator_name {
1702     my $initiator;
1703
1704     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return undef;
1705     while (defined(my $line = <$fh>)) {
1706         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1707         $initiator = $1;
1708         last;
1709     }
1710     $fh->close();
1711
1712     return $initiator;
1713 }
1714
1715 sub print_drive_full {
1716     my ($storecfg, $vmid, $drive) = @_;
1717
1718     my $path;
1719     my $volid = $drive->{file};
1720     my $format;
1721
1722     if (drive_is_cdrom($drive)) {
1723         $path = get_iso_path($storecfg, $vmid, $volid);
1724     } else {
1725         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1726         if ($storeid) {
1727             $path = PVE::Storage::path($storecfg, $volid);
1728             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
1729             $format = qemu_img_format($scfg, $volname);
1730         } else {
1731             $path = $volid;
1732             $format = "raw";
1733         }
1734    }
1735
1736     my $opts = '';
1737     my @qemu_drive_options = qw(heads secs cyls trans media format cache rerror werror aio discard);
1738     foreach my $o (@qemu_drive_options) {
1739         $opts .= ",$o=$drive->{$o}" if defined($drive->{$o});
1740     }
1741
1742     # snapshot only accepts on|off
1743     if (defined($drive->{snapshot})) {
1744         my $v = $drive->{snapshot} ? 'on' : 'off';
1745         $opts .= ",snapshot=$v";
1746     }
1747
1748     foreach my $type (['', '-total'], [_rd => '-read'], [_wr => '-write']) {
1749         my ($dir, $qmpname) = @$type;
1750         if (my $v = $drive->{"mbps$dir"}) {
1751             $opts .= ",throttling.bps$qmpname=".int($v*1024*1024);
1752         }
1753         if (my $v = $drive->{"mbps${dir}_max"}) {
1754             $opts .= ",throttling.bps$qmpname-max=".int($v*1024*1024);
1755         }
1756         if (my $v = $drive->{"bps${dir}_max_length"}) {
1757             $opts .= ",throttling.bps$qmpname-max-length=$v";
1758         }
1759         if (my $v = $drive->{"iops${dir}"}) {
1760             $opts .= ",throttling.iops$qmpname=$v";
1761         }
1762         if (my $v = $drive->{"iops${dir}_max"}) {
1763             $opts .= ",throttling.iops$qmpname-max=$v";
1764         }
1765         if (my $v = $drive->{"iops${dir}_max_length"}) {
1766             $opts .= ",throttling.iops$qmpname-max-length=$v";
1767         }
1768     }
1769
1770     $opts .= ",format=$format" if $format && !$drive->{format};
1771
1772     my $cache_direct = 0;
1773
1774     if (my $cache = $drive->{cache}) {
1775         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1776     } elsif (!drive_is_cdrom($drive)) {
1777         $opts .= ",cache=none";
1778         $cache_direct = 1;
1779     }
1780
1781     # aio native works only with O_DIRECT
1782     if (!$drive->{aio}) {
1783         if($cache_direct) {
1784             $opts .= ",aio=native";
1785         } else {
1786             $opts .= ",aio=threads";
1787         }
1788     }
1789
1790     if (!drive_is_cdrom($drive)) {
1791         my $detectzeroes;
1792         if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
1793             $detectzeroes = 'off';
1794         } elsif ($drive->{discard}) {
1795             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
1796         } else {
1797             # This used to be our default with discard not being specified:
1798             $detectzeroes = 'on';
1799         }
1800         $opts .= ",detect-zeroes=$detectzeroes" if $detectzeroes;
1801     }
1802
1803     my $pathinfo = $path ? "file=$path," : '';
1804
1805     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1806 }
1807
1808 sub print_netdevice_full {
1809     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files) = @_;
1810
1811     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
1812
1813     my $device = $net->{model};
1814     if ($net->{model} eq 'virtio') {
1815          $device = 'virtio-net-pci';
1816      };
1817
1818     my $pciaddr = print_pci_addr("$netid", $bridges);
1819     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1820     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
1821         #Consider we have N queues, the number of vectors needed is 2*N + 2 (plus one config interrupt and control vq)
1822         my $vectors = $net->{queues} * 2 + 2;
1823         $tmpstr .= ",vectors=$vectors,mq=on";
1824     }
1825     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
1826
1827     if ($use_old_bios_files) {
1828         my $romfile;
1829         if ($device eq 'virtio-net-pci') {
1830             $romfile = 'pxe-virtio.rom';
1831         } elsif ($device eq 'e1000') {
1832             $romfile = 'pxe-e1000.rom';
1833         } elsif ($device eq 'ne2k') {
1834             $romfile = 'pxe-ne2k_pci.rom';
1835         } elsif ($device eq 'pcnet') {
1836             $romfile = 'pxe-pcnet.rom';
1837         } elsif ($device eq 'rtl8139') {
1838             $romfile = 'pxe-rtl8139.rom';
1839         }
1840         $tmpstr .= ",romfile=$romfile" if $romfile;
1841     }
1842
1843     return $tmpstr;
1844 }
1845
1846 sub print_netdev_full {
1847     my ($vmid, $conf, $net, $netid, $hotplug) = @_;
1848
1849     my $i = '';
1850     if ($netid =~ m/^net(\d+)$/) {
1851         $i = int($1);
1852     }
1853
1854     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
1855
1856     my $ifname = "tap${vmid}i$i";
1857
1858     # kvm uses TUNSETIFF ioctl, and that limits ifname length
1859     die "interface name '$ifname' is too long (max 15 character)\n"
1860         if length($ifname) >= 16;
1861
1862     my $vhostparam = '';
1863     $vhostparam = ',vhost=on' if $kernel_has_vhost_net && $net->{model} eq 'virtio';
1864
1865     my $vmname = $conf->{name} || "vm$vmid";
1866
1867     my $netdev = "";
1868     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
1869
1870     if ($net->{bridge}) {
1871         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script,downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1872     } else {
1873         $netdev = "type=user,id=$netid,hostname=$vmname";
1874     }
1875
1876     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
1877
1878     return $netdev;
1879 }
1880
1881
1882 sub print_cpu_device {
1883     my ($conf, $id) = @_;
1884
1885     my $kvm = $conf->{kvm} // 1;
1886     my $cpu = $kvm ? "kvm64" : "qemu64";
1887     if (my $cputype = $conf->{cpu}) {
1888         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
1889             or die "Cannot parse cpu description: $cputype\n";
1890         $cpu = $cpuconf->{cputype};
1891     }
1892
1893     my $cores = $conf->{cores} || 1;
1894
1895     my $current_core = ($id - 1) % $cores;
1896     my $current_socket = int(($id - 1 - $current_core)/$cores);
1897
1898     return "$cpu-x86_64-cpu,id=cpu$id,socket-id=$current_socket,core-id=$current_core,thread-id=0";
1899 }
1900
1901 sub drive_is_cloudinit {
1902     my ($drive) = @_;
1903     return $drive->{file} =~ m@[:/]vm-\d+-cloudinit(?:\.$QEMU_FORMAT_RE)?$@;
1904 }
1905
1906 sub drive_is_cdrom {
1907     my ($drive, $exclude_cloudinit) = @_;
1908
1909     return 0 if $exclude_cloudinit && drive_is_cloudinit($drive);
1910
1911     return $drive && $drive->{media} && ($drive->{media} eq 'cdrom');
1912
1913 }
1914
1915 sub parse_number_sets {
1916     my ($set) = @_;
1917     my $res = [];
1918     foreach my $part (split(/;/, $set)) {
1919         if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
1920             die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
1921             push @$res, [ $1, $2 ];
1922         } else {
1923             die "invalid range: $part\n";
1924         }
1925     }
1926     return $res;
1927 }
1928
1929 sub parse_numa {
1930     my ($data) = @_;
1931
1932     my $res = PVE::JSONSchema::parse_property_string($numa_fmt, $data);
1933     $res->{cpus} = parse_number_sets($res->{cpus}) if defined($res->{cpus});
1934     $res->{hostnodes} = parse_number_sets($res->{hostnodes}) if defined($res->{hostnodes});
1935     return $res;
1936 }
1937
1938 sub parse_hostpci {
1939     my ($value) = @_;
1940
1941     return undef if !$value;
1942
1943     my $res = PVE::JSONSchema::parse_property_string($hostpci_fmt, $value);
1944
1945     my @idlist = split(/;/, $res->{host});
1946     delete $res->{host};
1947     foreach my $id (@idlist) {
1948         if ($id =~ /^$PCIRE$/) {
1949             if (defined($2)) {
1950                 push @{$res->{pciid}}, { id => $1, function => $2 };
1951             } else {
1952                 my $pcidevices = lspci($1);
1953                 $res->{pciid} = $pcidevices->{$1};
1954             }
1955         } else {
1956             # should have been caught by parse_property_string already
1957             die "failed to parse PCI id: $id\n";
1958         }
1959     }
1960     return $res;
1961 }
1962
1963 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
1964 sub parse_net {
1965     my ($data) = @_;
1966
1967     my $res = eval { PVE::JSONSchema::parse_property_string($net_fmt, $data) };
1968     if ($@) {
1969         warn $@;
1970         return undef;
1971     }
1972     if (!defined($res->{macaddr})) {
1973         my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
1974         $res->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix});
1975     }
1976     return $res;
1977 }
1978
1979 # ipconfigX ip=cidr,gw=ip,ip6=cidr,gw6=ip
1980 sub parse_ipconfig {
1981     my ($data) = @_;
1982
1983     my $res = eval { PVE::JSONSchema::parse_property_string($ipconfig_fmt, $data) };
1984     if ($@) {
1985         warn $@;
1986         return undef;
1987     }
1988
1989     if ($res->{gw} && !$res->{ip}) {
1990         warn 'gateway specified without specifying an IP address';
1991         return undef;
1992     }
1993     if ($res->{gw6} && !$res->{ip6}) {
1994         warn 'IPv6 gateway specified without specifying an IPv6 address';
1995         return undef;
1996     }
1997     if ($res->{gw} && $res->{ip} eq 'dhcp') {
1998         warn 'gateway specified together with DHCP';
1999         return undef;
2000     }
2001     if ($res->{gw6} && $res->{ip6} !~ /^$IPV6RE/) {
2002         # gw6 + auto/dhcp
2003         warn "IPv6 gateway specified together with $res->{ip6} address";
2004         return undef;
2005     }
2006
2007     if (!$res->{ip} && !$res->{ip6}) {
2008         return { ip => 'dhcp', ip6 => 'dhcp' };
2009     }
2010
2011     return $res;
2012 }
2013
2014 sub print_net {
2015     my $net = shift;
2016
2017     return PVE::JSONSchema::print_property_string($net, $net_fmt);
2018 }
2019
2020 sub add_random_macs {
2021     my ($settings) = @_;
2022
2023     foreach my $opt (keys %$settings) {
2024         next if $opt !~ m/^net(\d+)$/;
2025         my $net = parse_net($settings->{$opt});
2026         next if !$net;
2027         $settings->{$opt} = print_net($net);
2028     }
2029 }
2030
2031 sub vm_is_volid_owner {
2032     my ($storecfg, $vmid, $volid) = @_;
2033
2034     if ($volid !~  m|^/|) {
2035         my ($path, $owner);
2036         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
2037         if ($owner && ($owner == $vmid)) {
2038             return 1;
2039         }
2040     }
2041
2042     return undef;
2043 }
2044
2045 sub split_flagged_list {
2046     my $text = shift || '';
2047     $text =~ s/[,;]/ /g;
2048     $text =~ s/^\s+//;
2049     return { map { /^(!?)(.*)$/ && ($2, $1) } ($text =~ /\S+/g) };
2050 }
2051
2052 sub join_flagged_list {
2053     my ($how, $lst) = @_;
2054     join $how, map { $lst->{$_} . $_ } keys %$lst;
2055 }
2056
2057 sub vmconfig_delete_pending_option {
2058     my ($conf, $key, $force) = @_;
2059
2060     delete $conf->{pending}->{$key};
2061     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
2062     $pending_delete_hash->{$key} = $force ? '!' : '';
2063     $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
2064 }
2065
2066 sub vmconfig_undelete_pending_option {
2067     my ($conf, $key) = @_;
2068
2069     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
2070     delete $pending_delete_hash->{$key};
2071
2072     if (%$pending_delete_hash) {
2073         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
2074     } else {
2075         delete $conf->{pending}->{delete};
2076     }
2077 }
2078
2079 sub vmconfig_register_unused_drive {
2080     my ($storecfg, $vmid, $conf, $drive) = @_;
2081
2082     if (drive_is_cloudinit($drive)) {
2083         eval { PVE::Storage::vdisk_free($storecfg, $drive->{file}) };
2084         warn $@ if $@;
2085     } elsif (!drive_is_cdrom($drive)) {
2086         my $volid = $drive->{file};
2087         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
2088             PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
2089         }
2090     }
2091 }
2092
2093 sub vmconfig_cleanup_pending {
2094     my ($conf) = @_;
2095
2096     # remove pending changes when nothing changed
2097     my $changes;
2098     foreach my $opt (keys %{$conf->{pending}}) {
2099         if (defined($conf->{$opt}) && ($conf->{pending}->{$opt} eq  $conf->{$opt})) {
2100             $changes = 1;
2101             delete $conf->{pending}->{$opt};
2102         }
2103     }
2104
2105     my $current_delete_hash = split_flagged_list($conf->{pending}->{delete});
2106     my $pending_delete_hash = {};
2107     while (my ($opt, $force) = each %$current_delete_hash) {
2108         if (defined($conf->{$opt})) {
2109             $pending_delete_hash->{$opt} = $force;
2110         } else {
2111             $changes = 1;
2112         }
2113     }
2114
2115     if (%$pending_delete_hash) {
2116         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
2117     } else {
2118         delete $conf->{pending}->{delete};
2119     }
2120
2121     return $changes;
2122 }
2123
2124 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str]
2125 my $smbios1_fmt = {
2126     uuid => {
2127         type => 'string',
2128         pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
2129         format_description => 'UUID',
2130         description => "Set SMBIOS1 UUID.",
2131         optional => 1,
2132     },
2133     version => {
2134         type => 'string',
2135         pattern => '\S+',
2136         format_description => 'string',
2137         description => "Set SMBIOS1 version.",
2138         optional => 1,
2139     },
2140     serial => {
2141         type => 'string',
2142         pattern => '\S+',
2143         format_description => 'string',
2144         description => "Set SMBIOS1 serial number.",
2145         optional => 1,
2146     },
2147     manufacturer => {
2148         type => 'string',
2149         pattern => '\S+',
2150         format_description => 'string',
2151         description => "Set SMBIOS1 manufacturer.",
2152         optional => 1,
2153     },
2154     product => {
2155         type => 'string',
2156         pattern => '\S+',
2157         format_description => 'string',
2158         description => "Set SMBIOS1 product ID.",
2159         optional => 1,
2160     },
2161     sku => {
2162         type => 'string',
2163         pattern => '\S+',
2164         format_description => 'string',
2165         description => "Set SMBIOS1 SKU string.",
2166         optional => 1,
2167     },
2168     family => {
2169         type => 'string',
2170         pattern => '\S+',
2171         format_description => 'string',
2172         description => "Set SMBIOS1 family string.",
2173         optional => 1,
2174     },
2175 };
2176
2177 sub parse_smbios1 {
2178     my ($data) = @_;
2179
2180     my $res = eval { PVE::JSONSchema::parse_property_string($smbios1_fmt, $data) };
2181     warn $@ if $@;
2182     return $res;
2183 }
2184
2185 sub print_smbios1 {
2186     my ($smbios1) = @_;
2187     return PVE::JSONSchema::print_property_string($smbios1, $smbios1_fmt);
2188 }
2189
2190 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_fmt);
2191
2192 PVE::JSONSchema::register_format('pve-qm-bootdisk', \&verify_bootdisk);
2193 sub verify_bootdisk {
2194     my ($value, $noerr) = @_;
2195
2196     return $value if is_valid_drivename($value);
2197
2198     return undef if $noerr;
2199
2200     die "invalid boot disk '$value'\n";
2201 }
2202
2203 sub parse_watchdog {
2204     my ($value) = @_;
2205
2206     return undef if !$value;
2207
2208     my $res = eval { PVE::JSONSchema::parse_property_string($watchdog_fmt, $value) };
2209     warn $@ if $@;
2210     return $res;
2211 }
2212
2213 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
2214 sub verify_usb_device {
2215     my ($value, $noerr) = @_;
2216
2217     return $value if parse_usb_device($value);
2218
2219     return undef if $noerr;
2220
2221     die "unable to parse usb device\n";
2222 }
2223
2224 # add JSON properties for create and set function
2225 sub json_config_properties {
2226     my $prop = shift;
2227
2228     foreach my $opt (keys %$confdesc) {
2229         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate';
2230         $prop->{$opt} = $confdesc->{$opt};
2231     }
2232
2233     return $prop;
2234 }
2235
2236 # return copy of $confdesc_cloudinit to generate documentation
2237 sub cloudinit_config_properties {
2238
2239     return dclone($confdesc_cloudinit);
2240 }
2241
2242 sub check_type {
2243     my ($key, $value) = @_;
2244
2245     die "unknown setting '$key'\n" if !$confdesc->{$key};
2246
2247     my $type = $confdesc->{$key}->{type};
2248
2249     if (!defined($value)) {
2250         die "got undefined value\n";
2251     }
2252
2253     if ($value =~ m/[\n\r]/) {
2254         die "property contains a line feed\n";
2255     }
2256
2257     if ($type eq 'boolean') {
2258         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
2259         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
2260         die "type check ('boolean') failed - got '$value'\n";
2261     } elsif ($type eq 'integer') {
2262         return int($1) if $value =~ m/^(\d+)$/;
2263         die "type check ('integer') failed - got '$value'\n";
2264     } elsif ($type eq 'number') {
2265         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
2266         die "type check ('number') failed - got '$value'\n";
2267     } elsif ($type eq 'string') {
2268         if (my $fmt = $confdesc->{$key}->{format}) {
2269             PVE::JSONSchema::check_format($fmt, $value);
2270             return $value;
2271         }
2272         $value =~ s/^\"(.*)\"$/$1/;
2273         return $value;
2274     } else {
2275         die "internal error"
2276     }
2277 }
2278
2279 sub check_iommu_support{
2280     #fixme : need to check IOMMU support
2281     #http://www.linux-kvm.org/page/How_to_assign_devices_with_VT-d_in_KVM
2282
2283     my $iommu=1;
2284     return $iommu;
2285
2286 }
2287
2288 sub touch_config {
2289     my ($vmid) = @_;
2290
2291     my $conf = PVE::QemuConfig->config_file($vmid);
2292     utime undef, undef, $conf;
2293 }
2294
2295 sub destroy_vm {
2296     my ($storecfg, $vmid, $keep_empty_config, $skiplock) = @_;
2297
2298     my $conffile = PVE::QemuConfig->config_file($vmid);
2299
2300     my $conf = PVE::QemuConfig->load_config($vmid);
2301
2302     PVE::QemuConfig->check_lock($conf) if !$skiplock;
2303
2304     if ($conf->{template}) {
2305         # check if any base image is still used by a linked clone
2306         foreach_drive($conf, sub {
2307                 my ($ds, $drive) = @_;
2308
2309                 return if drive_is_cdrom($drive);
2310
2311                 my $volid = $drive->{file};
2312
2313                 return if !$volid || $volid =~ m|^/|;
2314
2315                 die "base volume '$volid' is still in use by linked cloned\n"
2316                     if PVE::Storage::volume_is_base_and_used($storecfg, $volid);
2317
2318         });
2319     }
2320
2321     # only remove disks owned by this VM
2322     foreach_drive($conf, sub {
2323         my ($ds, $drive) = @_;
2324
2325         return if drive_is_cdrom($drive, 1);
2326
2327         my $volid = $drive->{file};
2328
2329         return if !$volid || $volid =~ m|^/|;
2330
2331         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
2332         return if !$path || !$owner || ($owner != $vmid);
2333
2334         eval {
2335             PVE::Storage::vdisk_free($storecfg, $volid);
2336         };
2337         warn "Could not remove disk '$volid', check manually: $@" if $@;
2338
2339     });
2340
2341     if ($keep_empty_config) {
2342         PVE::Tools::file_set_contents($conffile, "memory: 128\n");
2343     } else {
2344         unlink $conffile;
2345     }
2346
2347     # also remove unused disk
2348     eval {
2349         my $dl = PVE::Storage::vdisk_list($storecfg, undef, $vmid);
2350
2351         eval {
2352             PVE::Storage::foreach_volid($dl, sub {
2353                 my ($volid, $sid, $volname, $d) = @_;
2354                 PVE::Storage::vdisk_free($storecfg, $volid);
2355             });
2356         };
2357         warn $@ if $@;
2358
2359     };
2360     warn $@ if $@;
2361 }
2362
2363 sub parse_vm_config {
2364     my ($filename, $raw) = @_;
2365
2366     return undef if !defined($raw);
2367
2368     my $res = {
2369         digest => Digest::SHA::sha1_hex($raw),
2370         snapshots => {},
2371         pending => {},
2372     };
2373
2374     $filename =~ m|/qemu-server/(\d+)\.conf$|
2375         || die "got strange filename '$filename'";
2376
2377     my $vmid = $1;
2378
2379     my $conf = $res;
2380     my $descr;
2381     my $section = '';
2382
2383     my @lines = split(/\n/, $raw);
2384     foreach my $line (@lines) {
2385         next if $line =~ m/^\s*$/;
2386
2387         if ($line =~ m/^\[PENDING\]\s*$/i) {
2388             $section = 'pending';
2389             if (defined($descr)) {
2390                 $descr =~ s/\s+$//;
2391                 $conf->{description} = $descr;
2392             }
2393             $descr = undef;
2394             $conf = $res->{$section} = {};
2395             next;
2396
2397         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2398             $section = $1;
2399             if (defined($descr)) {
2400                 $descr =~ s/\s+$//;
2401                 $conf->{description} = $descr;
2402             }
2403             $descr = undef;
2404             $conf = $res->{snapshots}->{$section} = {};
2405             next;
2406         }
2407
2408         if ($line =~ m/^\#(.*)\s*$/) {
2409             $descr = '' if !defined($descr);
2410             $descr .= PVE::Tools::decode_text($1) . "\n";
2411             next;
2412         }
2413
2414         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2415             $descr = '' if !defined($descr);
2416             $descr .= PVE::Tools::decode_text($2);
2417         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2418             $conf->{snapstate} = $1;
2419         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2420             my $key = $1;
2421             my $value = $2;
2422             $conf->{$key} = $value;
2423         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2424             my $value = $1;
2425             if ($section eq 'pending') {
2426                 $conf->{delete} = $value; # we parse this later
2427             } else {
2428                 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
2429             }
2430         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(.+?)\s*$/) {
2431             my $key = $1;
2432             my $value = $2;
2433             eval { $value = check_type($key, $value); };
2434             if ($@) {
2435                 warn "vm $vmid - unable to parse value of '$key' - $@";
2436             } else {
2437                 $key = 'ide2' if $key eq 'cdrom';
2438                 my $fmt = $confdesc->{$key}->{format};
2439                 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2440                     my $v = parse_drive($key, $value);
2441                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2442                         $v->{file} = $volid;
2443                         $value = print_drive($vmid, $v);
2444                     } else {
2445                         warn "vm $vmid - unable to parse value of '$key'\n";
2446                         next;
2447                     }
2448                 }
2449
2450                 $conf->{$key} = $value;
2451             }
2452         }
2453     }
2454
2455     if (defined($descr)) {
2456         $descr =~ s/\s+$//;
2457         $conf->{description} = $descr;
2458     }
2459     delete $res->{snapstate}; # just to be sure
2460
2461     return $res;
2462 }
2463
2464 sub write_vm_config {
2465     my ($filename, $conf) = @_;
2466
2467     delete $conf->{snapstate}; # just to be sure
2468
2469     if ($conf->{cdrom}) {
2470         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2471         $conf->{ide2} = $conf->{cdrom};
2472         delete $conf->{cdrom};
2473     }
2474
2475     # we do not use 'smp' any longer
2476     if ($conf->{sockets}) {
2477         delete $conf->{smp};
2478     } elsif ($conf->{smp}) {
2479         $conf->{sockets} = $conf->{smp};
2480         delete $conf->{cores};
2481         delete $conf->{smp};
2482     }
2483
2484     my $used_volids = {};
2485
2486     my $cleanup_config = sub {
2487         my ($cref, $pending, $snapname) = @_;
2488
2489         foreach my $key (keys %$cref) {
2490             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2491                 $key eq 'snapstate' || $key eq 'pending';
2492             my $value = $cref->{$key};
2493             if ($key eq 'delete') {
2494                 die "propertry 'delete' is only allowed in [PENDING]\n"
2495                     if !$pending;
2496                 # fixme: check syntax?
2497                 next;
2498             }
2499             eval { $value = check_type($key, $value); };
2500             die "unable to parse value of '$key' - $@" if $@;
2501
2502             $cref->{$key} = $value;
2503
2504             if (!$snapname && is_valid_drivename($key)) {
2505                 my $drive = parse_drive($key, $value);
2506                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2507             }
2508         }
2509     };
2510
2511     &$cleanup_config($conf);
2512
2513     &$cleanup_config($conf->{pending}, 1);
2514
2515     foreach my $snapname (keys %{$conf->{snapshots}}) {
2516         die "internal error" if $snapname eq 'pending';
2517         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2518     }
2519
2520     # remove 'unusedX' settings if we re-add a volume
2521     foreach my $key (keys %$conf) {
2522         my $value = $conf->{$key};
2523         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2524             delete $conf->{$key};
2525         }
2526     }
2527
2528     my $generate_raw_config = sub {
2529         my ($conf, $pending) = @_;
2530
2531         my $raw = '';
2532
2533         # add description as comment to top of file
2534         if (defined(my $descr = $conf->{description})) {
2535             if ($descr) {
2536                 foreach my $cl (split(/\n/, $descr)) {
2537                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2538                 }
2539             } else {
2540                 $raw .= "#\n" if $pending;
2541             }
2542         }
2543
2544         foreach my $key (sort keys %$conf) {
2545             next if $key eq 'digest' || $key eq 'description' || $key eq 'pending' || $key eq 'snapshots';
2546             $raw .= "$key: $conf->{$key}\n";
2547         }
2548         return $raw;
2549     };
2550
2551     my $raw = &$generate_raw_config($conf);
2552
2553     if (scalar(keys %{$conf->{pending}})){
2554         $raw .= "\n[PENDING]\n";
2555         $raw .= &$generate_raw_config($conf->{pending}, 1);
2556     }
2557
2558     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2559         $raw .= "\n[$snapname]\n";
2560         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2561     }
2562
2563     return $raw;
2564 }
2565
2566 sub load_defaults {
2567
2568     my $res = {};
2569
2570     # we use static defaults from our JSON schema configuration
2571     foreach my $key (keys %$confdesc) {
2572         if (defined(my $default = $confdesc->{$key}->{default})) {
2573             $res->{$key} = $default;
2574         }
2575     }
2576
2577     return $res;
2578 }
2579
2580 sub config_list {
2581     my $vmlist = PVE::Cluster::get_vmlist();
2582     my $res = {};
2583     return $res if !$vmlist || !$vmlist->{ids};
2584     my $ids = $vmlist->{ids};
2585
2586     foreach my $vmid (keys %$ids) {
2587         my $d = $ids->{$vmid};
2588         next if !$d->{node} || $d->{node} ne $nodename;
2589         next if !$d->{type} || $d->{type} ne 'qemu';
2590         $res->{$vmid}->{exists} = 1;
2591     }
2592     return $res;
2593 }
2594
2595 # test if VM uses local resources (to prevent migration)
2596 sub check_local_resources {
2597     my ($conf, $noerr) = @_;
2598
2599     my $loc_res = 0;
2600
2601     $loc_res = 1 if $conf->{hostusb}; # old syntax
2602     $loc_res = 1 if $conf->{hostpci}; # old syntax
2603
2604     foreach my $k (keys %$conf) {
2605         next if $k =~ m/^usb/ && ($conf->{$k} eq 'spice');
2606         # sockets are safe: they will recreated be on the target side post-migrate
2607         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2608         $loc_res = 1 if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2609     }
2610
2611     die "VM uses local resources\n" if $loc_res && !$noerr;
2612
2613     return $loc_res;
2614 }
2615
2616 # check if used storages are available on all nodes (use by migrate)
2617 sub check_storage_availability {
2618     my ($storecfg, $conf, $node) = @_;
2619
2620     foreach_drive($conf, sub {
2621         my ($ds, $drive) = @_;
2622
2623         my $volid = $drive->{file};
2624         return if !$volid;
2625
2626         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2627         return if !$sid;
2628
2629         # check if storage is available on both nodes
2630         my $scfg = PVE::Storage::storage_check_node($storecfg, $sid);
2631         PVE::Storage::storage_check_node($storecfg, $sid, $node);
2632    });
2633 }
2634
2635 # list nodes where all VM images are available (used by has_feature API)
2636 sub shared_nodes {
2637     my ($conf, $storecfg) = @_;
2638
2639     my $nodelist = PVE::Cluster::get_nodelist();
2640     my $nodehash = { map { $_ => 1 } @$nodelist };
2641     my $nodename = PVE::INotify::nodename();
2642
2643     foreach_drive($conf, sub {
2644         my ($ds, $drive) = @_;
2645
2646         my $volid = $drive->{file};
2647         return if !$volid;
2648
2649         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2650         if ($storeid) {
2651             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2652             if ($scfg->{disable}) {
2653                 $nodehash = {};
2654             } elsif (my $avail = $scfg->{nodes}) {
2655                 foreach my $node (keys %$nodehash) {
2656                     delete $nodehash->{$node} if !$avail->{$node};
2657                 }
2658             } elsif (!$scfg->{shared}) {
2659                 foreach my $node (keys %$nodehash) {
2660                     delete $nodehash->{$node} if $node ne $nodename
2661                 }
2662             }
2663         }
2664     });
2665
2666     return $nodehash
2667 }
2668
2669 sub check_cmdline {
2670     my ($pidfile, $pid) = @_;
2671
2672     my $fh = IO::File->new("/proc/$pid/cmdline", "r");
2673     if (defined($fh)) {
2674         my $line = <$fh>;
2675         $fh->close;
2676         return undef if !$line;
2677         my @param = split(/\0/, $line);
2678
2679         my $cmd = $param[0];
2680         return if !$cmd || ($cmd !~ m|kvm$| && $cmd !~ m|qemu-system-x86_64$|);
2681
2682         for (my $i = 0; $i < scalar (@param); $i++) {
2683             my $p = $param[$i];
2684             next if !$p;
2685             if (($p eq '-pidfile') || ($p eq '--pidfile')) {
2686                 my $p = $param[$i+1];
2687                 return 1 if $p && ($p eq $pidfile);
2688                 return undef;
2689             }
2690         }
2691     }
2692     return undef;
2693 }
2694
2695 sub check_running {
2696     my ($vmid, $nocheck, $node) = @_;
2697
2698     my $filename = PVE::QemuConfig->config_file($vmid, $node);
2699
2700     die "unable to find configuration file for VM $vmid - no such machine\n"
2701         if !$nocheck && ! -f $filename;
2702
2703     my $pidfile = pidfile_name($vmid);
2704
2705     if (my $fd = IO::File->new("<$pidfile")) {
2706         my $st = stat($fd);
2707         my $line = <$fd>;
2708         close($fd);
2709
2710         my $mtime = $st->mtime;
2711         if ($mtime > time()) {
2712             warn "file '$filename' modified in future\n";
2713         }
2714
2715         if ($line =~ m/^(\d+)$/) {
2716             my $pid = $1;
2717             if (check_cmdline($pidfile, $pid)) {
2718                 if (my $pinfo = PVE::ProcFSTools::check_process_running($pid)) {
2719                     return $pid;
2720                 }
2721             }
2722         }
2723     }
2724
2725     return undef;
2726 }
2727
2728 sub vzlist {
2729
2730     my $vzlist = config_list();
2731
2732     my $fd = IO::Dir->new($var_run_tmpdir) || return $vzlist;
2733
2734     while (defined(my $de = $fd->read)) {
2735         next if $de !~ m/^(\d+)\.pid$/;
2736         my $vmid = $1;
2737         next if !defined($vzlist->{$vmid});
2738         if (my $pid = check_running($vmid)) {
2739             $vzlist->{$vmid}->{pid} = $pid;
2740         }
2741     }
2742
2743     return $vzlist;
2744 }
2745
2746 sub disksize {
2747     my ($storecfg, $conf) = @_;
2748
2749     my $bootdisk = $conf->{bootdisk};
2750     return undef if !$bootdisk;
2751     return undef if !is_valid_drivename($bootdisk);
2752
2753     return undef if !$conf->{$bootdisk};
2754
2755     my $drive = parse_drive($bootdisk, $conf->{$bootdisk});
2756     return undef if !defined($drive);
2757
2758     return undef if drive_is_cdrom($drive);
2759
2760     my $volid = $drive->{file};
2761     return undef if !$volid;
2762
2763     return $drive->{size};
2764 }
2765
2766 my $last_proc_pid_stat;
2767
2768 # get VM status information
2769 # This must be fast and should not block ($full == false)
2770 # We only query KVM using QMP if $full == true (this can be slow)
2771 sub vmstatus {
2772     my ($opt_vmid, $full) = @_;
2773
2774     my $res = {};
2775
2776     my $storecfg = PVE::Storage::config();
2777
2778     my $list = vzlist();
2779     my $defaults = load_defaults();
2780
2781     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
2782
2783     my $cpucount = $cpuinfo->{cpus} || 1;
2784
2785     foreach my $vmid (keys %$list) {
2786         next if $opt_vmid && ($vmid ne $opt_vmid);
2787
2788         my $cfspath = PVE::QemuConfig->cfs_config_path($vmid);
2789         my $conf = PVE::Cluster::cfs_read_file($cfspath) || {};
2790
2791         my $d = {};
2792         $d->{pid} = $list->{$vmid}->{pid};
2793
2794         # fixme: better status?
2795         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
2796
2797         my $size = disksize($storecfg, $conf);
2798         if (defined($size)) {
2799             $d->{disk} = 0; # no info available
2800             $d->{maxdisk} = $size;
2801         } else {
2802             $d->{disk} = 0;
2803             $d->{maxdisk} = 0;
2804         }
2805
2806         $d->{cpus} = ($conf->{sockets} || $defaults->{sockets})
2807             * ($conf->{cores} || $defaults->{cores});
2808         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
2809         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
2810
2811         $d->{name} = $conf->{name} || "VM $vmid";
2812         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024)
2813             : $defaults->{memory}*(1024*1024);
2814
2815         if ($conf->{balloon}) {
2816             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
2817             $d->{shares} = defined($conf->{shares}) ? $conf->{shares}
2818                 : $defaults->{shares};
2819         }
2820
2821         $d->{uptime} = 0;
2822         $d->{cpu} = 0;
2823         $d->{mem} = 0;
2824
2825         $d->{netout} = 0;
2826         $d->{netin} = 0;
2827
2828         $d->{diskread} = 0;
2829         $d->{diskwrite} = 0;
2830
2831         $d->{template} = PVE::QemuConfig->is_template($conf);
2832
2833         $d->{serial} = 1 if conf_has_serial($conf);
2834
2835         $res->{$vmid} = $d;
2836     }
2837
2838     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
2839     foreach my $dev (keys %$netdev) {
2840         next if $dev !~ m/^tap([1-9]\d*)i/;
2841         my $vmid = $1;
2842         my $d = $res->{$vmid};
2843         next if !$d;
2844
2845         $d->{netout} += $netdev->{$dev}->{receive};
2846         $d->{netin} += $netdev->{$dev}->{transmit};
2847
2848         if ($full) {
2849             $d->{nics}->{$dev}->{netout} = $netdev->{$dev}->{receive};
2850             $d->{nics}->{$dev}->{netin} = $netdev->{$dev}->{transmit};
2851         }
2852
2853     }
2854
2855     my $ctime = gettimeofday;
2856
2857     foreach my $vmid (keys %$list) {
2858
2859         my $d = $res->{$vmid};
2860         my $pid = $d->{pid};
2861         next if !$pid;
2862
2863         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
2864         next if !$pstat; # not running
2865
2866         my $used = $pstat->{utime} + $pstat->{stime};
2867
2868         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
2869
2870         if ($pstat->{vsize}) {
2871             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
2872         }
2873
2874         my $old = $last_proc_pid_stat->{$pid};
2875         if (!$old) {
2876             $last_proc_pid_stat->{$pid} = {
2877                 time => $ctime,
2878                 used => $used,
2879                 cpu => 0,
2880             };
2881             next;
2882         }
2883
2884         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
2885
2886         if ($dtime > 1000) {
2887             my $dutime = $used -  $old->{used};
2888
2889             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
2890             $last_proc_pid_stat->{$pid} = {
2891                 time => $ctime,
2892                 used => $used,
2893                 cpu => $d->{cpu},
2894             };
2895         } else {
2896             $d->{cpu} = $old->{cpu};
2897         }
2898     }
2899
2900     return $res if !$full;
2901
2902     my $qmpclient = PVE::QMPClient->new();
2903
2904     my $ballooncb = sub {
2905         my ($vmid, $resp) = @_;
2906
2907         my $info = $resp->{'return'};
2908         return if !$info->{max_mem};
2909
2910         my $d = $res->{$vmid};
2911
2912         # use memory assigned to VM
2913         $d->{maxmem} = $info->{max_mem};
2914         $d->{balloon} = $info->{actual};
2915
2916         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
2917             $d->{mem} = $info->{total_mem} - $info->{free_mem};
2918             $d->{freemem} = $info->{free_mem};
2919         }
2920
2921         $d->{ballooninfo} = $info;
2922     };
2923
2924     my $blockstatscb = sub {
2925         my ($vmid, $resp) = @_;
2926         my $data = $resp->{'return'} || [];
2927         my $totalrdbytes = 0;
2928         my $totalwrbytes = 0;
2929
2930         for my $blockstat (@$data) {
2931             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
2932             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
2933
2934             $blockstat->{device} =~ s/drive-//;
2935             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
2936         }
2937         $res->{$vmid}->{diskread} = $totalrdbytes;
2938         $res->{$vmid}->{diskwrite} = $totalwrbytes;
2939     };
2940
2941     my $statuscb = sub {
2942         my ($vmid, $resp) = @_;
2943
2944         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
2945         # this fails if ballon driver is not loaded, so this must be
2946         # the last commnand (following command are aborted if this fails).
2947         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
2948
2949         my $status = 'unknown';
2950         if (!defined($status = $resp->{'return'}->{status})) {
2951             warn "unable to get VM status\n";
2952             return;
2953         }
2954
2955         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
2956     };
2957
2958     foreach my $vmid (keys %$list) {
2959         next if $opt_vmid && ($vmid ne $opt_vmid);
2960         next if !$res->{$vmid}->{pid}; # not running
2961         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
2962     }
2963
2964     $qmpclient->queue_execute(undef, 2);
2965
2966     foreach my $vmid (keys %$list) {
2967         next if $opt_vmid && ($vmid ne $opt_vmid);
2968         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
2969     }
2970
2971     return $res;
2972 }
2973
2974 sub foreach_drive {
2975     my ($conf, $func, @param) = @_;
2976
2977     foreach my $ds (valid_drive_names()) {
2978         next if !defined($conf->{$ds});
2979
2980         my $drive = parse_drive($ds, $conf->{$ds});
2981         next if !$drive;
2982
2983         &$func($ds, $drive, @param);
2984     }
2985 }
2986
2987 sub foreach_volid {
2988     my ($conf, $func, @param) = @_;
2989
2990     my $volhash = {};
2991
2992     my $test_volid = sub {
2993         my ($volid, $is_cdrom, $replicate, $shared, $snapname) = @_;
2994
2995         return if !$volid;
2996
2997         $volhash->{$volid}->{cdrom} //= 1;
2998         $volhash->{$volid}->{cdrom} = 0 if !$is_cdrom;
2999
3000         $volhash->{$volid}->{replicate} //= 0;
3001         $volhash->{$volid}->{replicate} = 1 if $replicate;
3002
3003         $volhash->{$volid}->{shared} //= 0;
3004         $volhash->{$volid}->{shared} = 1 if $shared;
3005
3006         $volhash->{$volid}->{referenced_in_config} //= 0;
3007         $volhash->{$volid}->{referenced_in_config} = 1 if !defined($snapname);
3008
3009         $volhash->{$volid}->{referenced_in_snapshot}->{$snapname} = 1
3010             if defined($snapname);
3011     };
3012
3013     foreach_drive($conf, sub {
3014         my ($ds, $drive) = @_;
3015         $test_volid->($drive->{file}, drive_is_cdrom($drive), $drive->{replicate} // 1, $drive->{shared}, undef);
3016     });
3017
3018     foreach my $snapname (keys %{$conf->{snapshots}}) {
3019         my $snap = $conf->{snapshots}->{$snapname};
3020         $test_volid->($snap->{vmstate}, 0, 1, $snapname);
3021         foreach_drive($snap, sub {
3022             my ($ds, $drive) = @_;
3023             $test_volid->($drive->{file}, drive_is_cdrom($drive), $drive->{replicate} // 1, $drive->{shared}, $snapname);
3024         });
3025     }
3026
3027     foreach my $volid (keys %$volhash) {
3028         &$func($volid, $volhash->{$volid}, @param);
3029     }
3030 }
3031
3032 sub conf_has_serial {
3033     my ($conf) = @_;
3034
3035     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3036         if ($conf->{"serial$i"}) {
3037             return 1;
3038         }
3039     }
3040
3041     return 0;
3042 }
3043
3044 sub vga_conf_has_spice {
3045     my ($vga) = @_;
3046
3047     return 0 if !$vga || $vga !~ m/^qxl([234])?$/;
3048
3049     return $1 || 1;
3050 }
3051
3052 sub config_to_command {
3053     my ($storecfg, $vmid, $conf, $defaults, $forcemachine) = @_;
3054
3055     my $cmd = [];
3056     my $globalFlags = [];
3057     my $machineFlags = [];
3058     my $rtcFlags = [];
3059     my $cpuFlags = [];
3060     my $devices = [];
3061     my $pciaddr = '';
3062     my $bridges = {};
3063     my $kvmver = kvm_user_version();
3064     my $vernum = 0; # unknown
3065     my $ostype = $conf->{ostype};
3066     my $winversion = windows_version($ostype);
3067     my $kvm = $conf->{kvm} // 1;
3068
3069     die "KVM virtualisation configured, but not available. Either disable in VM configuration or enable in BIOS.\n" if (!$cpuinfo->{hvm} && $kvm);
3070
3071     if ($kvmver =~ m/^(\d+)\.(\d+)$/) {
3072         $vernum = $1*1000000+$2*1000;
3073     } elsif ($kvmver =~ m/^(\d+)\.(\d+)\.(\d+)$/) {
3074         $vernum = $1*1000000+$2*1000+$3;
3075     }
3076
3077     die "detected old qemu-kvm binary ($kvmver)\n" if $vernum < 15000;
3078
3079     my $have_ovz = -f '/proc/vz/vestat';
3080
3081     my $q35 = machine_type_is_q35($conf);
3082     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
3083     my $machine_type = $forcemachine || $conf->{machine};
3084     my $use_old_bios_files = undef;
3085     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3086
3087     my $cpuunits = defined($conf->{cpuunits}) ?
3088             $conf->{cpuunits} : $defaults->{cpuunits};
3089
3090     push @$cmd, '/usr/bin/kvm';
3091
3092     push @$cmd, '-id', $vmid;
3093
3094     my $vmname = $conf->{name} || "vm$vmid";
3095
3096     push @$cmd, '-name', $vmname;
3097
3098     my $use_virtio = 0;
3099
3100     my $qmpsocket = qmp_socket($vmid);
3101     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server,nowait";
3102     push @$cmd, '-mon', "chardev=qmp,mode=control";
3103
3104
3105     push @$cmd, '-pidfile' , pidfile_name($vmid);
3106
3107     push @$cmd, '-daemonize';
3108
3109     if ($conf->{smbios1}) {
3110         push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
3111     }
3112
3113     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3114         die "uefi base image not found\n" if ! -f $OVMF_CODE;
3115
3116         my $path;
3117         my $format;
3118         if (my $efidisk = $conf->{efidisk0}) {
3119             my $d = PVE::JSONSchema::parse_property_string($efidisk_fmt, $efidisk);
3120             my ($storeid, $volname) = PVE::Storage::parse_volume_id($d->{file}, 1);
3121             $format = $d->{format};
3122             if ($storeid) {
3123                 $path = PVE::Storage::path($storecfg, $d->{file});
3124                 if (!defined($format)) {
3125                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
3126                     $format = qemu_img_format($scfg, $volname);
3127                 }
3128             } else {
3129                 $path = $d->{file};
3130                 die "efidisk format must be specified\n"
3131                     if !defined($format);
3132             }
3133         } else {
3134             warn "no efidisk configured! Using temporary efivars disk.\n";
3135             $path = "/tmp/$vmid-ovmf.fd";
3136             PVE::Tools::file_copy($OVMF_VARS, $path, -s $OVMF_VARS);
3137             $format = 'raw';
3138         }
3139
3140         push @$cmd, '-drive', "if=pflash,unit=0,format=raw,readonly,file=$OVMF_CODE";
3141         push @$cmd, '-drive', "if=pflash,unit=1,format=$format,id=drive-efidisk0,file=$path";
3142     }
3143
3144
3145     # add usb controllers
3146     my @usbcontrollers = PVE::QemuServer::USB::get_usb_controllers($conf, $bridges, $q35, $usbdesc->{format}, $MAX_USB_DEVICES);
3147     push @$devices, @usbcontrollers if @usbcontrollers;
3148     my $vga = $conf->{vga};
3149
3150     my $qxlnum = vga_conf_has_spice($vga);
3151     $vga = 'qxl' if $qxlnum;
3152
3153     if (!$vga) {
3154         if (qemu_machine_feature_enabled($machine_type, $kvmver, 2, 9)) {
3155             $vga = (!$winversion || $winversion >= 6) ? 'std' : 'cirrus';
3156         } else {
3157             $vga = ($winversion >= 6) ? 'std' : 'cirrus';
3158         }
3159     }
3160
3161     # enable absolute mouse coordinates (needed by vnc)
3162     my $tablet;
3163     if (defined($conf->{tablet})) {
3164         $tablet = $conf->{tablet};
3165     } else {
3166         $tablet = $defaults->{tablet};
3167         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
3168         $tablet = 0 if $vga =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
3169     }
3170
3171     push @$devices, '-device', print_tabletdevice_full($conf) if $tablet;
3172
3173     my $kvm_off = 0;
3174     my $gpu_passthrough;
3175
3176     # host pci devices
3177     for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
3178         my $d = parse_hostpci($conf->{"hostpci$i"});
3179         next if !$d;
3180
3181         my $pcie = $d->{pcie};
3182         if($pcie){
3183             die "q35 machine model is not enabled" if !$q35;
3184             $pciaddr = print_pcie_addr("hostpci$i");
3185         }else{
3186             $pciaddr = print_pci_addr("hostpci$i", $bridges);
3187         }
3188
3189         my $rombar = defined($d->{rombar}) && !$d->{rombar} ? ',rombar=0' : '';
3190         my $romfile = $d->{romfile};
3191
3192         my $xvga = '';
3193         if ($d->{'x-vga'}) {
3194             $xvga = ',x-vga=on';
3195             $kvm_off = 1;
3196             $vga = 'none';
3197             $gpu_passthrough = 1;
3198
3199             if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3200                 $xvga = "";
3201             }
3202         }
3203         my $pcidevices = $d->{pciid};
3204         my $multifunction = 1 if @$pcidevices > 1;
3205
3206         my $j=0;
3207         foreach my $pcidevice (@$pcidevices) {
3208
3209             my $id = "hostpci$i";
3210             $id .= ".$j" if $multifunction;
3211             my $addr = $pciaddr;
3212             $addr .= ".$j" if $multifunction;
3213             my $devicestr = "vfio-pci,host=$pcidevice->{id}.$pcidevice->{function},id=$id$addr";
3214
3215             if($j == 0){
3216                 $devicestr .= "$rombar$xvga";
3217                 $devicestr .= ",multifunction=on" if $multifunction;
3218                 $devicestr .= ",romfile=/usr/share/kvm/$romfile" if $romfile;
3219             }
3220
3221             push @$devices, '-device', $devicestr;
3222             $j++;
3223         }
3224     }
3225
3226     # usb devices
3227     my @usbdevices = PVE::QemuServer::USB::get_usb_devices($conf, $usbdesc->{format}, $MAX_USB_DEVICES);
3228     push @$devices, @usbdevices if @usbdevices;
3229     # serial devices
3230     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3231         if (my $path = $conf->{"serial$i"}) {
3232             if ($path eq 'socket') {
3233                 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
3234                 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server,nowait";
3235                 push @$devices, '-device', "isa-serial,chardev=serial$i";
3236             } else {
3237                 die "no such serial device\n" if ! -c $path;
3238                 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
3239                 push @$devices, '-device', "isa-serial,chardev=serial$i";
3240             }
3241         }
3242     }
3243
3244     # parallel devices
3245     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
3246         if (my $path = $conf->{"parallel$i"}) {
3247             die "no such parallel device\n" if ! -c $path;
3248             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
3249             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
3250             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
3251         }
3252     }
3253
3254
3255     my $sockets = 1;
3256     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3257     $sockets = $conf->{sockets} if  $conf->{sockets};
3258
3259     my $cores = $conf->{cores} || 1;
3260
3261     my $maxcpus = $sockets * $cores;
3262
3263     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
3264
3265     my $allowed_vcpus = $cpuinfo->{cpus};
3266
3267     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n"
3268         if ($allowed_vcpus < $maxcpus);
3269
3270     if($hotplug_features->{cpu} && qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 7)) {
3271
3272         push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3273         for (my $i = 2; $i <= $vcpus; $i++)  {
3274             my $cpustr = print_cpu_device($conf,$i);
3275             push @$cmd, '-device', $cpustr;
3276         }
3277
3278     } else {
3279
3280         push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3281     }
3282     push @$cmd, '-nodefaults';
3283
3284     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
3285
3286     my $bootindex_hash = {};
3287     my $i = 1;
3288     foreach my $o (split(//, $bootorder)) {
3289         $bootindex_hash->{$o} = $i*100;
3290         $i++;
3291     }
3292
3293     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
3294
3295     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
3296
3297     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
3298
3299     push @$cmd, '-vga', $vga if $vga && $vga !~ m/^serial\d+$/; # for kvm 77 and later
3300
3301     if ($vga && $vga !~ m/^serial\d+$/ && $vga ne 'none'){
3302         my $socket = vnc_socket($vmid);
3303         push @$cmd,  '-vnc', "unix:$socket,x509,password";
3304     } else {
3305         push @$cmd, '-nographic';
3306     }
3307
3308     # time drift fix
3309     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
3310
3311     my $useLocaltime = $conf->{localtime};
3312
3313     if ($winversion >= 5) { # windows
3314         $useLocaltime = 1 if !defined($conf->{localtime});
3315
3316         # use time drift fix when acpi is enabled
3317         if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
3318             $tdf = 1 if !defined($conf->{tdf});
3319         }
3320     }
3321
3322     if ($winversion >= 6) {
3323         push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3324         push @$cmd, '-no-hpet';
3325     }
3326
3327     push @$rtcFlags, 'driftfix=slew' if $tdf;
3328
3329     if (!$kvm) {
3330         push @$machineFlags, 'accel=tcg';
3331     }
3332
3333     if ($machine_type) {
3334         push @$machineFlags, "type=${machine_type}";
3335     }
3336
3337     if ($conf->{startdate}) {
3338         push @$rtcFlags, "base=$conf->{startdate}";
3339     } elsif ($useLocaltime) {
3340         push @$rtcFlags, 'base=localtime';
3341     }
3342
3343     my $cpu = $kvm ? "kvm64" : "qemu64";
3344     if (my $cputype = $conf->{cpu}) {
3345         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
3346             or die "Cannot parse cpu description: $cputype\n";
3347         $cpu = $cpuconf->{cputype};
3348         $kvm_off = 1 if $cpuconf->{hidden};
3349
3350         if (defined(my $flags = $cpuconf->{flags})) {
3351             push @$cpuFlags, split(";", $flags);
3352         }
3353     }
3354
3355     push @$cpuFlags , '+lahf_lm' if $cpu eq 'kvm64';
3356
3357     push @$cpuFlags , '-x2apic'
3358         if $conf->{ostype} && $conf->{ostype} eq 'solaris';
3359
3360     push @$cpuFlags, '+sep' if $cpu eq 'kvm64' || $cpu eq 'kvm32';
3361
3362     push @$cpuFlags, '-rdtscp' if $cpu =~ m/^Opteron/;
3363
3364     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3365
3366         push @$cpuFlags , '+kvm_pv_unhalt' if $kvm;
3367         push @$cpuFlags , '+kvm_pv_eoi' if $kvm;
3368     }
3369
3370     add_hyperv_enlightenments($cpuFlags, $winversion, $machine_type, $kvmver, $conf->{bios}, $gpu_passthrough) if $kvm;
3371
3372     push @$cpuFlags, 'enforce' if $cpu ne 'host' && $kvm;
3373
3374     push @$cpuFlags, 'kvm=off' if $kvm_off;
3375
3376     my $cpu_vendor = $cpu_vendor_list->{$cpu} ||
3377         die "internal error"; # should not happen
3378
3379     push @$cpuFlags, "vendor=${cpu_vendor}"
3380         if $cpu_vendor ne 'default';
3381
3382     $cpu .= "," . join(',', @$cpuFlags) if scalar(@$cpuFlags);
3383
3384     push @$cmd, '-cpu', $cpu;
3385
3386     PVE::QemuServer::Memory::config($conf, $vmid, $sockets, $cores, $defaults, $hotplug_features, $cmd);
3387
3388     push @$cmd, '-S' if $conf->{freeze};
3389
3390     push @$cmd, '-k', $conf->{keyboard} if defined($conf->{keyboard});
3391
3392     # enable sound
3393     #my $soundhw = $conf->{soundhw} || $defaults->{soundhw};
3394     #push @$cmd, '-soundhw', 'es1370';
3395     #push @$cmd, '-soundhw', $soundhw if $soundhw;
3396
3397     if($conf->{agent}) {
3398         my $qgasocket = qmp_socket($vmid, 1);
3399         my $pciaddr = print_pci_addr("qga0", $bridges);
3400         push @$devices, '-chardev', "socket,path=$qgasocket,server,nowait,id=qga0";
3401         push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3402         push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3403     }
3404
3405     my $spice_port;
3406
3407     if ($qxlnum) {
3408         if ($qxlnum > 1) {
3409             if ($winversion){
3410                 for(my $i = 1; $i < $qxlnum; $i++){
3411                     my $pciaddr = print_pci_addr("vga$i", $bridges);
3412                     push @$cmd, '-device', "qxl,id=vga$i,ram_size=67108864,vram_size=33554432$pciaddr";
3413                 }
3414             } else {
3415                 # assume other OS works like Linux
3416                 push @$cmd, '-global', 'qxl-vga.ram_size=134217728';
3417                 push @$cmd, '-global', 'qxl-vga.vram_size=67108864';
3418             }
3419         }
3420
3421         my $pciaddr = print_pci_addr("spice", $bridges);
3422
3423         my $nodename = PVE::INotify::nodename();
3424         my $pfamily = PVE::Tools::get_host_address_family($nodename);
3425         my @nodeaddrs = PVE::Tools::getaddrinfo_all('localhost', family => $pfamily);
3426         die "failed to get an ip address of type $pfamily for 'localhost'\n" if !@nodeaddrs;
3427         my $localhost = PVE::Network::addr_to_ip($nodeaddrs[0]->{addr});
3428         $spice_port = PVE::Tools::next_spice_port($pfamily, $localhost);
3429
3430         push @$devices, '-spice', "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";
3431
3432         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3433         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3434         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3435     }
3436
3437     # enable balloon by default, unless explicitly disabled
3438     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3439         $pciaddr = print_pci_addr("balloon0", $bridges);
3440         push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
3441     }
3442
3443     if ($conf->{watchdog}) {
3444         my $wdopts = parse_watchdog($conf->{watchdog});
3445         $pciaddr = print_pci_addr("watchdog", $bridges);
3446         my $watchdog = $wdopts->{model} || 'i6300esb';
3447         push @$devices, '-device', "$watchdog$pciaddr";
3448         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3449     }
3450
3451     my $vollist = [];
3452     my $scsicontroller = {};
3453     my $ahcicontroller = {};
3454     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3455
3456     # Add iscsi initiator name if available
3457     if (my $initiator = get_initiator_name()) {
3458         push @$devices, '-iscsi', "initiator-name=$initiator";
3459     }
3460
3461     foreach_drive($conf, sub {
3462         my ($ds, $drive) = @_;
3463
3464         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3465             push @$vollist, $drive->{file};
3466         }
3467
3468         # ignore efidisk here, already added in bios/fw handling code above
3469         return if $drive->{interface} eq 'efidisk';
3470
3471         $use_virtio = 1 if $ds =~ m/^virtio/;
3472
3473         if (drive_is_cdrom ($drive)) {
3474             if ($bootindex_hash->{d}) {
3475                 $drive->{bootindex} = $bootindex_hash->{d};
3476                 $bootindex_hash->{d} += 1;
3477             }
3478         } else {
3479             if ($bootindex_hash->{c}) {
3480                 $drive->{bootindex} = $bootindex_hash->{c} if $conf->{bootdisk} && ($conf->{bootdisk} eq $ds);
3481                 $bootindex_hash->{c} += 1;
3482             }
3483         }
3484
3485         if($drive->{interface} eq 'virtio'){
3486            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3487         }
3488
3489         if ($drive->{interface} eq 'scsi') {
3490
3491             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
3492
3493             $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges);
3494             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
3495
3496             my $iothread = '';
3497             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
3498                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3499                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3500             } elsif ($drive->{iothread}) {
3501                 warn "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n";
3502             }
3503
3504             my $queues = '';
3505             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
3506                 $queues = ",num_queues=$drive->{queues}";
3507             }
3508
3509             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues" if !$scsicontroller->{$controller};
3510             $scsicontroller->{$controller}=1;
3511         }
3512
3513         if ($drive->{interface} eq 'sata') {
3514            my $controller = int($drive->{index} / $MAX_SATA_DISKS);
3515            $pciaddr = print_pci_addr("ahci$controller", $bridges);
3516            push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr" if !$ahcicontroller->{$controller};
3517            $ahcicontroller->{$controller}=1;
3518         }
3519
3520         my $drive_cmd = print_drive_full($storecfg, $vmid, $drive);
3521         push @$devices, '-drive',$drive_cmd;
3522         push @$devices, '-device', print_drivedevice_full($storecfg, $conf, $vmid, $drive, $bridges);
3523     });
3524
3525     for (my $i = 0; $i < $MAX_NETS; $i++) {
3526          next if !$conf->{"net$i"};
3527          my $d = parse_net($conf->{"net$i"});
3528          next if !$d;
3529
3530          $use_virtio = 1 if $d->{model} eq 'virtio';
3531
3532          if ($bootindex_hash->{n}) {
3533             $d->{bootindex} = $bootindex_hash->{n};
3534             $bootindex_hash->{n} += 1;
3535          }
3536
3537          my $netdevfull = print_netdev_full($vmid,$conf,$d,"net$i");
3538          push @$devices, '-netdev', $netdevfull;
3539
3540          my $netdevicefull = print_netdevice_full($vmid, $conf, $d, "net$i", $bridges, $use_old_bios_files);
3541          push @$devices, '-device', $netdevicefull;
3542     }
3543
3544     if (!$q35) {
3545         # add pci bridges
3546         if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3547            $bridges->{1} = 1;
3548            $bridges->{2} = 1;
3549         }
3550
3551         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
3552
3553         while (my ($k, $v) = each %$bridges) {
3554             $pciaddr = print_pci_addr("pci.$k");
3555             unshift @$devices, '-device', "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr" if $k > 0;
3556         }
3557     }
3558
3559     # add custom args
3560     if ($conf->{args}) {
3561         my $aa = PVE::Tools::split_args($conf->{args});
3562         push @$cmd, @$aa;
3563     }
3564
3565     push @$cmd, @$devices;
3566     push @$cmd, '-rtc', join(',', @$rtcFlags)
3567         if scalar(@$rtcFlags);
3568     push @$cmd, '-machine', join(',', @$machineFlags)
3569         if scalar(@$machineFlags);
3570     push @$cmd, '-global', join(',', @$globalFlags)
3571         if scalar(@$globalFlags);
3572
3573     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
3574 }
3575
3576 sub vnc_socket {
3577     my ($vmid) = @_;
3578     return "${var_run_tmpdir}/$vmid.vnc";
3579 }
3580
3581 sub spice_port {
3582     my ($vmid) = @_;
3583
3584     my $res = vm_mon_cmd($vmid, 'query-spice');
3585
3586     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
3587 }
3588
3589 sub qmp_socket {
3590     my ($vmid, $qga) = @_;
3591     my $sockettype = $qga ? 'qga' : 'qmp';
3592     return "${var_run_tmpdir}/$vmid.$sockettype";
3593 }
3594
3595 sub pidfile_name {
3596     my ($vmid) = @_;
3597     return "${var_run_tmpdir}/$vmid.pid";
3598 }
3599
3600 sub vm_devices_list {
3601     my ($vmid) = @_;
3602
3603     my $res = vm_mon_cmd($vmid, 'query-pci');
3604     my $devices = {};
3605     foreach my $pcibus (@$res) {
3606         foreach my $device (@{$pcibus->{devices}}) {
3607             next if !$device->{'qdev_id'};
3608             if ($device->{'pci_bridge'}) {
3609                 $devices->{$device->{'qdev_id'}} = 1;
3610                 foreach my $bridge_device (@{$device->{'pci_bridge'}->{devices}}) {
3611                     next if !$bridge_device->{'qdev_id'};
3612                     $devices->{$bridge_device->{'qdev_id'}} = 1;
3613                     $devices->{$device->{'qdev_id'}}++;
3614                 }
3615             } else {
3616                 $devices->{$device->{'qdev_id'}} = 1;
3617             }
3618         }
3619     }
3620
3621     my $resblock = vm_mon_cmd($vmid, 'query-block');
3622     foreach my $block (@$resblock) {
3623         if($block->{device} =~ m/^drive-(\S+)/){
3624                 $devices->{$1} = 1;
3625         }
3626     }
3627
3628     my $resmice = vm_mon_cmd($vmid, 'query-mice');
3629     foreach my $mice (@$resmice) {
3630         if ($mice->{name} eq 'QEMU HID Tablet') {
3631             $devices->{tablet} = 1;
3632             last;
3633         }
3634     }
3635
3636     # for usb devices there is no query-usb
3637     # but we can iterate over the entries in
3638     # qom-list path=/machine/peripheral
3639     my $resperipheral = vm_mon_cmd($vmid, 'qom-list', path => '/machine/peripheral');
3640     foreach my $per (@$resperipheral) {
3641         if ($per->{name} =~ m/^usb\d+$/) {
3642             $devices->{$per->{name}} = 1;
3643         }
3644     }
3645
3646     return $devices;
3647 }
3648
3649 sub vm_deviceplug {
3650     my ($storecfg, $conf, $vmid, $deviceid, $device) = @_;
3651
3652     my $q35 = machine_type_is_q35($conf);
3653
3654     my $devices_list = vm_devices_list($vmid);
3655     return 1 if defined($devices_list->{$deviceid});
3656
3657     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid); # add PCI bridge if we need it for the device
3658
3659     if ($deviceid eq 'tablet') {
3660
3661         qemu_deviceadd($vmid, print_tabletdevice_full($conf));
3662
3663     } elsif ($deviceid =~ m/^usb(\d+)$/) {
3664
3665         die "usb hotplug currently not reliable\n";
3666         # since we can't reliably hot unplug all added usb devices
3667         # and usb passthrough disables live migration
3668         # we disable usb hotplugging for now
3669         qemu_deviceadd($vmid, PVE::QemuServer::USB::print_usbdevice_full($conf, $deviceid, $device));
3670
3671     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3672
3673         qemu_iothread_add($vmid, $deviceid, $device);
3674
3675         qemu_driveadd($storecfg, $vmid, $device);
3676         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
3677
3678         qemu_deviceadd($vmid, $devicefull);
3679         eval { qemu_deviceaddverify($vmid, $deviceid); };
3680         if (my $err = $@) {
3681             eval { qemu_drivedel($vmid, $deviceid); };
3682             warn $@ if $@;
3683             die $err;
3684         }
3685
3686     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3687
3688
3689         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
3690         my $pciaddr = print_pci_addr($deviceid);
3691         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
3692
3693         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
3694
3695         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
3696             qemu_iothread_add($vmid, $deviceid, $device);
3697             $devicefull .= ",iothread=iothread-$deviceid";
3698         }
3699
3700         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
3701             $devicefull .= ",num_queues=$device->{queues}";
3702         }
3703
3704         qemu_deviceadd($vmid, $devicefull);
3705         qemu_deviceaddverify($vmid, $deviceid);
3706
3707     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3708
3709         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device);
3710         qemu_driveadd($storecfg, $vmid, $device);
3711
3712         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
3713         eval { qemu_deviceadd($vmid, $devicefull); };
3714         if (my $err = $@) {
3715             eval { qemu_drivedel($vmid, $deviceid); };
3716             warn $@ if $@;
3717             die $err;
3718         }
3719
3720     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3721
3722         return undef if !qemu_netdevadd($vmid, $conf, $device, $deviceid);
3723
3724         my $machine_type = PVE::QemuServer::qemu_machine_pxe($vmid, $conf);
3725         my $use_old_bios_files = undef;
3726         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3727
3728         my $netdevicefull = print_netdevice_full($vmid, $conf, $device, $deviceid, undef, $use_old_bios_files);
3729         qemu_deviceadd($vmid, $netdevicefull);
3730         eval { qemu_deviceaddverify($vmid, $deviceid); };
3731         if (my $err = $@) {
3732             eval { qemu_netdevdel($vmid, $deviceid); };
3733             warn $@ if $@;
3734             die $err;
3735         }
3736
3737     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
3738
3739         my $bridgeid = $2;
3740         my $pciaddr = print_pci_addr($deviceid);
3741         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
3742
3743         qemu_deviceadd($vmid, $devicefull);
3744         qemu_deviceaddverify($vmid, $deviceid);
3745
3746     } else {
3747         die "can't hotplug device '$deviceid'\n";
3748     }
3749
3750     return 1;
3751 }
3752
3753 # fixme: this should raise exceptions on error!
3754 sub vm_deviceunplug {
3755     my ($vmid, $conf, $deviceid) = @_;
3756
3757     my $devices_list = vm_devices_list($vmid);
3758     return 1 if !defined($devices_list->{$deviceid});
3759
3760     die "can't unplug bootdisk" if $conf->{bootdisk} && $conf->{bootdisk} eq $deviceid;
3761
3762     if ($deviceid eq 'tablet') {
3763
3764         qemu_devicedel($vmid, $deviceid);
3765
3766     } elsif ($deviceid =~ m/^usb\d+$/) {
3767
3768         die "usb hotplug currently not reliable\n";
3769         # when unplugging usb devices this way,
3770         # there may be remaining usb controllers/hubs
3771         # so we disable it for now
3772         qemu_devicedel($vmid, $deviceid);
3773         qemu_devicedelverify($vmid, $deviceid);
3774
3775     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3776
3777         qemu_devicedel($vmid, $deviceid);
3778         qemu_devicedelverify($vmid, $deviceid);
3779         qemu_drivedel($vmid, $deviceid);
3780         qemu_iothread_del($conf, $vmid, $deviceid);
3781
3782     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3783
3784         qemu_devicedel($vmid, $deviceid);
3785         qemu_devicedelverify($vmid, $deviceid);
3786         qemu_iothread_del($conf, $vmid, $deviceid);
3787
3788     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3789
3790         qemu_devicedel($vmid, $deviceid);
3791         qemu_drivedel($vmid, $deviceid);
3792         qemu_deletescsihw($conf, $vmid, $deviceid);
3793
3794     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3795
3796         qemu_devicedel($vmid, $deviceid);
3797         qemu_devicedelverify($vmid, $deviceid);
3798         qemu_netdevdel($vmid, $deviceid);
3799
3800     } else {
3801         die "can't unplug device '$deviceid'\n";
3802     }
3803
3804     return 1;
3805 }
3806
3807 sub qemu_deviceadd {
3808     my ($vmid, $devicefull) = @_;
3809
3810     $devicefull = "driver=".$devicefull;
3811     my %options =  split(/[=,]/, $devicefull);
3812
3813     vm_mon_cmd($vmid, "device_add" , %options);
3814 }
3815
3816 sub qemu_devicedel {
3817     my ($vmid, $deviceid) = @_;
3818
3819     my $ret = vm_mon_cmd($vmid, "device_del", id => $deviceid);
3820 }
3821
3822 sub qemu_iothread_add {
3823     my($vmid, $deviceid, $device) = @_;
3824
3825     if ($device->{iothread}) {
3826         my $iothreads = vm_iothreads_list($vmid);
3827         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
3828     }
3829 }
3830
3831 sub qemu_iothread_del {
3832     my($conf, $vmid, $deviceid) = @_;
3833
3834     my $device = parse_drive($deviceid, $conf->{$deviceid});
3835     if ($device->{iothread}) {
3836         my $iothreads = vm_iothreads_list($vmid);
3837         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
3838     }
3839 }
3840
3841 sub qemu_objectadd {
3842     my($vmid, $objectid, $qomtype) = @_;
3843
3844     vm_mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
3845
3846     return 1;
3847 }
3848
3849 sub qemu_objectdel {
3850     my($vmid, $objectid) = @_;
3851
3852     vm_mon_cmd($vmid, "object-del", id => $objectid);
3853
3854     return 1;
3855 }
3856
3857 sub qemu_driveadd {
3858     my ($storecfg, $vmid, $device) = @_;
3859
3860     my $drive = print_drive_full($storecfg, $vmid, $device);
3861     $drive =~ s/\\/\\\\/g;
3862     my $ret = vm_human_monitor_command($vmid, "drive_add auto \"$drive\"");
3863
3864     # If the command succeeds qemu prints: "OK"
3865     return 1 if $ret =~ m/OK/s;
3866
3867     die "adding drive failed: $ret\n";
3868 }
3869
3870 sub qemu_drivedel {
3871     my($vmid, $deviceid) = @_;
3872
3873     my $ret = vm_human_monitor_command($vmid, "drive_del drive-$deviceid");
3874     $ret =~ s/^\s+//;
3875
3876     return 1 if $ret eq "";
3877
3878     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
3879     return 1 if $ret =~ m/Device \'.*?\' not found/s;
3880
3881     die "deleting drive $deviceid failed : $ret\n";
3882 }
3883
3884 sub qemu_deviceaddverify {
3885     my ($vmid, $deviceid) = @_;
3886
3887     for (my $i = 0; $i <= 5; $i++) {
3888          my $devices_list = vm_devices_list($vmid);
3889          return 1 if defined($devices_list->{$deviceid});
3890          sleep 1;
3891     }
3892
3893     die "error on hotplug device '$deviceid'\n";
3894 }
3895
3896
3897 sub qemu_devicedelverify {
3898     my ($vmid, $deviceid) = @_;
3899
3900     # need to verify that the device is correctly removed as device_del
3901     # is async and empty return is not reliable
3902
3903     for (my $i = 0; $i <= 5; $i++) {
3904          my $devices_list = vm_devices_list($vmid);
3905          return 1 if !defined($devices_list->{$deviceid});
3906          sleep 1;
3907     }
3908
3909     die "error on hot-unplugging device '$deviceid'\n";
3910 }
3911
3912 sub qemu_findorcreatescsihw {
3913     my ($storecfg, $conf, $vmid, $device) = @_;
3914
3915     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3916
3917     my $scsihwid="$controller_prefix$controller";
3918     my $devices_list = vm_devices_list($vmid);
3919
3920     if(!defined($devices_list->{$scsihwid})) {
3921         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device);
3922     }
3923
3924     return 1;
3925 }
3926
3927 sub qemu_deletescsihw {
3928     my ($conf, $vmid, $opt) = @_;
3929
3930     my $device = parse_drive($opt, $conf->{$opt});
3931
3932     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
3933         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
3934         return 1;
3935     }
3936
3937     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3938
3939     my $devices_list = vm_devices_list($vmid);
3940     foreach my $opt (keys %{$devices_list}) {
3941         if (PVE::QemuServer::is_valid_drivename($opt)) {
3942             my $drive = PVE::QemuServer::parse_drive($opt, $conf->{$opt});
3943             if($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
3944                 return 1;
3945             }
3946         }
3947     }
3948
3949     my $scsihwid="scsihw$controller";
3950
3951     vm_deviceunplug($vmid, $conf, $scsihwid);
3952
3953     return 1;
3954 }
3955
3956 sub qemu_add_pci_bridge {
3957     my ($storecfg, $conf, $vmid, $device) = @_;
3958
3959     my $bridges = {};
3960
3961     my $bridgeid;
3962
3963     print_pci_addr($device, $bridges);
3964
3965     while (my ($k, $v) = each %$bridges) {
3966         $bridgeid = $k;
3967     }
3968     return 1 if !defined($bridgeid) || $bridgeid < 1;
3969
3970     my $bridge = "pci.$bridgeid";
3971     my $devices_list = vm_devices_list($vmid);
3972
3973     if (!defined($devices_list->{$bridge})) {
3974         vm_deviceplug($storecfg, $conf, $vmid, $bridge);
3975     }
3976
3977     return 1;
3978 }
3979
3980 sub qemu_set_link_status {
3981     my ($vmid, $device, $up) = @_;
3982
3983     vm_mon_cmd($vmid, "set_link", name => $device,
3984                up => $up ? JSON::true : JSON::false);
3985 }
3986
3987 sub qemu_netdevadd {
3988     my ($vmid, $conf, $device, $deviceid) = @_;
3989
3990     my $netdev = print_netdev_full($vmid, $conf, $device, $deviceid, 1);
3991     my %options =  split(/[=,]/, $netdev);
3992
3993     vm_mon_cmd($vmid, "netdev_add",  %options);
3994     return 1;
3995 }
3996
3997 sub qemu_netdevdel {
3998     my ($vmid, $deviceid) = @_;
3999
4000     vm_mon_cmd($vmid, "netdev_del", id => $deviceid);
4001 }
4002
4003 sub qemu_usb_hotplug {
4004     my ($storecfg, $conf, $vmid, $deviceid, $device) = @_;
4005
4006     return if !$device;
4007
4008     # remove the old one first
4009     vm_deviceunplug($vmid, $conf, $deviceid);
4010
4011     # check if xhci controller is necessary and available
4012     if ($device->{usb3}) {
4013
4014         my $devicelist = vm_devices_list($vmid);
4015
4016         if (!$devicelist->{xhci}) {
4017             my $pciaddr = print_pci_addr("xhci");
4018             qemu_deviceadd($vmid, "nec-usb-xhci,id=xhci$pciaddr");
4019         }
4020     }
4021     my $d = parse_usb_device($device->{host});
4022     $d->{usb3} = $device->{usb3};
4023
4024     # add the new one
4025     vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $d);
4026 }
4027
4028 sub qemu_cpu_hotplug {
4029     my ($vmid, $conf, $vcpus) = @_;
4030
4031     my $machine_type = PVE::QemuServer::get_current_qemu_machine($vmid);
4032
4033     my $sockets = 1;
4034     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
4035     $sockets = $conf->{sockets} if  $conf->{sockets};
4036     my $cores = $conf->{cores} || 1;
4037     my $maxcpus = $sockets * $cores;
4038
4039     $vcpus = $maxcpus if !$vcpus;
4040
4041     die "you can't add more vcpus than maxcpus\n"
4042         if $vcpus > $maxcpus;
4043
4044     my $currentvcpus = $conf->{vcpus} || $maxcpus;
4045
4046     if ($vcpus < $currentvcpus) {
4047
4048         if (qemu_machine_feature_enabled ($machine_type, undef, 2, 7)) {
4049
4050             for (my $i = $currentvcpus; $i > $vcpus; $i--) {
4051                 qemu_devicedel($vmid, "cpu$i");
4052                 my $retry = 0;
4053                 my $currentrunningvcpus = undef;
4054                 while (1) {
4055                     $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
4056                     last if scalar(@{$currentrunningvcpus}) == $i-1;
4057                     raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
4058                     $retry++;
4059                     sleep 1;
4060                 }
4061                 #update conf after each succesfull cpu unplug
4062                 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4063                 PVE::QemuConfig->write_config($vmid, $conf);
4064             }
4065         } else {
4066             die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
4067         }
4068
4069         return;
4070     }
4071
4072     my $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
4073     die "vcpus in running vm does not match its configuration\n"
4074         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
4075
4076     if (qemu_machine_feature_enabled ($machine_type, undef, 2, 7)) {
4077
4078         for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
4079             my $cpustr = print_cpu_device($conf, $i);
4080             qemu_deviceadd($vmid, $cpustr);
4081
4082             my $retry = 0;
4083             my $currentrunningvcpus = undef;
4084             while (1) {
4085                 $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
4086                 last if scalar(@{$currentrunningvcpus}) == $i;
4087                 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
4088                 sleep 1;
4089                 $retry++;
4090             }
4091             #update conf after each succesfull cpu hotplug
4092             $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4093             PVE::QemuConfig->write_config($vmid, $conf);
4094         }
4095     } else {
4096
4097         for (my $i = $currentvcpus; $i < $vcpus; $i++) {
4098             vm_mon_cmd($vmid, "cpu-add", id => int($i));
4099         }
4100     }
4101 }
4102
4103 sub qemu_block_set_io_throttle {
4104     my ($vmid, $deviceid,
4105         $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
4106         $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max,
4107         $bps_max_length, $bps_rd_max_length, $bps_wr_max_length,
4108         $iops_max_length, $iops_rd_max_length, $iops_wr_max_length) = @_;
4109
4110     return if !check_running($vmid) ;
4111
4112     vm_mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
4113         bps => int($bps),
4114         bps_rd => int($bps_rd),
4115         bps_wr => int($bps_wr),
4116         iops => int($iops),
4117         iops_rd => int($iops_rd),
4118         iops_wr => int($iops_wr),
4119         bps_max => int($bps_max),
4120         bps_rd_max => int($bps_rd_max),
4121         bps_wr_max => int($bps_wr_max),
4122         iops_max => int($iops_max),
4123         iops_rd_max => int($iops_rd_max),
4124         iops_wr_max => int($iops_wr_max),
4125         bps_max_length => int($bps_max_length),
4126         bps_rd_max_length => int($bps_rd_max_length),
4127         bps_wr_max_length => int($bps_wr_max_length),
4128         iops_max_length => int($iops_max_length),
4129         iops_rd_max_length => int($iops_rd_max_length),
4130         iops_wr_max_length => int($iops_wr_max_length),
4131     );
4132
4133 }
4134
4135 # old code, only used to shutdown old VM after update
4136 sub __read_avail {
4137     my ($fh, $timeout) = @_;
4138
4139     my $sel = new IO::Select;
4140     $sel->add($fh);
4141
4142     my $res = '';
4143     my $buf;
4144
4145     my @ready;
4146     while (scalar (@ready = $sel->can_read($timeout))) {
4147         my $count;
4148         if ($count = $fh->sysread($buf, 8192)) {
4149             if ($buf =~ /^(.*)\(qemu\) $/s) {
4150                 $res .= $1;
4151                 last;
4152             } else {
4153                 $res .= $buf;
4154             }
4155         } else {
4156             if (!defined($count)) {
4157                 die "$!\n";
4158             }
4159             last;
4160         }
4161     }
4162
4163     die "monitor read timeout\n" if !scalar(@ready);
4164
4165     return $res;
4166 }
4167
4168 sub qemu_block_resize {
4169     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4170
4171     my $running = check_running($vmid);
4172
4173     $size = 0 if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4174
4175     return if !$running;
4176
4177     vm_mon_cmd($vmid, "block_resize", device => $deviceid, size => int($size));
4178
4179 }
4180
4181 sub qemu_volume_snapshot {
4182     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4183
4184     my $running = check_running($vmid);
4185
4186     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4187         vm_mon_cmd($vmid, "snapshot-drive", device => $deviceid, name => $snap);
4188     } else {
4189         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4190     }
4191 }
4192
4193 sub qemu_volume_snapshot_delete {
4194     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4195
4196     my $running = check_running($vmid);
4197
4198     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4199         vm_mon_cmd($vmid, "delete-drive-snapshot", device => $deviceid, name => $snap);
4200     } else {
4201         PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4202     }
4203 }
4204
4205 sub set_migration_caps {
4206     my ($vmid) = @_;
4207
4208     my $cap_ref = [];
4209
4210     my $enabled_cap = {
4211         "auto-converge" => 1,
4212         "xbzrle" => 1,
4213         "x-rdma-pin-all" => 0,
4214         "zero-blocks" => 0,
4215         "compress" => 0
4216     };
4217
4218     my $supported_capabilities = vm_mon_cmd_nocheck($vmid, "query-migrate-capabilities");
4219
4220     for my $supported_capability (@$supported_capabilities) {
4221         push @$cap_ref, {
4222             capability => $supported_capability->{capability},
4223             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4224         };
4225     }
4226
4227     vm_mon_cmd_nocheck($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4228 }
4229
4230 my $fast_plug_option = {
4231     'lock' => 1,
4232     'name' => 1,
4233     'onboot' => 1,
4234     'shares' => 1,
4235     'startup' => 1,
4236     'description' => 1,
4237     'protection' => 1,
4238     'vmstatestorage' => 1,
4239 };
4240
4241 # hotplug changes in [PENDING]
4242 # $selection hash can be used to only apply specified options, for
4243 # example: { cores => 1 } (only apply changed 'cores')
4244 # $errors ref is used to return error messages
4245 sub vmconfig_hotplug_pending {
4246     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4247
4248     my $defaults = load_defaults();
4249
4250     # commit values which do not have any impact on running VM first
4251     # Note: those option cannot raise errors, we we do not care about
4252     # $selection and always apply them.
4253
4254     my $add_error = sub {
4255         my ($opt, $msg) = @_;
4256         $errors->{$opt} = "hotplug problem - $msg";
4257     };
4258
4259     my $changes = 0;
4260     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4261         if ($fast_plug_option->{$opt}) {
4262             $conf->{$opt} = $conf->{pending}->{$opt};
4263             delete $conf->{pending}->{$opt};
4264             $changes = 1;
4265         }
4266     }
4267
4268     if ($changes) {
4269         PVE::QemuConfig->write_config($vmid, $conf);
4270         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4271     }
4272
4273     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4274
4275     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4276     while (my ($opt, $force) = each %$pending_delete_hash) {
4277         next if $selection && !$selection->{$opt};
4278         eval {
4279             if ($opt eq 'hotplug') {
4280                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
4281             } elsif ($opt eq 'tablet') {
4282                 die "skip\n" if !$hotplug_features->{usb};
4283                 if ($defaults->{tablet}) {
4284                     vm_deviceplug($storecfg, $conf, $vmid, $opt);
4285                 } else {
4286                     vm_deviceunplug($vmid, $conf, $opt);
4287                 }
4288             } elsif ($opt =~ m/^usb\d+/) {
4289                 die "skip\n";
4290                 # since we cannot reliably hot unplug usb devices
4291                 # we are disabling it
4292                 die "skip\n" if !$hotplug_features->{usb} || $conf->{$opt} =~ m/spice/i;
4293                 vm_deviceunplug($vmid, $conf, $opt);
4294             } elsif ($opt eq 'vcpus') {
4295                 die "skip\n" if !$hotplug_features->{cpu};
4296                 qemu_cpu_hotplug($vmid, $conf, undef);
4297             } elsif ($opt eq 'balloon') {
4298                 # enable balloon device is not hotpluggable
4299                 die "skip\n" if !defined($conf->{balloon}) || $conf->{balloon};
4300             } elsif ($fast_plug_option->{$opt}) {
4301                 # do nothing
4302             } elsif ($opt =~ m/^net(\d+)$/) {
4303                 die "skip\n" if !$hotplug_features->{network};
4304                 vm_deviceunplug($vmid, $conf, $opt);
4305             } elsif (is_valid_drivename($opt)) {
4306                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4307                 vm_deviceunplug($vmid, $conf, $opt);
4308                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4309             } elsif ($opt =~ m/^memory$/) {
4310                 die "skip\n" if !$hotplug_features->{memory};
4311                 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
4312             } elsif ($opt eq 'cpuunits') {
4313                 cgroups_write("cpu", $vmid, "cpu.shares", $defaults->{cpuunits});
4314             } elsif ($opt eq 'cpulimit') {
4315                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", -1);
4316             } else {
4317                 die "skip\n";
4318             }
4319         };
4320         if (my $err = $@) {
4321             &$add_error($opt, $err) if $err ne "skip\n";
4322         } else {
4323             # save new config if hotplug was successful
4324             delete $conf->{$opt};
4325             vmconfig_undelete_pending_option($conf, $opt);
4326             PVE::QemuConfig->write_config($vmid, $conf);
4327             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4328         }
4329     }
4330
4331     my $apply_pending_cloudinit;
4332     $apply_pending_cloudinit = sub {
4333         my ($key, $value) = @_;
4334         $apply_pending_cloudinit = sub {}; # once is enough
4335
4336         my @cloudinit_opts = keys %$confdesc_cloudinit;
4337         foreach my $opt (keys %{$conf->{pending}}) {
4338             next if !grep { $_ eq $opt } @cloudinit_opts;
4339             $conf->{$opt} = delete $conf->{pending}->{$opt};
4340         }
4341
4342         my $new_conf = { %$conf };
4343         $new_conf->{$key} = $value;
4344         PVE::QemuServer::Cloudinit::generate_cloudinitconfig($new_conf, $vmid);
4345     };
4346
4347     foreach my $opt (keys %{$conf->{pending}}) {
4348         next if $selection && !$selection->{$opt};
4349         my $value = $conf->{pending}->{$opt};
4350         eval {
4351             if ($opt eq 'hotplug') {
4352                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
4353             } elsif ($opt eq 'tablet') {
4354                 die "skip\n" if !$hotplug_features->{usb};
4355                 if ($value == 1) {
4356                     vm_deviceplug($storecfg, $conf, $vmid, $opt);
4357                 } elsif ($value == 0) {
4358                     vm_deviceunplug($vmid, $conf, $opt);
4359                 }
4360             } elsif ($opt =~ m/^usb\d+$/) {
4361                 die "skip\n";
4362                 # since we cannot reliably hot unplug usb devices
4363                 # we are disabling it
4364                 die "skip\n" if !$hotplug_features->{usb} || $value =~ m/spice/i;
4365                 my $d = eval { PVE::JSONSchema::parse_property_string($usbdesc->{format}, $value) };
4366                 die "skip\n" if !$d;
4367                 qemu_usb_hotplug($storecfg, $conf, $vmid, $opt, $d);
4368             } elsif ($opt eq 'vcpus') {
4369                 die "skip\n" if !$hotplug_features->{cpu};
4370                 qemu_cpu_hotplug($vmid, $conf, $value);
4371             } elsif ($opt eq 'balloon') {
4372                 # enable/disable balloning device is not hotpluggable
4373                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
4374                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
4375                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
4376
4377                 # allow manual ballooning if shares is set to zero
4378                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
4379                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
4380                     vm_mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4381                 }
4382             } elsif ($opt =~ m/^net(\d+)$/) {
4383                 # some changes can be done without hotplug
4384                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
4385                                     $vmid, $opt, $value);
4386             } elsif (is_valid_drivename($opt)) {
4387                 # some changes can be done without hotplug
4388                 my $drive = parse_drive($opt, $value);
4389                 if (drive_is_cloudinit($drive)) {
4390                     &$apply_pending_cloudinit($opt, $value);
4391                 }
4392                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
4393                                      $vmid, $opt, $value, 1);
4394             } elsif ($opt =~ m/^memory$/) { #dimms
4395                 die "skip\n" if !$hotplug_features->{memory};
4396                 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
4397             } elsif ($opt eq 'cpuunits') {
4398                 cgroups_write("cpu", $vmid, "cpu.shares", $conf->{pending}->{$opt});
4399             } elsif ($opt eq 'cpulimit') {
4400                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
4401                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", $cpulimit);
4402             } else {
4403                 die "skip\n";  # skip non-hot-pluggable options
4404             }
4405         };
4406         if (my $err = $@) {
4407             &$add_error($opt, $err) if $err ne "skip\n";
4408         } else {
4409             # save new config if hotplug was successful
4410             $conf->{$opt} = $value;
4411             delete $conf->{pending}->{$opt};
4412             PVE::QemuConfig->write_config($vmid, $conf);
4413             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4414         }
4415     }
4416 }
4417
4418 sub try_deallocate_drive {
4419     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
4420
4421     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
4422         my $volid = $drive->{file};
4423         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
4424             my $sid = PVE::Storage::parse_volume_id($volid);
4425             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
4426
4427             # check if the disk is really unused
4428             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
4429                 if is_volume_in_use($storecfg, $conf, $key, $volid);
4430             PVE::Storage::vdisk_free($storecfg, $volid);
4431             return 1;
4432         } else {
4433             # If vm is not owner of this disk remove from config
4434             return 1;
4435         }
4436     }
4437
4438     return undef;
4439 }
4440
4441 sub vmconfig_delete_or_detach_drive {
4442     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
4443
4444     my $drive = parse_drive($opt, $conf->{$opt});
4445
4446     my $rpcenv = PVE::RPCEnvironment::get();
4447     my $authuser = $rpcenv->get_user();
4448
4449     if ($force) {
4450         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
4451         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
4452     } else {
4453         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
4454     }
4455 }
4456
4457 sub vmconfig_apply_pending {
4458     my ($vmid, $conf, $storecfg) = @_;
4459
4460     # cold plug
4461
4462     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4463     while (my ($opt, $force) = each %$pending_delete_hash) {
4464         die "internal error" if $opt =~ m/^unused/;
4465         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4466         if (!defined($conf->{$opt})) {
4467             vmconfig_undelete_pending_option($conf, $opt);
4468             PVE::QemuConfig->write_config($vmid, $conf);
4469         } elsif (is_valid_drivename($opt)) {
4470             vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4471             vmconfig_undelete_pending_option($conf, $opt);
4472             delete $conf->{$opt};
4473             PVE::QemuConfig->write_config($vmid, $conf);
4474         } else {
4475             vmconfig_undelete_pending_option($conf, $opt);
4476             delete $conf->{$opt};
4477             PVE::QemuConfig->write_config($vmid, $conf);
4478         }
4479     }
4480
4481     $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4482
4483     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4484         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4485
4486         if (defined($conf->{$opt}) && ($conf->{$opt} eq $conf->{pending}->{$opt})) {
4487             # skip if nothing changed
4488         } elsif (is_valid_drivename($opt)) {
4489             vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
4490                 if defined($conf->{$opt});
4491             $conf->{$opt} = $conf->{pending}->{$opt};
4492         } else {
4493             $conf->{$opt} = $conf->{pending}->{$opt};
4494         }
4495
4496         delete $conf->{pending}->{$opt};
4497         PVE::QemuConfig->write_config($vmid, $conf);
4498     }
4499 }
4500
4501 my $safe_num_ne = sub {
4502     my ($a, $b) = @_;
4503
4504     return 0 if !defined($a) && !defined($b);
4505     return 1 if !defined($a);
4506     return 1 if !defined($b);
4507
4508     return $a != $b;
4509 };
4510
4511 my $safe_string_ne = sub {
4512     my ($a, $b) = @_;
4513
4514     return 0 if !defined($a) && !defined($b);
4515     return 1 if !defined($a);
4516     return 1 if !defined($b);
4517
4518     return $a ne $b;
4519 };
4520
4521 sub vmconfig_update_net {
4522     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value) = @_;
4523
4524     my $newnet = parse_net($value);
4525
4526     if ($conf->{$opt}) {
4527         my $oldnet = parse_net($conf->{$opt});
4528
4529         if (&$safe_string_ne($oldnet->{model}, $newnet->{model}) ||
4530             &$safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
4531             &$safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
4532             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
4533
4534             # for non online change, we try to hot-unplug
4535             die "skip\n" if !$hotplug;
4536             vm_deviceunplug($vmid, $conf, $opt);
4537         } else {
4538
4539             die "internal error" if $opt !~ m/net(\d+)/;
4540             my $iface = "tap${vmid}i$1";
4541
4542             if (&$safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
4543                 &$safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
4544                 &$safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
4545                 &$safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
4546                 PVE::Network::tap_unplug($iface);
4547                 PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
4548             } elsif (&$safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
4549                 # Rate can be applied on its own but any change above needs to
4550                 # include the rate in tap_plug since OVS resets everything.
4551                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
4552             }
4553
4554             if (&$safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
4555                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
4556             }
4557
4558             return 1;
4559         }
4560     }
4561
4562     if ($hotplug) {
4563         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet);
4564     } else {
4565         die "skip\n";
4566     }
4567 }
4568
4569 sub vmconfig_update_disk {
4570     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $force) = @_;
4571
4572     # fixme: do we need force?
4573
4574     my $drive = parse_drive($opt, $value);
4575
4576     if ($conf->{$opt}) {
4577
4578         if (my $old_drive = parse_drive($opt, $conf->{$opt}))  {
4579
4580             my $media = $drive->{media} || 'disk';
4581             my $oldmedia = $old_drive->{media} || 'disk';
4582             die "unable to change media type\n" if $media ne $oldmedia;
4583
4584             if (!drive_is_cdrom($old_drive)) {
4585
4586                 if ($drive->{file} ne $old_drive->{file}) {
4587
4588                     die "skip\n" if !$hotplug;
4589
4590                     # unplug and register as unused
4591                     vm_deviceunplug($vmid, $conf, $opt);
4592                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
4593
4594                 } else {
4595                     # update existing disk
4596
4597                     # skip non hotpluggable value
4598                     if (&$safe_string_ne($drive->{discard}, $old_drive->{discard}) ||
4599                         &$safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
4600                         &$safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
4601                         &$safe_string_ne($drive->{cache}, $old_drive->{cache})) {
4602                         die "skip\n";
4603                     }
4604
4605                     # apply throttle
4606                     if (&$safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
4607                         &$safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
4608                         &$safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
4609                         &$safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
4610                         &$safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
4611                         &$safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
4612                         &$safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
4613                         &$safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
4614                         &$safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
4615                         &$safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
4616                         &$safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
4617                         &$safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max}) ||
4618                         &$safe_num_ne($drive->{bps_max_length}, $old_drive->{bps_max_length}) ||
4619                         &$safe_num_ne($drive->{bps_rd_max_length}, $old_drive->{bps_rd_max_length}) ||
4620                         &$safe_num_ne($drive->{bps_wr_max_length}, $old_drive->{bps_wr_max_length}) ||
4621                         &$safe_num_ne($drive->{iops_max_length}, $old_drive->{iops_max_length}) ||
4622                         &$safe_num_ne($drive->{iops_rd_max_length}, $old_drive->{iops_rd_max_length}) ||
4623                         &$safe_num_ne($drive->{iops_wr_max_length}, $old_drive->{iops_wr_max_length})) {
4624
4625                         qemu_block_set_io_throttle($vmid,"drive-$opt",
4626                                                    ($drive->{mbps} || 0)*1024*1024,
4627                                                    ($drive->{mbps_rd} || 0)*1024*1024,
4628                                                    ($drive->{mbps_wr} || 0)*1024*1024,
4629                                                    $drive->{iops} || 0,
4630                                                    $drive->{iops_rd} || 0,
4631                                                    $drive->{iops_wr} || 0,
4632                                                    ($drive->{mbps_max} || 0)*1024*1024,
4633                                                    ($drive->{mbps_rd_max} || 0)*1024*1024,
4634                                                    ($drive->{mbps_wr_max} || 0)*1024*1024,
4635                                                    $drive->{iops_max} || 0,
4636                                                    $drive->{iops_rd_max} || 0,
4637                                                    $drive->{iops_wr_max} || 0,
4638                                                    $drive->{bps_max_length} || 1,
4639                                                    $drive->{bps_rd_max_length} || 1,
4640                                                    $drive->{bps_wr_max_length} || 1,
4641                                                    $drive->{iops_max_length} || 1,
4642                                                    $drive->{iops_rd_max_length} || 1,
4643                                                    $drive->{iops_wr_max_length} || 1);
4644
4645                     }
4646
4647                     return 1;
4648                 }
4649
4650             } else { # cdrom
4651
4652                 if ($drive->{file} eq 'none') {
4653                     vm_mon_cmd($vmid, "eject",force => JSON::true,device => "drive-$opt");
4654                     if (drive_is_cloudinit($old_drive)) {
4655                         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive);
4656                     }
4657                 } else {
4658                     my $path = get_iso_path($storecfg, $vmid, $drive->{file});
4659                     vm_mon_cmd($vmid, "eject", force => JSON::true,device => "drive-$opt"); # force eject if locked
4660                     vm_mon_cmd($vmid, "change", device => "drive-$opt",target => "$path") if $path;
4661                 }
4662
4663                 return 1;
4664             }
4665         }
4666     }
4667
4668     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
4669     # hotplug new disks
4670     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
4671     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive);
4672 }
4673
4674 sub vm_start {
4675     my ($storecfg, $vmid, $statefile, $skiplock, $migratedfrom, $paused,
4676         $forcemachine, $spice_ticket, $migration_network, $migration_type, $targetstorage) = @_;
4677
4678     PVE::QemuConfig->lock_config($vmid, sub {
4679         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
4680
4681         die "you can't start a vm if it's a template\n" if PVE::QemuConfig->is_template($conf);
4682
4683         PVE::QemuConfig->check_lock($conf) if !$skiplock;
4684
4685         die "VM $vmid already running\n" if check_running($vmid, undef, $migratedfrom);
4686
4687         if (!$statefile && scalar(keys %{$conf->{pending}})) {
4688             vmconfig_apply_pending($vmid, $conf, $storecfg);
4689             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4690         }
4691
4692         PVE::QemuServer::Cloudinit::generate_cloudinitconfig($conf, $vmid);
4693
4694         my $defaults = load_defaults();
4695
4696         # set environment variable useful inside network script
4697         $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
4698
4699         my $local_volumes = {};
4700
4701         if ($targetstorage) {
4702             foreach_drive($conf, sub {
4703                 my ($ds, $drive) = @_;
4704
4705                 return if drive_is_cdrom($drive);
4706
4707                 my $volid = $drive->{file};
4708
4709                 return if !$volid;
4710
4711                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
4712
4713                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
4714                 return if $scfg->{shared};
4715                 $local_volumes->{$ds} = [$volid, $storeid, $volname];
4716             });
4717
4718             my $format = undef;
4719
4720             foreach my $opt (sort keys %$local_volumes) {
4721
4722                 my ($volid, $storeid, $volname) = @{$local_volumes->{$opt}};
4723                 my $drive = parse_drive($opt, $conf->{$opt});
4724
4725                 #if remote storage is specified, use default format
4726                 if ($targetstorage && $targetstorage ne "1") {
4727                     $storeid = $targetstorage;
4728                     my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
4729                     $format = $defFormat;
4730                 } else {
4731                     #else we use same format than original
4732                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
4733                     $format = qemu_img_format($scfg, $volid);
4734                 }
4735
4736                 my $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $format, undef, ($drive->{size}/1024));
4737                 my $newdrive = $drive;
4738                 $newdrive->{format} = $format;
4739                 $newdrive->{file} = $newvolid;
4740                 my $drivestr = PVE::QemuServer::print_drive($vmid, $newdrive);
4741                 $local_volumes->{$opt} = $drivestr;
4742                 #pass drive to conf for command line
4743                 $conf->{$opt} = $drivestr;
4744             }
4745         }
4746
4747         my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine);
4748
4749         my $migrate_port = 0;
4750         my $migrate_uri;
4751         if ($statefile) {
4752             if ($statefile eq 'tcp') {
4753                 my $localip = "localhost";
4754                 my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
4755                 my $nodename = PVE::INotify::nodename();
4756
4757                 if (!defined($migration_type)) {
4758                     if (defined($datacenterconf->{migration}->{type})) {
4759                         $migration_type = $datacenterconf->{migration}->{type};
4760                     } else {
4761                         $migration_type = 'secure';
4762                     }
4763                 }
4764
4765                 if ($migration_type eq 'insecure') {
4766                     my $migrate_network_addr = PVE::Cluster::get_local_migration_ip($migration_network);
4767                     if ($migrate_network_addr) {
4768                         $localip = $migrate_network_addr;
4769                     } else {
4770                         $localip = PVE::Cluster::remote_node_ip($nodename, 1);
4771                     }
4772
4773                     $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
4774                 }
4775
4776                 my $pfamily = PVE::Tools::get_host_address_family($nodename);
4777                 $migrate_port = PVE::Tools::next_migrate_port($pfamily);
4778                 $migrate_uri = "tcp:${localip}:${migrate_port}";
4779                 push @$cmd, '-incoming', $migrate_uri;
4780                 push @$cmd, '-S';
4781
4782             } elsif ($statefile eq 'unix') {
4783                 # should be default for secure migrations as a ssh TCP forward
4784                 # tunnel is not deterministic reliable ready and fails regurarly
4785                 # to set up in time, so use UNIX socket forwards
4786                 my $socket_addr = "/run/qemu-server/$vmid.migrate";
4787                 unlink $socket_addr;
4788
4789                 $migrate_uri = "unix:$socket_addr";
4790
4791                 push @$cmd, '-incoming', $migrate_uri;
4792                 push @$cmd, '-S';
4793
4794             } else {
4795                 push @$cmd, '-loadstate', $statefile;
4796             }
4797         } elsif ($paused) {
4798             push @$cmd, '-S';
4799         }
4800
4801         # host pci devices
4802         for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
4803           my $d = parse_hostpci($conf->{"hostpci$i"});
4804           next if !$d;
4805           my $pcidevices = $d->{pciid};
4806           foreach my $pcidevice (@$pcidevices) {
4807                 my $pciid = $pcidevice->{id}.".".$pcidevice->{function};
4808
4809                 my $info = pci_device_info("0000:$pciid");
4810                 die "IOMMU not present\n" if !check_iommu_support();
4811                 die "no pci device info for device '$pciid'\n" if !$info;
4812                 die "can't unbind/bind pci group to vfio '$pciid'\n" if !pci_dev_group_bind_to_vfio($pciid);
4813                 die "can't reset pci device '$pciid'\n" if $info->{has_fl_reset} and !pci_dev_reset($info);
4814           }
4815         }
4816
4817         PVE::Storage::activate_volumes($storecfg, $vollist);
4818
4819         if (!check_running($vmid, 1)) {
4820             eval {
4821                 run_command(['/bin/systemctl', 'stop', "$vmid.scope"],
4822                     outfunc => sub {}, errfunc => sub {});
4823             };
4824         }
4825
4826         my $cpuunits = defined($conf->{cpuunits}) ? $conf->{cpuunits}
4827                                                   : $defaults->{cpuunits};
4828
4829         my $start_timeout = $conf->{hugepages} ? 300 : 30;
4830         my %run_params = (timeout => $statefile ? undef : $start_timeout, umask => 0077);
4831
4832         my %properties = (
4833             Slice => 'qemu.slice',
4834             KillMode => 'none',
4835             CPUShares => $cpuunits
4836         );
4837
4838         if (my $cpulimit = $conf->{cpulimit}) {
4839             $properties{CPUQuota} = int($cpulimit * 100);
4840         }
4841         $properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
4842
4843         if ($conf->{hugepages}) {
4844
4845             my $code = sub {
4846                 my $hugepages_topology = PVE::QemuServer::Memory::hugepages_topology($conf);
4847                 my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
4848
4849                 PVE::QemuServer::Memory::hugepages_mount();
4850                 PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
4851
4852                 eval  {
4853                     PVE::Tools::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
4854                     run_command($cmd, %run_params);
4855                 };
4856
4857                 if (my $err = $@) {
4858                     PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology);
4859                     die $err;
4860                 }
4861
4862                 PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology);
4863             };
4864             eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
4865
4866         } else {
4867             eval  {
4868                 PVE::Tools::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
4869                 run_command($cmd, %run_params);
4870             };
4871         }
4872
4873         if (my $err = $@) {
4874             # deactivate volumes if start fails
4875             eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
4876             die "start failed: $err";
4877         }
4878
4879         print "migration listens on $migrate_uri\n" if $migrate_uri;
4880
4881         if ($statefile && $statefile ne 'tcp' && $statefile ne 'unix')  {
4882             eval { vm_mon_cmd_nocheck($vmid, "cont"); };
4883             warn $@ if $@;
4884         }
4885
4886         #start nbd server for storage migration
4887         if ($targetstorage) {
4888             my $nodename = PVE::INotify::nodename();
4889             my $migrate_network_addr = PVE::Cluster::get_local_migration_ip($migration_network);
4890             my $localip = $migrate_network_addr ? $migrate_network_addr : PVE::Cluster::remote_node_ip($nodename, 1);
4891             my $pfamily = PVE::Tools::get_host_address_family($nodename);
4892             $migrate_port = PVE::Tools::next_migrate_port($pfamily);
4893
4894             vm_mon_cmd_nocheck($vmid, "nbd-server-start", addr => { type => 'inet', data => { host => "${localip}", port => "${migrate_port}" } } );
4895
4896             $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
4897
4898             foreach my $opt (sort keys %$local_volumes) {
4899                 my $volid = $local_volumes->{$opt};
4900                 vm_mon_cmd_nocheck($vmid, "nbd-server-add", device => "drive-$opt", writable => JSON::true );
4901                 my $migrate_storage_uri = "nbd:${localip}:${migrate_port}:exportname=drive-$opt";
4902                 print "storage migration listens on $migrate_storage_uri volume:$volid\n";
4903             }
4904         }
4905
4906         if ($migratedfrom) {
4907             eval {
4908                 set_migration_caps($vmid);
4909             };
4910             warn $@ if $@;
4911
4912             if ($spice_port) {
4913                 print "spice listens on port $spice_port\n";
4914                 if ($spice_ticket) {
4915                     vm_mon_cmd_nocheck($vmid, "set_password", protocol => 'spice', password => $spice_ticket);
4916                     vm_mon_cmd_nocheck($vmid, "expire_password", protocol => 'spice', time => "+30");
4917                 }
4918             }
4919
4920         } else {
4921             if (!$statefile && (!defined($conf->{balloon}) || $conf->{balloon})) {
4922                 vm_mon_cmd_nocheck($vmid, "balloon", value => $conf->{balloon}*1024*1024)
4923                     if $conf->{balloon};
4924             }
4925
4926             foreach my $opt (keys %$conf) {
4927                 next if $opt !~  m/^net\d+$/;
4928                 my $nicconf = parse_net($conf->{$opt});
4929                 qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
4930             }
4931         }
4932
4933         vm_mon_cmd_nocheck($vmid, 'qom-set',
4934                     path => "machine/peripheral/balloon0",
4935                     property => "guest-stats-polling-interval",
4936                     value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
4937
4938     });
4939 }
4940
4941 sub vm_mon_cmd {
4942     my ($vmid, $execute, %params) = @_;
4943
4944     my $cmd = { execute => $execute, arguments => \%params };
4945     vm_qmp_command($vmid, $cmd);
4946 }
4947
4948 sub vm_mon_cmd_nocheck {
4949     my ($vmid, $execute, %params) = @_;
4950
4951     my $cmd = { execute => $execute, arguments => \%params };
4952     vm_qmp_command($vmid, $cmd, 1);
4953 }
4954
4955 sub vm_qmp_command {
4956     my ($vmid, $cmd, $nocheck) = @_;
4957
4958     my $res;
4959
4960     my $timeout;
4961     if ($cmd->{arguments} && $cmd->{arguments}->{timeout}) {
4962         $timeout = $cmd->{arguments}->{timeout};
4963         delete $cmd->{arguments}->{timeout};
4964     }
4965
4966     eval {
4967         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
4968         my $sname = qmp_socket($vmid);
4969         if (-e $sname) { # test if VM is reasonambe new and supports qmp/qga
4970             my $qmpclient = PVE::QMPClient->new();
4971
4972             $res = $qmpclient->cmd($vmid, $cmd, $timeout);
4973         } else {
4974             die "unable to open monitor socket\n";
4975         }
4976     };
4977     if (my $err = $@) {
4978         syslog("err", "VM $vmid qmp command failed - $err");
4979         die $err;
4980     }
4981
4982     return $res;
4983 }
4984
4985 sub vm_human_monitor_command {
4986     my ($vmid, $cmdline) = @_;
4987
4988     my $res;
4989
4990     my $cmd = {
4991         execute => 'human-monitor-command',
4992         arguments => { 'command-line' => $cmdline},
4993     };
4994
4995     return vm_qmp_command($vmid, $cmd);
4996 }
4997
4998 sub vm_commandline {
4999     my ($storecfg, $vmid) = @_;
5000
5001     my $conf = PVE::QemuConfig->load_config($vmid);
5002
5003     my $defaults = load_defaults();
5004
5005     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults);
5006
5007     return PVE::Tools::cmd2string($cmd);
5008 }
5009
5010 sub vm_reset {
5011     my ($vmid, $skiplock) = @_;
5012
5013     PVE::QemuConfig->lock_config($vmid, sub {
5014
5015         my $conf = PVE::QemuConfig->load_config($vmid);
5016
5017         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5018
5019         vm_mon_cmd($vmid, "system_reset");
5020     });
5021 }
5022
5023 sub get_vm_volumes {
5024     my ($conf) = @_;
5025
5026     my $vollist = [];
5027     foreach_volid($conf, sub {
5028         my ($volid, $attr) = @_;
5029
5030         return if $volid =~ m|^/|;
5031
5032         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5033         return if !$sid;
5034
5035         push @$vollist, $volid;
5036     });
5037
5038     return $vollist;
5039 }
5040
5041 sub vm_stop_cleanup {
5042     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
5043
5044     eval {
5045
5046         if (!$keepActive) {
5047             my $vollist = get_vm_volumes($conf);
5048             PVE::Storage::deactivate_volumes($storecfg, $vollist);
5049         }
5050
5051         foreach my $ext (qw(mon qmp pid vnc qga)) {
5052             unlink "/var/run/qemu-server/${vmid}.$ext";
5053         }
5054
5055         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
5056     };
5057     warn $@ if $@; # avoid errors - just warn
5058 }
5059
5060 # Note: use $nockeck to skip tests if VM configuration file exists.
5061 # We need that when migration VMs to other nodes (files already moved)
5062 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
5063 sub vm_stop {
5064     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
5065
5066     $force = 1 if !defined($force) && !$shutdown;
5067
5068     if ($migratedfrom){
5069         my $pid = check_running($vmid, $nocheck, $migratedfrom);
5070         kill 15, $pid if $pid;
5071         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
5072         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
5073         return;
5074     }
5075
5076     PVE::QemuConfig->lock_config($vmid, sub {
5077
5078         my $pid = check_running($vmid, $nocheck);
5079         return if !$pid;
5080
5081         my $conf;
5082         if (!$nocheck) {
5083             $conf = PVE::QemuConfig->load_config($vmid);
5084             PVE::QemuConfig->check_lock($conf) if !$skiplock;
5085             if (!defined($timeout) && $shutdown && $conf->{startup}) {
5086                 my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
5087                 $timeout = $opts->{down} if $opts->{down};
5088             }
5089         }
5090
5091         $timeout = 60 if !defined($timeout);
5092
5093         eval {
5094             if ($shutdown) {
5095                 if (defined($conf) && $conf->{agent}) {
5096                     vm_qmp_command($vmid, { execute => "guest-shutdown" }, $nocheck);
5097                 } else {
5098                     vm_qmp_command($vmid, { execute => "system_powerdown" }, $nocheck);
5099                 }
5100             } else {
5101                 vm_qmp_command($vmid, { execute => "quit" }, $nocheck);
5102             }
5103         };
5104         my $err = $@;
5105
5106         if (!$err) {
5107             my $count = 0;
5108             while (($count < $timeout) && check_running($vmid, $nocheck)) {
5109                 $count++;
5110                 sleep 1;
5111             }
5112
5113             if ($count >= $timeout) {
5114                 if ($force) {
5115                     warn "VM still running - terminating now with SIGTERM\n";
5116                     kill 15, $pid;
5117                 } else {
5118                     die "VM quit/powerdown failed - got timeout\n";
5119                 }
5120             } else {
5121                 vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5122                 return;
5123             }
5124         } else {
5125             if ($force) {
5126                 warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
5127                 kill 15, $pid;
5128             } else {
5129                 die "VM quit/powerdown failed\n";
5130             }
5131         }
5132
5133         # wait again
5134         $timeout = 10;
5135
5136         my $count = 0;
5137         while (($count < $timeout) && check_running($vmid, $nocheck)) {
5138             $count++;
5139             sleep 1;
5140         }
5141
5142         if ($count >= $timeout) {
5143             warn "VM still running - terminating now with SIGKILL\n";
5144             kill 9, $pid;
5145             sleep 1;
5146         }
5147
5148         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5149    });
5150 }
5151
5152 sub vm_suspend {
5153     my ($vmid, $skiplock) = @_;
5154
5155     PVE::QemuConfig->lock_config($vmid, sub {
5156
5157         my $conf = PVE::QemuConfig->load_config($vmid);
5158
5159         PVE::QemuConfig->check_lock($conf)
5160             if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
5161
5162         vm_mon_cmd($vmid, "stop");
5163     });
5164 }
5165
5166 sub vm_resume {
5167     my ($vmid, $skiplock, $nocheck) = @_;
5168
5169     PVE::QemuConfig->lock_config($vmid, sub {
5170
5171         if (!$nocheck) {
5172
5173             my $conf = PVE::QemuConfig->load_config($vmid);
5174
5175             PVE::QemuConfig->check_lock($conf)
5176                 if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
5177
5178             vm_mon_cmd($vmid, "cont");
5179
5180         } else {
5181             vm_mon_cmd_nocheck($vmid, "cont");
5182         }
5183     });
5184 }
5185
5186 sub vm_sendkey {
5187     my ($vmid, $skiplock, $key) = @_;
5188
5189     PVE::QemuConfig->lock_config($vmid, sub {
5190
5191         my $conf = PVE::QemuConfig->load_config($vmid);
5192
5193         # there is no qmp command, so we use the human monitor command
5194         vm_human_monitor_command($vmid, "sendkey $key");
5195     });
5196 }
5197
5198 sub vm_destroy {
5199     my ($storecfg, $vmid, $skiplock) = @_;
5200
5201     PVE::QemuConfig->lock_config($vmid, sub {
5202
5203         my $conf = PVE::QemuConfig->load_config($vmid);
5204
5205         if (!check_running($vmid)) {
5206             destroy_vm($storecfg, $vmid, undef, $skiplock);
5207         } else {
5208             die "VM $vmid is running - destroy failed\n";
5209         }
5210     });
5211 }
5212
5213 # pci helpers
5214
5215 sub file_write {
5216     my ($filename, $buf) = @_;
5217
5218     my $fh = IO::File->new($filename, "w");
5219     return undef if !$fh;
5220
5221     my $res = print $fh $buf;
5222
5223     $fh->close();
5224
5225     return $res;
5226 }
5227
5228 sub pci_device_info {
5229     my ($name) = @_;
5230
5231     my $res;
5232
5233     return undef if $name !~ m/^([a-f0-9]{4}):([a-f0-9]{2}):([a-f0-9]{2})\.([a-f0-9])$/;
5234     my ($domain, $bus, $slot, $func) = ($1, $2, $3, $4);
5235
5236     my $irq = file_read_firstline("$pcisysfs/devices/$name/irq");
5237     return undef if !defined($irq) || $irq !~ m/^\d+$/;
5238
5239     my $vendor = file_read_firstline("$pcisysfs/devices/$name/vendor");
5240     return undef if !defined($vendor) || $vendor !~ s/^0x//;
5241
5242     my $product = file_read_firstline("$pcisysfs/devices/$name/device");
5243     return undef if !defined($product) || $product !~ s/^0x//;
5244
5245     $res = {
5246         name => $name,
5247         vendor => $vendor,
5248         product => $product,
5249         domain => $domain,
5250         bus => $bus,
5251         slot => $slot,
5252         func => $func,
5253         irq => $irq,
5254         has_fl_reset => -f "$pcisysfs/devices/$name/reset" || 0,
5255     };
5256
5257     return $res;
5258 }
5259
5260 sub pci_dev_reset {
5261     my ($dev) = @_;
5262
5263     my $name = $dev->{name};
5264
5265     my $fn = "$pcisysfs/devices/$name/reset";
5266
5267     return file_write($fn, "1");
5268 }
5269
5270 sub pci_dev_bind_to_vfio {
5271     my ($dev) = @_;
5272
5273     my $name = $dev->{name};
5274
5275     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
5276
5277     if (!-d $vfio_basedir) {
5278         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
5279     }
5280     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
5281
5282     my $testdir = "$vfio_basedir/$name";
5283     return 1 if -d $testdir;
5284
5285     my $data = "$dev->{vendor} $dev->{product}";
5286     return undef if !file_write("$vfio_basedir/new_id", $data);
5287
5288     my $fn = "$pcisysfs/devices/$name/driver/unbind";
5289     if (!file_write($fn, $name)) {
5290         return undef if -f $fn;
5291     }
5292
5293     $fn = "$vfio_basedir/bind";
5294     if (! -d $testdir) {
5295         return undef if !file_write($fn, $name);
5296     }
5297
5298     return -d $testdir;
5299 }
5300
5301 sub pci_dev_group_bind_to_vfio {
5302     my ($pciid) = @_;
5303
5304     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
5305
5306     if (!-d $vfio_basedir) {
5307         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
5308     }
5309     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
5310
5311     # get IOMMU group devices
5312     opendir(my $D, "$pcisysfs/devices/0000:$pciid/iommu_group/devices/") || die "Cannot open iommu_group: $!\n";
5313       my @devs = grep /^0000:/, readdir($D);
5314     closedir($D);
5315
5316     foreach my $pciid (@devs) {
5317         $pciid =~ m/^([:\.\da-f]+)$/ or die "PCI ID $pciid not valid!\n";
5318
5319         # pci bridges, switches or root ports are not supported
5320         # they have a pci_bus subdirectory so skip them
5321         next if (-e "$pcisysfs/devices/$pciid/pci_bus");
5322
5323         my $info = pci_device_info($1);
5324         pci_dev_bind_to_vfio($info) || die "Cannot bind $pciid to vfio\n";
5325     }
5326
5327     return 1;
5328 }
5329
5330 # vzdump restore implementaion
5331
5332 sub tar_archive_read_firstfile {
5333     my $archive = shift;
5334
5335     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
5336
5337     # try to detect archive type first
5338     my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
5339         die "unable to open file '$archive'\n";
5340     my $firstfile = <$fh>;
5341     kill 15, $pid;
5342     close $fh;
5343
5344     die "ERROR: archive contaions no data\n" if !$firstfile;
5345     chomp $firstfile;
5346
5347     return $firstfile;
5348 }
5349
5350 sub tar_restore_cleanup {
5351     my ($storecfg, $statfile) = @_;
5352
5353     print STDERR "starting cleanup\n";
5354
5355     if (my $fd = IO::File->new($statfile, "r")) {
5356         while (defined(my $line = <$fd>)) {
5357             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5358                 my $volid = $2;
5359                 eval {
5360                     if ($volid =~ m|^/|) {
5361                         unlink $volid || die 'unlink failed\n';
5362                     } else {
5363                         PVE::Storage::vdisk_free($storecfg, $volid);
5364                     }
5365                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
5366                 };
5367                 print STDERR "unable to cleanup '$volid' - $@" if $@;
5368             } else {
5369                 print STDERR "unable to parse line in statfile - $line";
5370             }
5371         }
5372         $fd->close();
5373     }
5374 }
5375
5376 sub restore_archive {
5377     my ($archive, $vmid, $user, $opts) = @_;
5378
5379     my $format = $opts->{format};
5380     my $comp;
5381
5382     if ($archive =~ m/\.tgz$/ || $archive =~ m/\.tar\.gz$/) {
5383         $format = 'tar' if !$format;
5384         $comp = 'gzip';
5385     } elsif ($archive =~ m/\.tar$/) {
5386         $format = 'tar' if !$format;
5387     } elsif ($archive =~ m/.tar.lzo$/) {
5388         $format = 'tar' if !$format;
5389         $comp = 'lzop';
5390     } elsif ($archive =~ m/\.vma$/) {
5391         $format = 'vma' if !$format;
5392     } elsif ($archive =~ m/\.vma\.gz$/) {
5393         $format = 'vma' if !$format;
5394         $comp = 'gzip';
5395     } elsif ($archive =~ m/\.vma\.lzo$/) {
5396         $format = 'vma' if !$format;
5397         $comp = 'lzop';
5398     } else {
5399         $format = 'vma' if !$format; # default
5400     }
5401
5402     # try to detect archive format
5403     if ($format eq 'tar') {
5404         return restore_tar_archive($archive, $vmid, $user, $opts);
5405     } else {
5406         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
5407     }
5408 }
5409
5410 sub restore_update_config_line {
5411     my ($outfd, $cookie, $vmid, $map, $line, $unique) = @_;
5412
5413     return if $line =~ m/^\#qmdump\#/;
5414     return if $line =~ m/^\#vzdump\#/;
5415     return if $line =~ m/^lock:/;
5416     return if $line =~ m/^unused\d+:/;
5417     return if $line =~ m/^parent:/;
5418     return if $line =~ m/^template:/; # restored VM is never a template
5419
5420     my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
5421     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
5422         # try to convert old 1.X settings
5423         my ($id, $ind, $ethcfg) = ($1, $2, $3);
5424         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
5425             my ($model, $macaddr) = split(/\=/, $devconfig);
5426             $macaddr = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if !$macaddr || $unique;
5427             my $net = {
5428                 model => $model,
5429                 bridge => "vmbr$ind",
5430                 macaddr => $macaddr,
5431             };
5432             my $netstr = print_net($net);
5433
5434             print $outfd "net$cookie->{netcount}: $netstr\n";
5435             $cookie->{netcount}++;
5436         }
5437     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
5438         my ($id, $netstr) = ($1, $2);
5439         my $net = parse_net($netstr);
5440         $net->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if $net->{macaddr};
5441         $netstr = print_net($net);
5442         print $outfd "$id: $netstr\n";
5443     } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk)\d+):\s*(\S+)\s*$/) {
5444         my $virtdev = $1;
5445         my $value = $3;
5446         my $di = parse_drive($virtdev, $value);
5447         if (defined($di->{backup}) && !$di->{backup}) {
5448             print $outfd "#$line";
5449         } elsif ($map->{$virtdev}) {
5450             delete $di->{format}; # format can change on restore
5451             $di->{file} = $map->{$virtdev};
5452             $value = print_drive($vmid, $di);
5453             print $outfd "$virtdev: $value\n";
5454         } else {
5455             print $outfd $line;
5456         }
5457     } elsif (($line =~ m/^(smbios1: )(.*)/) && $unique) {
5458         my ($uuid, $uuid_str);
5459         UUID::generate($uuid);
5460         UUID::unparse($uuid, $uuid_str);
5461         my $smbios1 = parse_smbios1($2);
5462         $smbios1->{uuid} = $uuid_str;
5463         print $outfd $1.print_smbios1($smbios1)."\n";
5464     } else {
5465         print $outfd $line;
5466     }
5467 }
5468
5469 sub scan_volids {
5470     my ($cfg, $vmid) = @_;
5471
5472     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid);
5473
5474     my $volid_hash = {};
5475     foreach my $storeid (keys %$info) {
5476         foreach my $item (@{$info->{$storeid}}) {
5477             next if !($item->{volid} && $item->{size});
5478             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
5479             $volid_hash->{$item->{volid}} = $item;
5480         }
5481     }
5482
5483     return $volid_hash;
5484 }
5485
5486 sub is_volume_in_use {
5487     my ($storecfg, $conf, $skip_drive, $volid) = @_;
5488
5489     my $path = PVE::Storage::path($storecfg, $volid);
5490
5491     my $scan_config = sub {
5492         my ($cref, $snapname) = @_;
5493
5494         foreach my $key (keys %$cref) {
5495             my $value = $cref->{$key};
5496             if (is_valid_drivename($key)) {
5497                 next if $skip_drive && $key eq $skip_drive;
5498                 my $drive = parse_drive($key, $value);
5499                 next if !$drive || !$drive->{file} || drive_is_cdrom($drive);
5500                 return 1 if $volid eq $drive->{file};
5501                 if ($drive->{file} =~ m!^/!) {
5502                     return 1 if $drive->{file} eq $path;
5503                 } else {
5504                     my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file}, 1);
5505                     next if !$storeid;
5506                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid, 1);
5507                     next if !$scfg;
5508                     return 1 if $path eq PVE::Storage::path($storecfg, $drive->{file}, $snapname);
5509                 }
5510             }
5511         }
5512
5513         return 0;
5514     };
5515
5516     return 1 if &$scan_config($conf);
5517
5518     undef $skip_drive;
5519
5520     foreach my $snapname (keys %{$conf->{snapshots}}) {
5521         return 1 if &$scan_config($conf->{snapshots}->{$snapname}, $snapname);
5522     }
5523
5524     return 0;
5525 }
5526
5527 sub update_disksize {
5528     my ($vmid, $conf, $volid_hash) = @_;
5529
5530     my $changes;
5531
5532     # used and unused disks
5533     my $referenced = {};
5534
5535     # Note: it is allowed to define multiple storages with same path (alias), so
5536     # we need to check both 'volid' and real 'path' (two different volid can point
5537     # to the same path).
5538
5539     my $referencedpath = {};
5540
5541     # update size info
5542     foreach my $opt (keys %$conf) {
5543         if (is_valid_drivename($opt)) {
5544             my $drive = parse_drive($opt, $conf->{$opt});
5545             my $volid = $drive->{file};
5546             next if !$volid;
5547
5548             $referenced->{$volid} = 1;
5549             if ($volid_hash->{$volid} &&
5550                 (my $path = $volid_hash->{$volid}->{path})) {
5551                 $referencedpath->{$path} = 1;
5552             }
5553
5554             next if drive_is_cdrom($drive);
5555             next if !$volid_hash->{$volid};
5556
5557             $drive->{size} = $volid_hash->{$volid}->{size};
5558             my $new = print_drive($vmid, $drive);
5559             if ($new ne $conf->{$opt}) {
5560                 $changes = 1;
5561                 $conf->{$opt} = $new;
5562             }
5563         }
5564     }
5565
5566     # remove 'unusedX' entry if volume is used
5567     foreach my $opt (keys %$conf) {
5568         next if $opt !~ m/^unused\d+$/;
5569         my $volid = $conf->{$opt};
5570         my $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
5571         if ($referenced->{$volid} || ($path && $referencedpath->{$path})) {
5572             $changes = 1;
5573             delete $conf->{$opt};
5574         }
5575
5576         $referenced->{$volid} = 1;
5577         $referencedpath->{$path} = 1 if $path;
5578     }
5579
5580     foreach my $volid (sort keys %$volid_hash) {
5581         next if $volid =~ m/vm-$vmid-state-/;
5582         next if $referenced->{$volid};
5583         my $path = $volid_hash->{$volid}->{path};
5584         next if !$path; # just to be sure
5585         next if $referencedpath->{$path};
5586         $changes = 1;
5587         PVE::QemuConfig->add_unused_volume($conf, $volid);
5588         $referencedpath->{$path} = 1; # avoid to add more than once (aliases)
5589     }
5590
5591     return $changes;
5592 }
5593
5594 sub rescan {
5595     my ($vmid, $nolock) = @_;
5596
5597     my $cfg = PVE::Storage::config();
5598
5599     my $volid_hash = scan_volids($cfg, $vmid);
5600
5601     my $updatefn =  sub {
5602         my ($vmid) = @_;
5603
5604         my $conf = PVE::QemuConfig->load_config($vmid);
5605
5606         PVE::QemuConfig->check_lock($conf);
5607
5608         my $vm_volids = {};
5609         foreach my $volid (keys %$volid_hash) {
5610             my $info = $volid_hash->{$volid};
5611             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
5612         }
5613
5614         my $changes = update_disksize($vmid, $conf, $vm_volids);
5615
5616         PVE::QemuConfig->write_config($vmid, $conf) if $changes;
5617     };
5618
5619     if (defined($vmid)) {
5620         if ($nolock) {
5621             &$updatefn($vmid);
5622         } else {
5623             PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
5624         }
5625     } else {
5626         my $vmlist = config_list();
5627         foreach my $vmid (keys %$vmlist) {
5628             if ($nolock) {
5629                 &$updatefn($vmid);
5630             } else {
5631                 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
5632             }
5633         }
5634     }
5635 }
5636
5637 sub restore_vma_archive {
5638     my ($archive, $vmid, $user, $opts, $comp) = @_;
5639
5640     my $readfrom = $archive;
5641
5642     my $cfg = PVE::Storage::config();
5643     my $commands = [];
5644     my $bwlimit = $opts->{bwlimit};
5645
5646     my $dbg_cmdstring = '';
5647     my $add_pipe = sub {
5648         my ($cmd) = @_;
5649         push @$commands, $cmd;
5650         $dbg_cmdstring .= ' | ' if length($dbg_cmdstring);
5651         $dbg_cmdstring .= PVE::Tools::cmd2string($cmd);
5652         $readfrom = '-';
5653     };
5654
5655     my $input = undef;
5656     if ($archive eq '-') {
5657         $input = '<&STDIN';
5658     } else {
5659         # If we use a backup from a PVE defined storage we also consider that
5660         # storage's rate limit:
5661         my (undef, $volid) = PVE::Storage::path_to_volume_id($cfg, $archive);
5662         if (defined($volid)) {
5663             my ($sid, undef) = PVE::Storage::parse_volume_id($volid);
5664             my $readlimit = PVE::Storage::get_bandwidth_limit('restore', [$sid], $bwlimit);
5665             if ($readlimit) {
5666                 print STDERR "applying read rate limit: $readlimit\n";
5667                 my $cstream = ['cstream', '-t', $readlimit*1024, '--', $readfrom];
5668                 $add_pipe->($cstream);
5669             }
5670         }
5671     }
5672
5673     if ($comp) {
5674         my $cmd;
5675         if ($comp eq 'gzip') {
5676             $cmd = ['zcat', $readfrom];
5677         } elsif ($comp eq 'lzop') {
5678             $cmd = ['lzop', '-d', '-c', $readfrom];
5679         } else {
5680             die "unknown compression method '$comp'\n";
5681         }
5682         $add_pipe->($cmd);
5683     }
5684
5685     my $tmpdir = "/var/tmp/vzdumptmp$$";
5686     rmtree $tmpdir;
5687
5688     # disable interrupts (always do cleanups)
5689     local $SIG{INT} =
5690         local $SIG{TERM} =
5691         local $SIG{QUIT} =
5692         local $SIG{HUP} = sub { warn "got interrupt - ignored\n"; };
5693
5694     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
5695     POSIX::mkfifo($mapfifo, 0600);
5696     my $fifofh;
5697
5698     my $openfifo = sub {
5699         open($fifofh, '>', $mapfifo) || die $!;
5700     };
5701
5702     $add_pipe->(['vma', 'extract', '-v', '-r', $mapfifo, $readfrom, $tmpdir]);
5703
5704     my $oldtimeout;
5705     my $timeout = 5;
5706
5707     my $devinfo = {};
5708
5709     my $rpcenv = PVE::RPCEnvironment::get();
5710
5711     my $conffile = PVE::QemuConfig->config_file($vmid);
5712     my $tmpfn = "$conffile.$$.tmp";
5713
5714     # Note: $oldconf is undef if VM does not exists
5715     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
5716     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
5717
5718     my %storage_limits;
5719
5720     my $print_devmap = sub {
5721         my $virtdev_hash = {};
5722
5723         my $cfgfn = "$tmpdir/qemu-server.conf";
5724
5725         # we can read the config - that is already extracted
5726         my $fh = IO::File->new($cfgfn, "r") ||
5727             "unable to read qemu-server.conf - $!\n";
5728
5729         my $fwcfgfn = "$tmpdir/qemu-server.fw";
5730         if (-f $fwcfgfn) {
5731             my $pve_firewall_dir = '/etc/pve/firewall';
5732             mkdir $pve_firewall_dir; # make sure the dir exists
5733             PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
5734         }
5735
5736         while (defined(my $line = <$fh>)) {
5737             if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
5738                 my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
5739                 die "archive does not contain data for drive '$virtdev'\n"
5740                     if !$devinfo->{$devname};
5741                 if (defined($opts->{storage})) {
5742                     $storeid = $opts->{storage} || 'local';
5743                 } elsif (!$storeid) {
5744                     $storeid = 'local';
5745                 }
5746                 $format = 'raw' if !$format;
5747                 $devinfo->{$devname}->{devname} = $devname;
5748                 $devinfo->{$devname}->{virtdev} = $virtdev;
5749                 $devinfo->{$devname}->{format} = $format;
5750                 $devinfo->{$devname}->{storeid} = $storeid;
5751
5752                 # check permission on storage
5753                 my $pool = $opts->{pool}; # todo: do we need that?
5754                 if ($user ne 'root@pam') {
5755                     $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
5756                 }
5757
5758                 $storage_limits{$storeid} = $bwlimit;
5759
5760                 $virtdev_hash->{$virtdev} = $devinfo->{$devname};
5761             }
5762         }
5763
5764         foreach my $key (keys %storage_limits) {
5765             my $limit = PVE::Storage::get_bandwidth_limit('restore', [$key], $bwlimit);
5766             next if !$limit;
5767             print STDERR "rate limit for storage $key: $limit KiB/s\n";
5768             $storage_limits{$key} = $limit * 1024;
5769         }
5770
5771         foreach my $devname (keys %$devinfo) {
5772             die "found no device mapping information for device '$devname'\n"
5773                 if !$devinfo->{$devname}->{virtdev};
5774         }
5775
5776         # create empty/temp config
5777         if ($oldconf) {
5778             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
5779             foreach_drive($oldconf, sub {
5780                 my ($ds, $drive) = @_;
5781
5782                 return if drive_is_cdrom($drive);
5783
5784                 my $volid = $drive->{file};
5785
5786                 return if !$volid || $volid =~ m|^/|;
5787
5788                 my ($path, $owner) = PVE::Storage::path($cfg, $volid);
5789                 return if !$path || !$owner || ($owner != $vmid);
5790
5791                 # Note: only delete disk we want to restore
5792                 # other volumes will become unused
5793                 if ($virtdev_hash->{$ds}) {
5794                     eval { PVE::Storage::vdisk_free($cfg, $volid); };
5795                     if (my $err = $@) {
5796                         warn $err;
5797                     }
5798                 }
5799             });
5800
5801             # delete vmstate files
5802             # since after the restore we have no snapshots anymore
5803             foreach my $snapname (keys %{$oldconf->{snapshots}}) {
5804                 my $snap = $oldconf->{snapshots}->{$snapname};
5805                 if ($snap->{vmstate}) {
5806                     eval { PVE::Storage::vdisk_free($cfg, $snap->{vmstate}); };
5807                     if (my $err = $@) {
5808                         warn $err;
5809                     }
5810                 }
5811             }
5812         }
5813
5814         my $map = {};
5815         foreach my $virtdev (sort keys %$virtdev_hash) {
5816             my $d = $virtdev_hash->{$virtdev};
5817             my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
5818             my $storeid = $d->{storeid};
5819             my $scfg = PVE::Storage::storage_config($cfg, $storeid);
5820
5821             my $map_opts = '';
5822             if (my $limit = $storage_limits{$storeid}) {
5823                 $map_opts .= "throttling.bps=$limit:throttling.group=$storeid:";
5824             }
5825
5826             # test if requested format is supported
5827             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($cfg, $storeid);
5828             my $supported = grep { $_ eq $d->{format} } @$validFormats;
5829             $d->{format} = $defFormat if !$supported;
5830
5831             my $volid = PVE::Storage::vdisk_alloc($cfg, $storeid, $vmid,
5832                                                   $d->{format}, undef, $alloc_size);
5833             print STDERR "new volume ID is '$volid'\n";
5834             $d->{volid} = $volid;
5835             my $path = PVE::Storage::path($cfg, $volid);
5836
5837             PVE::Storage::activate_volumes($cfg,[$volid]);
5838
5839             my $write_zeros = 1;
5840             if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
5841                 $write_zeros = 0;
5842             }
5843
5844             print $fifofh "${map_opts}format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
5845
5846             print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
5847             $map->{$virtdev} = $volid;
5848         }
5849
5850         $fh->seek(0, 0) || die "seek failed - $!\n";
5851
5852         my $outfd = new IO::File ($tmpfn, "w") ||
5853             die "unable to write config for VM $vmid\n";
5854
5855         my $cookie = { netcount => 0 };
5856         while (defined(my $line = <$fh>)) {
5857             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
5858         }
5859
5860         $fh->close();
5861         $outfd->close();
5862     };
5863
5864     eval {
5865         # enable interrupts
5866         local $SIG{INT} =
5867             local $SIG{TERM} =
5868             local $SIG{QUIT} =
5869             local $SIG{HUP} =
5870             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
5871         local $SIG{ALRM} = sub { die "got timeout\n"; };
5872
5873         $oldtimeout = alarm($timeout);
5874
5875         my $parser = sub {
5876             my $line = shift;
5877
5878             print "$line\n";
5879
5880             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
5881                 my ($dev_id, $size, $devname) = ($1, $2, $3);
5882                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
5883             } elsif ($line =~ m/^CTIME: /) {
5884                 # we correctly received the vma config, so we can disable
5885                 # the timeout now for disk allocation (set to 10 minutes, so
5886                 # that we always timeout if something goes wrong)
5887                 alarm(600);
5888                 &$print_devmap();
5889                 print $fifofh "done\n";
5890                 my $tmp = $oldtimeout || 0;
5891                 $oldtimeout = undef;
5892                 alarm($tmp);
5893                 close($fifofh);
5894             }
5895         };
5896
5897         print "restore vma archive: $dbg_cmdstring\n";
5898         run_command($commands, input => $input, outfunc => $parser, afterfork => $openfifo);
5899     };
5900     my $err = $@;
5901
5902     alarm($oldtimeout) if $oldtimeout;
5903
5904     my $vollist = [];
5905     foreach my $devname (keys %$devinfo) {
5906         my $volid = $devinfo->{$devname}->{volid};
5907         push @$vollist, $volid if $volid;
5908     }
5909
5910     PVE::Storage::deactivate_volumes($cfg, $vollist);
5911
5912     unlink $mapfifo;
5913
5914     if ($err) {
5915         rmtree $tmpdir;
5916         unlink $tmpfn;
5917
5918         foreach my $devname (keys %$devinfo) {
5919             my $volid = $devinfo->{$devname}->{volid};
5920             next if !$volid;
5921             eval {
5922                 if ($volid =~ m|^/|) {
5923                     unlink $volid || die 'unlink failed\n';
5924                 } else {
5925                     PVE::Storage::vdisk_free($cfg, $volid);
5926                 }
5927                 print STDERR "temporary volume '$volid' sucessfuly removed\n";
5928             };
5929             print STDERR "unable to cleanup '$volid' - $@" if $@;
5930         }
5931         die $err;
5932     }
5933
5934     rmtree $tmpdir;
5935
5936     rename($tmpfn, $conffile) ||
5937         die "unable to commit configuration file '$conffile'\n";
5938
5939     PVE::Cluster::cfs_update(); # make sure we read new file
5940
5941     eval { rescan($vmid, 1); };
5942     warn $@ if $@;
5943 }
5944
5945 sub restore_tar_archive {
5946     my ($archive, $vmid, $user, $opts) = @_;
5947
5948     if ($archive ne '-') {
5949         my $firstfile = tar_archive_read_firstfile($archive);
5950         die "ERROR: file '$archive' dos not lock like a QemuServer vzdump backup\n"
5951             if $firstfile ne 'qemu-server.conf';
5952     }
5953
5954     my $storecfg = PVE::Storage::config();
5955
5956     # destroy existing data - keep empty config
5957     my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
5958     destroy_vm($storecfg, $vmid, 1) if -f $vmcfgfn;
5959
5960     my $tocmd = "/usr/lib/qemu-server/qmextract";
5961
5962     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
5963     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
5964     $tocmd .= ' --prealloc' if $opts->{prealloc};
5965     $tocmd .= ' --info' if $opts->{info};
5966
5967     # tar option "xf" does not autodetect compression when read from STDIN,
5968     # so we pipe to zcat
5969     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
5970         PVE::Tools::shellquote("--to-command=$tocmd");
5971
5972     my $tmpdir = "/var/tmp/vzdumptmp$$";
5973     mkpath $tmpdir;
5974
5975     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
5976     local $ENV{VZDUMP_VMID} = $vmid;
5977     local $ENV{VZDUMP_USER} = $user;
5978
5979     my $conffile = PVE::QemuConfig->config_file($vmid);
5980     my $tmpfn = "$conffile.$$.tmp";
5981
5982     # disable interrupts (always do cleanups)
5983     local $SIG{INT} =
5984         local $SIG{TERM} =
5985         local $SIG{QUIT} =
5986         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
5987
5988     eval {
5989         # enable interrupts
5990         local $SIG{INT} =
5991             local $SIG{TERM} =
5992             local $SIG{QUIT} =
5993             local $SIG{HUP} =
5994             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
5995
5996         if ($archive eq '-') {
5997             print "extracting archive from STDIN\n";
5998             run_command($cmd, input => "<&STDIN");
5999         } else {
6000             print "extracting archive '$archive'\n";
6001             run_command($cmd);
6002         }
6003
6004         return if $opts->{info};
6005
6006         # read new mapping
6007         my $map = {};
6008         my $statfile = "$tmpdir/qmrestore.stat";
6009         if (my $fd = IO::File->new($statfile, "r")) {
6010             while (defined (my $line = <$fd>)) {
6011                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
6012                     $map->{$1} = $2 if $1;
6013                 } else {
6014                     print STDERR "unable to parse line in statfile - $line\n";
6015                 }
6016             }
6017             $fd->close();
6018         }
6019
6020         my $confsrc = "$tmpdir/qemu-server.conf";
6021
6022         my $srcfd = new IO::File($confsrc, "r") ||
6023             die "unable to open file '$confsrc'\n";
6024
6025         my $outfd = new IO::File ($tmpfn, "w") ||
6026             die "unable to write config for VM $vmid\n";
6027
6028         my $cookie = { netcount => 0 };
6029         while (defined (my $line = <$srcfd>)) {
6030             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
6031         }
6032
6033         $srcfd->close();
6034         $outfd->close();
6035     };
6036     my $err = $@;
6037
6038     if ($err) {
6039
6040         unlink $tmpfn;
6041
6042         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
6043
6044         die $err;
6045     }
6046
6047     rmtree $tmpdir;
6048
6049     rename $tmpfn, $conffile ||
6050         die "unable to commit configuration file '$conffile'\n";
6051
6052     PVE::Cluster::cfs_update(); # make sure we read new file
6053
6054     eval { rescan($vmid, 1); };
6055     warn $@ if $@;
6056 };
6057
6058 sub foreach_storage_used_by_vm {
6059     my ($conf, $func) = @_;
6060
6061     my $sidhash = {};
6062
6063     foreach_drive($conf, sub {
6064         my ($ds, $drive) = @_;
6065         return if drive_is_cdrom($drive);
6066
6067         my $volid = $drive->{file};
6068
6069         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
6070         $sidhash->{$sid} = $sid if $sid;
6071     });
6072
6073     foreach my $sid (sort keys %$sidhash) {
6074         &$func($sid);
6075     }
6076 }
6077
6078 sub do_snapshots_with_qemu {
6079     my ($storecfg, $volid) = @_;
6080
6081     my $storage_name = PVE::Storage::parse_volume_id($volid);
6082
6083     if ($qemu_snap_storage->{$storecfg->{ids}->{$storage_name}->{type}}
6084         && !$storecfg->{ids}->{$storage_name}->{krbd}){
6085         return 1;
6086     }
6087
6088     if ($volid =~ m/\.(qcow2|qed)$/){
6089         return 1;
6090     }
6091
6092     return undef;
6093 }
6094
6095 sub qga_check_running {
6096     my ($vmid) = @_;
6097
6098     eval { vm_mon_cmd($vmid, "guest-ping", timeout => 3); };
6099     if ($@) {
6100         warn "Qemu Guest Agent is not running - $@";
6101         return 0;
6102     }
6103     return 1;
6104 }
6105
6106 sub template_create {
6107     my ($vmid, $conf, $disk) = @_;
6108
6109     my $storecfg = PVE::Storage::config();
6110
6111     foreach_drive($conf, sub {
6112         my ($ds, $drive) = @_;
6113
6114         return if drive_is_cdrom($drive);
6115         return if $disk && $ds ne $disk;
6116
6117         my $volid = $drive->{file};
6118         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
6119
6120         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
6121         $drive->{file} = $voliddst;
6122         $conf->{$ds} = print_drive($vmid, $drive);
6123         PVE::QemuConfig->write_config($vmid, $conf);
6124     });
6125 }
6126
6127 sub qemu_img_convert {
6128     my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized) = @_;
6129
6130     my $storecfg = PVE::Storage::config();
6131     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
6132     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
6133
6134     if ($src_storeid && $dst_storeid) {
6135
6136         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
6137
6138         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
6139         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6140
6141         my $src_format = qemu_img_format($src_scfg, $src_volname);
6142         my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
6143
6144         my $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
6145         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6146
6147         my $cmd = [];
6148         push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
6149         push @$cmd, '-s', $snapname if($snapname && $src_format eq "qcow2");
6150         push @$cmd, '-t', 'none' if $dst_scfg->{type} eq 'zfspool';
6151         push @$cmd, '-T', 'none' if $src_scfg->{type} eq 'zfspool';
6152         push @$cmd, '-f', $src_format, '-O', $dst_format, $src_path;
6153         if ($is_zero_initialized) {
6154             push @$cmd, "zeroinit:$dst_path";
6155         } else {
6156             push @$cmd, $dst_path;
6157         }
6158
6159         my $parser = sub {
6160             my $line = shift;
6161             if($line =~ m/\((\S+)\/100\%\)/){
6162                 my $percent = $1;
6163                 my $transferred = int($size * $percent / 100);
6164                 my $remaining = $size - $transferred;
6165
6166                 print "transferred: $transferred bytes remaining: $remaining bytes total: $size bytes progression: $percent %\n";
6167             }
6168
6169         };
6170
6171         eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
6172         my $err = $@;
6173         die "copy failed: $err" if $err;
6174     }
6175 }
6176
6177 sub qemu_img_format {
6178     my ($scfg, $volname) = @_;
6179
6180     if ($scfg->{path} && $volname =~ m/\.($QEMU_FORMAT_RE)$/) {
6181         return $1;
6182     } else {
6183         return "raw";
6184     }
6185 }
6186
6187 sub qemu_drive_mirror {
6188     my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $skipcomplete, $qga) = @_;
6189
6190     $jobs = {} if !$jobs;
6191
6192     my $qemu_target;
6193     my $format;
6194     $jobs->{"drive-$drive"} = {};
6195
6196     if ($dst_volid =~ /^nbd:/) {
6197         $qemu_target = $dst_volid;
6198         $format = "nbd";
6199     } else {
6200         my $storecfg = PVE::Storage::config();
6201         my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
6202
6203         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6204
6205         $format = qemu_img_format($dst_scfg, $dst_volname);
6206
6207         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6208
6209         $qemu_target = $is_zero_initialized ? "zeroinit:$dst_path" : $dst_path;
6210     }
6211
6212     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $qemu_target };
6213     $opts->{format} = $format if $format;
6214
6215     print "drive mirror is starting for drive-$drive\n";
6216
6217     eval { vm_mon_cmd($vmid, "drive-mirror", %$opts); }; #if a job already run for this device,it's throw an error
6218
6219     if (my $err = $@) {
6220         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6221         die "mirroring error: $err";
6222     }
6223
6224     qemu_drive_mirror_monitor ($vmid, $vmiddst, $jobs, $skipcomplete, $qga);
6225 }
6226
6227 sub qemu_drive_mirror_monitor {
6228     my ($vmid, $vmiddst, $jobs, $skipcomplete, $qga) = @_;
6229
6230     eval {
6231         my $err_complete = 0;
6232
6233         while (1) {
6234             die "storage migration timed out\n" if $err_complete > 300;
6235
6236             my $stats = vm_mon_cmd($vmid, "query-block-jobs");
6237
6238             my $running_mirror_jobs = {};
6239             foreach my $stat (@$stats) {
6240                 next if $stat->{type} ne 'mirror';
6241                 $running_mirror_jobs->{$stat->{device}} = $stat;
6242             }
6243
6244             my $readycounter = 0;
6245
6246             foreach my $job (keys %$jobs) {
6247
6248                 if(defined($jobs->{$job}->{complete}) && !defined($running_mirror_jobs->{$job})) {
6249                     print "$job : finished\n";
6250                     delete $jobs->{$job};
6251                     next;
6252                 }
6253
6254                 die "$job: mirroring has been cancelled\n" if !defined($running_mirror_jobs->{$job});
6255
6256                 my $busy = $running_mirror_jobs->{$job}->{busy};
6257                 my $ready = $running_mirror_jobs->{$job}->{ready};
6258                 if (my $total = $running_mirror_jobs->{$job}->{len}) {
6259                     my $transferred = $running_mirror_jobs->{$job}->{offset} || 0;
6260                     my $remaining = $total - $transferred;
6261                     my $percent = sprintf "%.2f", ($transferred * 100 / $total);
6262
6263                     print "$job: transferred: $transferred bytes remaining: $remaining bytes total: $total bytes progression: $percent % busy: $busy ready: $ready \n";
6264                 }
6265
6266                 $readycounter++ if $running_mirror_jobs->{$job}->{ready};
6267             }
6268
6269             last if scalar(keys %$jobs) == 0;
6270
6271             if ($readycounter == scalar(keys %$jobs)) {
6272                 print "all mirroring jobs are ready \n";
6273                 last if $skipcomplete; #do the complete later
6274
6275                 if ($vmiddst && $vmiddst != $vmid) {
6276                     my $agent_running = $qga && qga_check_running($vmid);
6277                     if ($agent_running) {
6278                         print "freeze filesystem\n";
6279                         eval { PVE::QemuServer::vm_mon_cmd($vmid, "guest-fsfreeze-freeze"); };
6280                     } else {
6281                         print "suspend vm\n";
6282                         eval { PVE::QemuServer::vm_suspend($vmid, 1); };
6283                     }
6284
6285                     # if we clone a disk for a new target vm, we don't switch the disk
6286                     PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs);
6287
6288                     if ($agent_running) {
6289                         print "unfreeze filesystem\n";
6290                         eval { PVE::QemuServer::vm_mon_cmd($vmid, "guest-fsfreeze-thaw"); };
6291                     } else {
6292                         print "resume vm\n";
6293                         eval {  PVE::QemuServer::vm_resume($vmid, 1, 1); };
6294                     }
6295
6296                     last;
6297                 } else {
6298
6299                     foreach my $job (keys %$jobs) {
6300                         # try to switch the disk if source and destination are on the same guest
6301                         print "$job: Completing block job...\n";
6302
6303                         eval { vm_mon_cmd($vmid, "block-job-complete", device => $job) };
6304                         if ($@ =~ m/cannot be completed/) {
6305                             print "$job: Block job cannot be completed, try again.\n";
6306                             $err_complete++;
6307                         }else {
6308                             print "$job: Completed successfully.\n";
6309                             $jobs->{$job}->{complete} = 1;
6310                         }
6311                     }
6312                 }
6313             }
6314             sleep 1;
6315         }
6316     };
6317     my $err = $@;
6318
6319     if ($err) {
6320         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6321         die "mirroring error: $err";
6322     }
6323
6324 }
6325
6326 sub qemu_blockjobs_cancel {
6327     my ($vmid, $jobs) = @_;
6328
6329     foreach my $job (keys %$jobs) {
6330         print "$job: Cancelling block job\n";
6331         eval { vm_mon_cmd($vmid, "block-job-cancel", device => $job); };
6332         $jobs->{$job}->{cancel} = 1;
6333     }
6334
6335     while (1) {
6336         my $stats = vm_mon_cmd($vmid, "query-block-jobs");
6337
6338         my $running_jobs = {};
6339         foreach my $stat (@$stats) {
6340             $running_jobs->{$stat->{device}} = $stat;
6341         }
6342
6343         foreach my $job (keys %$jobs) {
6344
6345             if (defined($jobs->{$job}->{cancel}) && !defined($running_jobs->{$job})) {
6346                 print "$job: Done.\n";
6347                 delete $jobs->{$job};
6348             }
6349         }
6350
6351         last if scalar(keys %$jobs) == 0;
6352
6353         sleep 1;
6354     }
6355 }
6356
6357 sub clone_disk {
6358     my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
6359         $newvmid, $storage, $format, $full, $newvollist, $jobs, $skipcomplete, $qga) = @_;
6360
6361     my $newvolid;
6362
6363     if (!$full) {
6364         print "create linked clone of drive $drivename ($drive->{file})\n";
6365         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
6366         push @$newvollist, $newvolid;
6367     } else {
6368
6369         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6370         $storeid = $storage if $storage;
6371
6372         my $dst_format = resolve_dst_disk_format($storecfg, $storeid, $volname, $format);
6373         my ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 3);
6374
6375         print "create full clone of drive $drivename ($drive->{file})\n";
6376         my $name = undef;
6377         if (drive_is_cloudinit($drive)) {
6378             $name = "vm-$newvmid-cloudinit";
6379             # cloudinit only supports raw and qcow2 atm:
6380             if ($dst_format eq 'qcow2') {
6381                 $name .= '.qcow2';
6382             } elsif ($dst_format ne 'raw') {
6383                 die "clone: unhandled format for cloudinit image\n";
6384             }
6385         }
6386         $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $newvmid, $dst_format, $name, ($size/1024));
6387         push @$newvollist, $newvolid;
6388
6389         PVE::Storage::activate_volumes($storecfg, [$newvolid]);
6390
6391         my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
6392         if (!$running || $snapname) {
6393             qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit);
6394         } else {
6395
6396             my $kvmver = get_running_qemu_version ($vmid);
6397             if (!qemu_machine_feature_enabled (undef, $kvmver, 2, 7)) {
6398                 die "drive-mirror with iothread requires qemu version 2.7 or higher\n"
6399                     if $drive->{iothread};
6400             }
6401
6402             qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid, $sparseinit, $jobs, $skipcomplete, $qga);
6403         }
6404     }
6405
6406     my ($size) = PVE::Storage::volume_size_info($storecfg, $newvolid, 3);
6407
6408     my $disk = $drive;
6409     $disk->{format} = undef;
6410     $disk->{file} = $newvolid;
6411     $disk->{size} = $size;
6412
6413     return $disk;
6414 }
6415
6416 # this only works if VM is running
6417 sub get_current_qemu_machine {
6418     my ($vmid) = @_;
6419
6420     my $cmd = { execute => 'query-machines', arguments => {} };
6421     my $res = vm_qmp_command($vmid, $cmd);
6422
6423     my ($current, $default);
6424     foreach my $e (@$res) {
6425         $default = $e->{name} if $e->{'is-default'};
6426         $current = $e->{name} if $e->{'is-current'};
6427     }
6428
6429     # fallback to the default machine if current is not supported by qemu
6430     return $current || $default || 'pc';
6431 }
6432
6433 sub get_running_qemu_version {
6434     my ($vmid) = @_;
6435     my $cmd = { execute => 'query-version', arguments => {} };
6436     my $res = vm_qmp_command($vmid, $cmd);
6437     return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
6438 }
6439
6440 sub qemu_machine_feature_enabled {
6441     my ($machine, $kvmver, $version_major, $version_minor) = @_;
6442
6443     my $current_major;
6444     my $current_minor;
6445
6446     if ($machine && $machine =~ m/^(pc(-i440fx|-q35)?-(\d+)\.(\d+))/) {
6447
6448         $current_major = $3;
6449         $current_minor = $4;
6450
6451     } elsif ($kvmver =~ m/^(\d+)\.(\d+)/) {
6452
6453         $current_major = $1;
6454         $current_minor = $2;
6455     }
6456
6457     return 1 if $current_major >= $version_major && $current_minor >= $version_minor;
6458
6459
6460 }
6461
6462 sub qemu_machine_pxe {
6463     my ($vmid, $conf, $machine) = @_;
6464
6465     $machine =  PVE::QemuServer::get_current_qemu_machine($vmid) if !$machine;
6466
6467     foreach my $opt (keys %$conf) {
6468         next if $opt !~ m/^net(\d+)$/;
6469         my $net = PVE::QemuServer::parse_net($conf->{$opt});
6470         next if !$net;
6471         my $romfile = PVE::QemuServer::vm_mon_cmd_nocheck($vmid, 'qom-get', path => $opt, property => 'romfile');
6472         return $machine.".pxe" if $romfile =~ m/pxe/;
6473         last;
6474     }
6475
6476     return $machine;
6477 }
6478
6479 sub qemu_use_old_bios_files {
6480     my ($machine_type) = @_;
6481
6482     return if !$machine_type;
6483
6484     my $use_old_bios_files = undef;
6485
6486     if ($machine_type =~ m/^(\S+)\.pxe$/) {
6487         $machine_type = $1;
6488         $use_old_bios_files = 1;
6489     } else {
6490         my $kvmver = kvm_user_version();
6491         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
6492         # load new efi bios files on migration. So this hack is required to allow
6493         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
6494         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
6495         $use_old_bios_files = !qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 4);
6496     }
6497
6498     return ($use_old_bios_files, $machine_type);
6499 }
6500
6501 sub create_efidisk {
6502     my ($storecfg, $storeid, $vmid, $fmt) = @_;
6503
6504     die "EFI vars default image not found\n" if ! -f $OVMF_VARS;
6505
6506     my $vars_size = PVE::Tools::convert_size(-s $OVMF_VARS, 'b' => 'kb');
6507     my $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $fmt, undef, $vars_size);
6508     PVE::Storage::activate_volumes($storecfg, [$volid]);
6509
6510     my $path = PVE::Storage::path($storecfg, $volid);
6511     eval {
6512         run_command(['/usr/bin/qemu-img', 'convert', '-n', '-f', 'raw', '-O', $fmt, $OVMF_VARS, $path]);
6513     };
6514     die "Copying EFI vars image failed: $@" if $@;
6515
6516     return ($volid, $vars_size);
6517 }
6518
6519 sub lspci {
6520
6521     my $devices = {};
6522
6523     dir_glob_foreach("$pcisysfs/devices", '[a-f0-9]{4}:([a-f0-9]{2}:[a-f0-9]{2})\.([0-9])', sub {
6524             my (undef, $id, $function) = @_;
6525             my $res = { id => $id, function => $function};
6526             push @{$devices->{$id}}, $res;
6527     });
6528
6529     # Entries should be sorted by functions.
6530     foreach my $id (keys %$devices) {
6531         my $dev = $devices->{$id};
6532         $devices->{$id} = [ sort { $a->{function} <=> $b->{function} } @$dev ];
6533     }
6534
6535     return $devices;
6536 }
6537
6538 sub vm_iothreads_list {
6539     my ($vmid) = @_;
6540
6541     my $res = vm_mon_cmd($vmid, 'query-iothreads');
6542
6543     my $iothreads = {};
6544     foreach my $iothread (@$res) {
6545         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
6546     }
6547
6548     return $iothreads;
6549 }
6550
6551 sub scsihw_infos {
6552     my ($conf, $drive) = @_;
6553
6554     my $maxdev = 0;
6555
6556     if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)) {
6557         $maxdev = 7;
6558     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
6559         $maxdev = 1;
6560     } else {
6561         $maxdev = 256;
6562     }
6563
6564     my $controller = int($drive->{index} / $maxdev);
6565     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single') ? "virtioscsi" : "scsihw";
6566
6567     return ($maxdev, $controller, $controller_prefix);
6568 }
6569
6570 sub add_hyperv_enlightenments {
6571     my ($cpuFlags, $winversion, $machine_type, $kvmver, $bios, $gpu_passthrough) = @_;
6572
6573     return if $winversion < 6;
6574     return if $bios && $bios eq 'ovmf' && $winversion < 8;
6575
6576     push @$cpuFlags , 'hv_vendor_id=proxmox' if $gpu_passthrough;
6577
6578     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
6579         push @$cpuFlags , 'hv_spinlocks=0x1fff';
6580         push @$cpuFlags , 'hv_vapic';
6581         push @$cpuFlags , 'hv_time';
6582     } else {
6583         push @$cpuFlags , 'hv_spinlocks=0xffff';
6584     }
6585
6586     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 6)) {
6587         push @$cpuFlags , 'hv_reset';
6588         push @$cpuFlags , 'hv_vpindex';
6589         push @$cpuFlags , 'hv_runtime';
6590     }
6591
6592     if ($winversion >= 7) {
6593         push @$cpuFlags , 'hv_relaxed';
6594     }
6595 }
6596
6597 sub windows_version {
6598     my ($ostype) = @_;
6599
6600     return 0 if !$ostype;
6601
6602     my $winversion = 0;
6603
6604     if($ostype eq 'wxp' || $ostype eq 'w2k3' || $ostype eq 'w2k') {
6605         $winversion = 5;
6606     } elsif($ostype eq 'w2k8' || $ostype eq 'wvista') {
6607         $winversion = 6;
6608     } elsif ($ostype =~ m/^win(\d+)$/) {
6609         $winversion = $1;
6610     }
6611
6612     return $winversion;
6613 }
6614
6615 sub resolve_dst_disk_format {
6616         my ($storecfg, $storeid, $src_volname, $format) = @_;
6617         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
6618
6619         if (!$format) {
6620             # if no target format is specified, use the source disk format as hint
6621             if ($src_volname) {
6622                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6623                 $format = qemu_img_format($scfg, $src_volname);
6624             } else {
6625                 return $defFormat;
6626             }
6627         }
6628
6629         # test if requested format is supported - else use default
6630         my $supported = grep { $_ eq $format } @$validFormats;
6631         $format = $defFormat if !$supported;
6632         return $format;
6633 }
6634
6635 sub resolve_first_disk {
6636     my $conf = shift;
6637     my @disks = PVE::QemuServer::valid_drive_names();
6638     my $firstdisk;
6639     foreach my $ds (reverse @disks) {
6640         next if !$conf->{$ds};
6641         my $disk = PVE::QemuServer::parse_drive($ds, $conf->{$ds});
6642         next if PVE::QemuServer::drive_is_cdrom($disk);
6643         $firstdisk = $ds;
6644     }
6645     return $firstdisk;
6646 }
6647
6648 sub generate_smbios1_uuid {
6649     my ($uuid, $uuid_str);
6650     UUID::generate($uuid);
6651     UUID::unparse($uuid, $uuid_str);
6652     return "uuid=$uuid_str";
6653 }
6654
6655 # bash completion helper
6656
6657 sub complete_backup_archives {
6658     my ($cmdname, $pname, $cvalue) = @_;
6659
6660     my $cfg = PVE::Storage::config();
6661
6662     my $storeid;
6663
6664     if ($cvalue =~ m/^([^:]+):/) {
6665         $storeid = $1;
6666     }
6667
6668     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
6669
6670     my $res = [];
6671     foreach my $id (keys %$data) {
6672         foreach my $item (@{$data->{$id}}) {
6673             next if $item->{format} !~ m/^vma\.(gz|lzo)$/;
6674             push @$res, $item->{volid} if defined($item->{volid});
6675         }
6676     }
6677
6678     return $res;
6679 }
6680
6681 my $complete_vmid_full = sub {
6682     my ($running) = @_;
6683
6684     my $idlist = vmstatus();
6685
6686     my $res = [];
6687
6688     foreach my $id (keys %$idlist) {
6689         my $d = $idlist->{$id};
6690         if (defined($running)) {
6691             next if $d->{template};
6692             next if $running && $d->{status} ne 'running';
6693             next if !$running && $d->{status} eq 'running';
6694         }
6695         push @$res, $id;
6696
6697     }
6698     return $res;
6699 };
6700
6701 sub complete_vmid {
6702     return &$complete_vmid_full();
6703 }
6704
6705 sub complete_vmid_stopped {
6706     return &$complete_vmid_full(0);
6707 }
6708
6709 sub complete_vmid_running {
6710     return &$complete_vmid_full(1);
6711 }
6712
6713 sub complete_storage {
6714
6715     my $cfg = PVE::Storage::config();
6716     my $ids = $cfg->{ids};
6717
6718     my $res = [];
6719     foreach my $sid (keys %$ids) {
6720         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
6721         next if !$ids->{$sid}->{content}->{images};
6722         push @$res, $sid;
6723     }
6724
6725     return $res;
6726 }
6727
6728 1;